drivers/infiniband/hw/hns/hns_roce_hw_v2.c

   1 /*
   2  * Copyright (c) 2016-2017 Hisilicon Limited.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/acpi.h>
  34 #include <linux/etherdevice.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/kernel.h>
  37 #include <linux/types.h>
  38 #include <net/addrconf.h>
  39 #include <rdma/ib_addr.h>
  40 #include <rdma/ib_cache.h>
  41 #include <rdma/ib_umem.h>
  42 #include <rdma/uverbs_ioctl.h>
  43
  44 #include "hnae3.h"
  45 #include "hns_roce_common.h"
  46 #include "hns_roce_device.h"
  47 #include "hns_roce_cmd.h"
  48 #include "hns_roce_hem.h"
  49 #include "hns_roce_hw_v2.h"
  50
  51 static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
  52                                    struct ib_sge *sg)
  53 {
  54         dseg->lkey = cpu_to_le32(sg->lkey);
  55         dseg->addr = cpu_to_le64(sg->addr);
  56         dseg->len  = cpu_to_le32(sg->length);
  57 }
  58
  59 /*
  60  * mapped-value = 1 + real-value
  61  * The hns wr opcode real value is start from 0, In order to distinguish between
  62  * initialized and uninitialized map values, we plus 1 to the actual value when
  63  * defining the mapping, so that the validity can be identified by checking the
  64  * mapped value is greater than 0.
  65  */
  66 #define HR_OPC_MAP(ib_key, hr_key) \
  67                 [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key
  68
  69 static const u32 hns_roce_op_code[] = {
  70         HR_OPC_MAP(RDMA_WRITE,                  RDMA_WRITE),
  71         HR_OPC_MAP(RDMA_WRITE_WITH_IMM,         RDMA_WRITE_WITH_IMM),
  72         HR_OPC_MAP(SEND,                        SEND),
  73         HR_OPC_MAP(SEND_WITH_IMM,               SEND_WITH_IMM),
  74         HR_OPC_MAP(RDMA_READ,                   RDMA_READ),
  75         HR_OPC_MAP(ATOMIC_CMP_AND_SWP,          ATOM_CMP_AND_SWAP),
  76         HR_OPC_MAP(ATOMIC_FETCH_AND_ADD,        ATOM_FETCH_AND_ADD),
  77         HR_OPC_MAP(SEND_WITH_INV,               SEND_WITH_INV),
  78         HR_OPC_MAP(LOCAL_INV,                   LOCAL_INV),
  79         HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP,   ATOM_MSK_CMP_AND_SWAP),
  80         HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD),
  81         HR_OPC_MAP(REG_MR,                      FAST_REG_PMR),
  82 };
  83
  84 static u32 to_hr_opcode(u32 ib_opcode)
  85 {
  86         if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))
  87                 return HNS_ROCE_V2_WQE_OP_MASK;
  88
  89         return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :
  90                                              HNS_ROCE_V2_WQE_OP_MASK;
  91 }
  92
  93 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
  94                          const struct ib_reg_wr *wr)
  95 {
  96         struct hns_roce_wqe_frmr_seg *fseg =
  97                 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
  98         struct hns_roce_mr *mr = to_hr_mr(wr->mr);
  99         u64 pbl_ba;
 100
 101         /* use ib_access_flags */
 102         roce_set_bit(fseg->byte_40, V2_RC_FRMR_WQE_BYTE_40_BIND_EN_S,
 103                      !!(wr->access & IB_ACCESS_MW_BIND));
 104         roce_set_bit(fseg->byte_40, V2_RC_FRMR_WQE_BYTE_40_ATOMIC_S,
 105                      !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
 106         roce_set_bit(fseg->byte_40, V2_RC_FRMR_WQE_BYTE_40_RR_S,
 107                      !!(wr->access & IB_ACCESS_REMOTE_READ));
 108         roce_set_bit(fseg->byte_40, V2_RC_FRMR_WQE_BYTE_40_RW_S,
 109                      !!(wr->access & IB_ACCESS_REMOTE_WRITE));
 110         roce_set_bit(fseg->byte_40, V2_RC_FRMR_WQE_BYTE_40_LW_S,
 111                      !!(wr->access & IB_ACCESS_LOCAL_WRITE));
 112
 113         /* Data structure reuse may lead to confusion */
 114         pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
 115         rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
 116         rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));
 117
 118         rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
 119         rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
 120         rc_sq_wqe->rkey = cpu_to_le32(wr->key);
 121         rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
 122
 123         fseg->pbl_size = cpu_to_le32(mr->npages);
 124         roce_set_field(fseg->byte_40, V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
 125                        V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
 126                        to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
 127         roce_set_bit(fseg->byte_40, V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
 128 }
 129
 130 static void set_atomic_seg(const struct ib_send_wr *wr,
 131                            struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 132                            unsigned int valid_num_sge)
 133 {
 134         struct hns_roce_v2_wqe_data_seg *dseg =
 135                 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 136         struct hns_roce_wqe_atomic_seg *aseg =
 137                 (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
 138
 139         set_data_seg_v2(dseg, wr->sg_list);
 140
 141         if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 142                 aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
 143                 aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);
 144         } else {
 145                 aseg->fetchadd_swap_data =
 146                         cpu_to_le64(atomic_wr(wr)->compare_add);
 147                 aseg->cmp_data = 0;
 148         }
 149
 150         roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 151                        V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
 152 }
 153
 154 static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,
 155                                  const struct ib_send_wr *wr,
 156                                  unsigned int *sge_idx, u32 msg_len)
 157 {
 158         struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
 159         unsigned int dseg_len = sizeof(struct hns_roce_v2_wqe_data_seg);
 160         unsigned int ext_sge_sz = qp->sq.max_gs * dseg_len;
 161         unsigned int left_len_in_pg;
 162         unsigned int idx = *sge_idx;
 163         unsigned int i = 0;
 164         unsigned int len;
 165         void *addr;
 166         void *dseg;
 167
 168         if (msg_len > ext_sge_sz) {
 169                 ibdev_err(ibdev,
 170                           "no enough extended sge space for inline data.\n");
 171                 return -EINVAL;
 172         }
 173
 174         dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
 175         left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;
 176         len = wr->sg_list[0].length;
 177         addr = (void *)(unsigned long)(wr->sg_list[0].addr);
 178
 179         /* When copying data to extended sge space, the left length in page may
 180          * not long enough for current user's sge. So the data should be
 181          * splited into several parts, one in the first page, and the others in
 182          * the subsequent pages.
 183          */
 184         while (1) {
 185                 if (len <= left_len_in_pg) {
 186                         memcpy(dseg, addr, len);
 187
 188                         idx += len / dseg_len;
 189
 190                         i++;
 191                         if (i >= wr->num_sge)
 192                                 break;
 193
 194                         left_len_in_pg -= len;
 195                         len = wr->sg_list[i].length;
 196                         addr = (void *)(unsigned long)(wr->sg_list[i].addr);
 197                         dseg += len;
 198                 } else {
 199                         memcpy(dseg, addr, left_len_in_pg);
 200
 201                         len -= left_len_in_pg;
 202                         addr += left_len_in_pg;
 203                         idx += left_len_in_pg / dseg_len;
 204                         dseg = hns_roce_get_extend_sge(qp,
 205                                                 idx & (qp->sge.sge_cnt - 1));
 206                         left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT;
 207                 }
 208         }
 209
 210         *sge_idx = idx;
 211
 212         return 0;
 213 }
 214
 215 static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge,
 216                            unsigned int *sge_ind, unsigned int cnt)
 217 {
 218         struct hns_roce_v2_wqe_data_seg *dseg;
 219         unsigned int idx = *sge_ind;
 220
 221         while (cnt > 0) {
 222                 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
 223                 if (likely(sge->length)) {
 224                         set_data_seg_v2(dseg, sge);
 225                         idx++;
 226                         cnt--;
 227                 }
 228                 sge++;
 229         }
 230
 231         *sge_ind = idx;
 232 }
 233
 234 static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)
 235 {
 236         struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 237         int mtu = ib_mtu_enum_to_int(qp->path_mtu);
 238
 239         if (len > qp->max_inline_data || len > mtu) {
 240                 ibdev_err(&hr_dev->ib_dev,
 241                           "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",
 242                           len, qp->max_inline_data, mtu);
 243                 return false;
 244         }
 245
 246         return true;
 247 }
 248
 249 static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
 250                       struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 251                       unsigned int *sge_idx)
 252 {
 253         struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 254         u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len);
 255         struct ib_device *ibdev = &hr_dev->ib_dev;
 256         unsigned int curr_idx = *sge_idx;
 257         void *dseg = rc_sq_wqe;
 258         unsigned int i;
 259         int ret;
 260
 261         if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
 262                 ibdev_err(ibdev, "invalid inline parameters!\n");
 263                 return -EINVAL;
 264         }
 265
 266         if (!check_inl_data_len(qp, msg_len))
 267                 return -EINVAL;
 268
 269         dseg += sizeof(struct hns_roce_v2_rc_send_wqe);
 270
 271         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1);
 272
 273         if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {
 274                 roce_set_bit(rc_sq_wqe->byte_20,
 275                              V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 0);
 276
 277                 for (i = 0; i < wr->num_sge; i++) {
 278                         memcpy(dseg, ((void *)wr->sg_list[i].addr),
 279                                wr->sg_list[i].length);
 280                         dseg += wr->sg_list[i].length;
 281                 }
 282         } else {
 283                 roce_set_bit(rc_sq_wqe->byte_20,
 284                              V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 1);
 285
 286                 ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len);
 287                 if (ret)
 288                         return ret;
 289
 290                 roce_set_field(rc_sq_wqe->byte_16,
 291                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 292                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
 293                                curr_idx - *sge_idx);
 294         }
 295
 296         *sge_idx = curr_idx;
 297
 298         return 0;
 299 }
 300
 301 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 302                              struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 303                              unsigned int *sge_ind,
 304                              unsigned int valid_num_sge)
 305 {
 306         struct hns_roce_v2_wqe_data_seg *dseg =
 307                 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 308         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 309         int j = 0;
 310         int i;
 311
 312         roce_set_field(rc_sq_wqe->byte_20,
 313                        V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 314                        V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 315                        (*sge_ind) & (qp->sge.sge_cnt - 1));
 316
 317         if (wr->send_flags & IB_SEND_INLINE)
 318                 return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);
 319
 320         if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
 321                 for (i = 0; i < wr->num_sge; i++) {
 322                         if (likely(wr->sg_list[i].length)) {
 323                                 set_data_seg_v2(dseg, wr->sg_list + i);
 324                                 dseg++;
 325                         }
 326                 }
 327         } else {
 328                 for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) {
 329                         if (likely(wr->sg_list[i].length)) {
 330                                 set_data_seg_v2(dseg, wr->sg_list + i);
 331                                 dseg++;
 332                                 j++;
 333                         }
 334                 }
 335
 336                 set_extend_sge(qp, wr->sg_list + i, sge_ind,
 337                                valid_num_sge - HNS_ROCE_SGE_IN_WQE);
 338         }
 339
 340         roce_set_field(rc_sq_wqe->byte_16,
 341                        V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 342                        V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
 343
 344         return 0;
 345 }
 346
 347 static int check_send_valid(struct hns_roce_dev *hr_dev,
 348                             struct hns_roce_qp *hr_qp)
 349 {
 350         struct ib_device *ibdev = &hr_dev->ib_dev;
 351         struct ib_qp *ibqp = &hr_qp->ibqp;
 352
 353         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
 354                      ibqp->qp_type != IB_QPT_GSI &&
 355                      ibqp->qp_type != IB_QPT_UD)) {
 356                 ibdev_err(ibdev, "Not supported QP(0x%x)type!\n",
 357                           ibqp->qp_type);
 358                 return -EOPNOTSUPP;
 359         } else if (unlikely(hr_qp->state == IB_QPS_RESET ||
 360                    hr_qp->state == IB_QPS_INIT ||
 361                    hr_qp->state == IB_QPS_RTR)) {
 362                 ibdev_err(ibdev, "failed to post WQE, QP state %u!\n",
 363                           hr_qp->state);
 364                 return -EINVAL;
 365         } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {
 366                 ibdev_err(ibdev, "failed to post WQE, dev state %d!\n",
 367                           hr_dev->state);
 368                 return -EIO;
 369         }
 370
 371         return 0;
 372 }
 373
 374 static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
 375                                     unsigned int *sge_len)
 376 {
 377         unsigned int valid_num = 0;
 378         unsigned int len = 0;
 379         int i;
 380
 381         for (i = 0; i < wr->num_sge; i++) {
 382                 if (likely(wr->sg_list[i].length)) {
 383                         len += wr->sg_list[i].length;
 384                         valid_num++;
 385                 }
 386         }
 387
 388         *sge_len = len;
 389         return valid_num;
 390 }
 391
 392 static __le32 get_immtdata(const struct ib_send_wr *wr)
 393 {
 394         switch (wr->opcode) {
 395         case IB_WR_SEND_WITH_IMM:
 396         case IB_WR_RDMA_WRITE_WITH_IMM:
 397                 return cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 398         default:
 399                 return 0;
 400         }
 401 }
 402
 403 static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 404                          const struct ib_send_wr *wr)
 405 {
 406         u32 ib_op = wr->opcode;
 407
 408         if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM)
 409                 return -EINVAL;
 410
 411         ud_sq_wqe->immtdata = get_immtdata(wr);
 412
 413         roce_set_field(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
 414                        V2_UD_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
 415
 416         return 0;
 417 }
 418
 419 static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 420                       struct hns_roce_ah *ah)
 421 {
 422         struct ib_device *ib_dev = ah->ibah.device;
 423         struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
 424
 425         roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
 426                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
 427
 428         roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
 429                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
 430         roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
 431                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
 432         roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
 433                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
 434
 435         if (WARN_ON(ah->av.sl > MAX_SERVICE_LEVEL))
 436                 return -EINVAL;
 437
 438         roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
 439                        V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
 440
 441         ud_sq_wqe->sgid_index = ah->av.gid_index;
 442
 443         memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN);
 444         memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2);
 445
 446         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 447                 return 0;
 448
 449         roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
 450                      ah->av.vlan_en);
 451         roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M,
 452                        V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
 453
 454         return 0;
 455 }
 456
 457 static inline int set_ud_wqe(struct hns_roce_qp *qp,
 458                              const struct ib_send_wr *wr,
 459                              void *wqe, unsigned int *sge_idx,
 460                              unsigned int owner_bit)
 461 {
 462         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
 463         struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
 464         unsigned int curr_idx = *sge_idx;
 465         unsigned int valid_num_sge;
 466         u32 msg_len = 0;
 467         int ret;
 468
 469         valid_num_sge = calc_wr_sge_num(wr, &msg_len);
 470
 471         ret = set_ud_opcode(ud_sq_wqe, wr);
 472         if (WARN_ON(ret))
 473                 return ret;
 474
 475         ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
 476
 477         roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S,
 478                      !!(wr->send_flags & IB_SEND_SIGNALED));
 479
 480         roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S,
 481                      !!(wr->send_flags & IB_SEND_SOLICITED));
 482
 483         roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M,
 484                        V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn);
 485
 486         roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
 487                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
 488
 489         roce_set_field(ud_sq_wqe->byte_20,
 490                        V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 491                        V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 492                        curr_idx & (qp->sge.sge_cnt - 1));
 493
 494         ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
 495                           qp->qkey : ud_wr(wr)->remote_qkey);
 496         roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
 497                        V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);
 498
 499         ret = fill_ud_av(ud_sq_wqe, ah);
 500         if (ret)
 501                 return ret;
 502
 503         qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
 504
 505         set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
 506
 507         /*
 508          * The pipeline can sequentially post all valid WQEs into WQ buffer,
 509          * including new WQEs waiting for the doorbell to update the PI again.
 510          * Therefore, the owner bit of WQE MUST be updated after all fields
 511          * and extSGEs have been written into DDR instead of cache.
 512          */
 513         if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
 514                 dma_wmb();
 515
 516         *sge_idx = curr_idx;
 517         roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S,
 518                      owner_bit);
 519
 520         return 0;
 521 }
 522
 523 static int set_rc_opcode(struct hns_roce_dev *hr_dev,
 524                          struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 525                          const struct ib_send_wr *wr)
 526 {
 527         u32 ib_op = wr->opcode;
 528         int ret = 0;
 529
 530         rc_sq_wqe->immtdata = get_immtdata(wr);
 531
 532         switch (ib_op) {
 533         case IB_WR_RDMA_READ:
 534         case IB_WR_RDMA_WRITE:
 535         case IB_WR_RDMA_WRITE_WITH_IMM:
 536                 rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
 537                 rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
 538                 break;
 539         case IB_WR_SEND:
 540         case IB_WR_SEND_WITH_IMM:
 541                 break;
 542         case IB_WR_ATOMIC_CMP_AND_SWP:
 543         case IB_WR_ATOMIC_FETCH_AND_ADD:
 544                 rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);
 545                 rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);
 546                 break;
 547         case IB_WR_REG_MR:
 548                 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 549                         set_frmr_seg(rc_sq_wqe, reg_wr(wr));
 550                 else
 551                         ret = -EOPNOTSUPP;
 552                 break;
 553         case IB_WR_LOCAL_INV:
 554                 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
 555                 fallthrough;
 556         case IB_WR_SEND_WITH_INV:
 557                 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
 558                 break;
 559         default:
 560                 ret = -EINVAL;
 561         }
 562
 563         if (unlikely(ret))
 564                 return ret;
 565
 566         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 567                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
 568
 569         return ret;
 570 }
 571 static inline int set_rc_wqe(struct hns_roce_qp *qp,
 572                              const struct ib_send_wr *wr,
 573                              void *wqe, unsigned int *sge_idx,
 574                              unsigned int owner_bit)
 575 {
 576         struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 577         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
 578         unsigned int curr_idx = *sge_idx;
 579         unsigned int valid_num_sge;
 580         u32 msg_len = 0;
 581         int ret;
 582
 583         valid_num_sge = calc_wr_sge_num(wr, &msg_len);
 584
 585         rc_sq_wqe->msg_len = cpu_to_le32(msg_len);
 586
 587         ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr);
 588         if (WARN_ON(ret))
 589                 return ret;
 590
 591         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S,
 592                      (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
 593
 594         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S,
 595                      (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
 596
 597         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S,
 598                      (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 599
 600         if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
 601             wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
 602                 set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
 603         else if (wr->opcode != IB_WR_REG_MR)
 604                 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
 605                                         &curr_idx, valid_num_sge);
 606
 607         /*
 608          * The pipeline can sequentially post all valid WQEs into WQ buffer,
 609          * including new WQEs waiting for the doorbell to update the PI again.
 610          * Therefore, the owner bit of WQE MUST be updated after all fields
 611          * and extSGEs have been written into DDR instead of cache.
 612          */
 613         if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
 614                 dma_wmb();
 615
 616         *sge_idx = curr_idx;
 617         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
 618                      owner_bit);
 619
 620         return ret;
 621 }
 622
 623 static inline void update_sq_db(struct hns_roce_dev *hr_dev,
 624                                 struct hns_roce_qp *qp)
 625 {
 626         /*
 627          * Hip08 hardware cannot flush the WQEs in SQ if the QP state
 628          * gets into errored mode. Hence, as a workaround to this
 629          * hardware limitation, driver needs to assist in flushing. But
 630          * the flushing operation uses mailbox to convey the QP state to
 631          * the hardware and which can sleep due to the mutex protection
 632          * around the mailbox calls. Hence, use the deferred flush for
 633          * now.
 634          */
 635         if (qp->state == IB_QPS_ERR) {
 636                 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
 637                         init_flush_work(hr_dev, qp);
 638         } else {
 639                 struct hns_roce_v2_db sq_db = {};
 640
 641                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
 642                                V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
 643                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
 644                                V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
 645                 /* indicates data on new BAR, 0 : SQ doorbell, 1 : DWQE */
 646                 roce_set_bit(sq_db.byte_4, V2_DB_FLAG_S, 0);
 647                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
 648                                V2_DB_PARAMETER_IDX_S, qp->sq.head);
 649                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
 650                                V2_DB_PARAMETER_SL_S, qp->sl);
 651
 652                 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
 653         }
 654 }
 655
 656 static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val,
 657                               u64 __iomem *dest)
 658 {
 659 #define HNS_ROCE_WRITE_TIMES 8
 660         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 661         struct hnae3_handle *handle = priv->handle;
 662         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 663         int i;
 664
 665         if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle))
 666                 for (i = 0; i < HNS_ROCE_WRITE_TIMES; i++)
 667                         writeq_relaxed(*(val + i), dest + i);
 668 }
 669
 670 static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
 671                        void *wqe)
 672 {
 673         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
 674
 675         /* All kinds of DirectWQE have the same header field layout */
 676         roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FLAG_S, 1);
 677         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_L_M,
 678                        V2_RC_SEND_WQE_BYTE_4_DB_SL_L_S, qp->sl);
 679         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_H_M,
 680                        V2_RC_SEND_WQE_BYTE_4_DB_SL_H_S, qp->sl >> 2);
 681         roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_M,
 682                        V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head);
 683
 684         hns_roce_write512(hr_dev, wqe, hr_dev->mem_base +
 685                           HNS_ROCE_DWQE_SIZE * qp->ibqp.qp_num);
 686 }
 687
 688 static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 689                                  const struct ib_send_wr *wr,
 690                                  const struct ib_send_wr **bad_wr)
 691 {
 692         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 693         struct ib_device *ibdev = &hr_dev->ib_dev;
 694         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 695         unsigned long flags = 0;
 696         unsigned int owner_bit;
 697         unsigned int sge_idx;
 698         unsigned int wqe_idx;
 699         void *wqe = NULL;
 700         u32 nreq;
 701         int ret;
 702
 703         spin_lock_irqsave(&qp->sq.lock, flags);
 704
 705         ret = check_send_valid(hr_dev, qp);
 706         if (unlikely(ret)) {
 707                 *bad_wr = wr;
 708                 nreq = 0;
 709                 goto out;
 710         }
 711
 712         sge_idx = qp->next_sge;
 713
 714         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 715                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
 716                         ret = -ENOMEM;
 717                         *bad_wr = wr;
 718                         goto out;
 719                 }
 720
 721                 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
 722
 723                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
 724                         ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",
 725                                   wr->num_sge, qp->sq.max_gs);
 726                         ret = -EINVAL;
 727                         *bad_wr = wr;
 728                         goto out;
 729                 }
 730
 731                 wqe = hns_roce_get_send_wqe(qp, wqe_idx);
 732                 qp->sq.wrid[wqe_idx] = wr->wr_id;
 733                 owner_bit =
 734                        ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
 735
 736                 /* Corresponding to the QP type, wqe process separately */
 737                 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
 738                         ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 739                 else if (ibqp->qp_type == IB_QPT_RC)
 740                         ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 741
 742                 if (unlikely(ret)) {
 743                         *bad_wr = wr;
 744                         goto out;
 745                 }
 746         }
 747
 748 out:
 749         if (likely(nreq)) {
 750                 qp->sq.head += nreq;
 751                 qp->next_sge = sge_idx;
 752
 753                 if (nreq == 1 && qp->sq.head == qp->sq.tail + 1 &&
 754                     (qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
 755                         write_dwqe(hr_dev, qp, wqe);
 756                 else
 757                         update_sq_db(hr_dev, qp);
 758         }
 759
 760         spin_unlock_irqrestore(&qp->sq.lock, flags);
 761
 762         return ret;
 763 }
 764
 765 static int check_recv_valid(struct hns_roce_dev *hr_dev,
 766                             struct hns_roce_qp *hr_qp)
 767 {
 768         struct ib_device *ibdev = &hr_dev->ib_dev;
 769         struct ib_qp *ibqp = &hr_qp->ibqp;
 770
 771         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
 772                      ibqp->qp_type != IB_QPT_GSI &&
 773                      ibqp->qp_type != IB_QPT_UD)) {
 774                 ibdev_err(ibdev, "unsupported qp type, qp_type = %d.\n",
 775                           ibqp->qp_type);
 776                 return -EOPNOTSUPP;
 777         }
 778
 779         if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
 780                 return -EIO;
 781
 782         if (hr_qp->state == IB_QPS_RESET)
 783                 return -EINVAL;
 784
 785         return 0;
 786 }
 787
 788 static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe,
 789                                  u32 max_sge, bool rsv)
 790 {
 791         struct hns_roce_v2_wqe_data_seg *dseg = wqe;
 792         u32 i, cnt;
 793
 794         for (i = 0, cnt = 0; i < wr->num_sge; i++) {
 795                 /* Skip zero-length sge */
 796                 if (!wr->sg_list[i].length)
 797                         continue;
 798                 set_data_seg_v2(dseg + cnt, wr->sg_list + i);
 799                 cnt++;
 800         }
 801
 802         /* Fill a reserved sge to make hw stop reading remaining segments */
 803         if (rsv) {
 804                 dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
 805                 dseg[cnt].addr = 0;
 806                 dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);
 807         } else {
 808                 /* Clear remaining segments to make ROCEE ignore sges */
 809                 if (cnt < max_sge)
 810                         memset(dseg + cnt, 0,
 811                                (max_sge - cnt) * HNS_ROCE_SGE_SIZE);
 812         }
 813 }
 814
 815 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,
 816                         u32 wqe_idx, u32 max_sge)
 817 {
 818         struct hns_roce_rinl_sge *sge_list;
 819         void *wqe = NULL;
 820         u32 i;
 821
 822         wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
 823         fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge);
 824
 825         /* rq support inline data */
 826         if (hr_qp->rq_inl_buf.wqe_cnt) {
 827                 sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
 828                 hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = (u32)wr->num_sge;
 829                 for (i = 0; i < wr->num_sge; i++) {
 830                         sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr;
 831                         sge_list[i].len = wr->sg_list[i].length;
 832                 }
 833         }
 834 }
 835
 836 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 837                                  const struct ib_recv_wr *wr,
 838                                  const struct ib_recv_wr **bad_wr)
 839 {
 840         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 841         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 842         struct ib_device *ibdev = &hr_dev->ib_dev;
 843         u32 wqe_idx, nreq, max_sge;
 844         unsigned long flags;
 845         int ret;
 846
 847         spin_lock_irqsave(&hr_qp->rq.lock, flags);
 848
 849         ret = check_recv_valid(hr_dev, hr_qp);
 850         if (unlikely(ret)) {
 851                 *bad_wr = wr;
 852                 nreq = 0;
 853                 goto out;
 854         }
 855
 856         max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
 857         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 858                 if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
 859                                                   hr_qp->ibqp.recv_cq))) {
 860                         ret = -ENOMEM;
 861                         *bad_wr = wr;
 862                         goto out;
 863                 }
 864
 865                 if (unlikely(wr->num_sge > max_sge)) {
 866                         ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",
 867                                   wr->num_sge, max_sge);
 868                         ret = -EINVAL;
 869                         *bad_wr = wr;
 870                         goto out;
 871                 }
 872
 873                 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
 874                 fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge);
 875                 hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
 876         }
 877
 878 out:
 879         if (likely(nreq)) {
 880                 hr_qp->rq.head += nreq;
 881
 882                 /*
 883                  * Hip08 hardware cannot flush the WQEs in RQ if the QP state
 884                  * gets into errored mode. Hence, as a workaround to this
 885                  * hardware limitation, driver needs to assist in flushing. But
 886                  * the flushing operation uses mailbox to convey the QP state to
 887                  * the hardware and which can sleep due to the mutex protection
 888                  * around the mailbox calls. Hence, use the deferred flush for
 889                  * now.
 890                  */
 891                 if (hr_qp->state == IB_QPS_ERR) {
 892                         if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG,
 893                                               &hr_qp->flush_flag))
 894                                 init_flush_work(hr_dev, hr_qp);
 895                 } else {
 896                         *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
 897                 }
 898         }
 899         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 900
 901         return ret;
 902 }
 903
 904 static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n)
 905 {
 906         return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
 907 }
 908
 909 static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n)
 910 {
 911         return hns_roce_buf_offset(idx_que->mtr.kmem,
 912                                    n << idx_que->entry_shift);
 913 }
 914
 915 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index)
 916 {
 917         /* always called with interrupts disabled. */
 918         spin_lock(&srq->lock);
 919
 920         bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
 921         srq->idx_que.tail++;
 922
 923         spin_unlock(&srq->lock);
 924 }
 925
 926 static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq)
 927 {
 928         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 929
 930         return idx_que->head - idx_que->tail >= srq->wqe_cnt;
 931 }
 932
 933 static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge,
 934                                 const struct ib_recv_wr *wr)
 935 {
 936         struct ib_device *ib_dev = srq->ibsrq.device;
 937
 938         if (unlikely(wr->num_sge > max_sge)) {
 939                 ibdev_err(ib_dev,
 940                           "failed to check sge, wr->num_sge = %d, max_sge = %u.\n",
 941                           wr->num_sge, max_sge);
 942                 return -EINVAL;
 943         }
 944
 945         if (unlikely(hns_roce_srqwq_overflow(srq))) {
 946                 ibdev_err(ib_dev,
 947                           "failed to check srqwq status, srqwq is full.\n");
 948                 return -ENOMEM;
 949         }
 950
 951         return 0;
 952 }
 953
 954 static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx)
 955 {
 956         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 957         u32 pos;
 958
 959         pos = find_first_zero_bit(idx_que->bitmap, srq->wqe_cnt);
 960         if (unlikely(pos == srq->wqe_cnt))
 961                 return -ENOSPC;
 962
 963         bitmap_set(idx_que->bitmap, pos, 1);
 964         *wqe_idx = pos;
 965         return 0;
 966 }
 967
 968 static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)
 969 {
 970         struct hns_roce_idx_que *idx_que = &srq->idx_que;
 971         unsigned int head;
 972         __le32 *buf;
 973
 974         head = idx_que->head & (srq->wqe_cnt - 1);
 975
 976         buf = get_idx_buf(idx_que, head);
 977         *buf = cpu_to_le32(wqe_idx);
 978
 979         idx_que->head++;
 980 }
 981
 982 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
 983                                      const struct ib_recv_wr *wr,
 984                                      const struct ib_recv_wr **bad_wr)
 985 {
 986         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
 987         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
 988         struct hns_roce_v2_db srq_db;
 989         unsigned long flags;
 990         int ret = 0;
 991         u32 max_sge;
 992         u32 wqe_idx;
 993         void *wqe;
 994         u32 nreq;
 995
 996         spin_lock_irqsave(&srq->lock, flags);
 997
 998         max_sge = srq->max_gs - srq->rsv_sge;
 999         for (nreq = 0; wr; ++nreq, wr = wr->next) {
1000                 ret = check_post_srq_valid(srq, max_sge, wr);
1001                 if (ret) {
1002                         *bad_wr = wr;
1003                         break;
1004                 }
1005
1006                 ret = get_srq_wqe_idx(srq, &wqe_idx);
1007                 if (unlikely(ret)) {
1008                         *bad_wr = wr;
1009                         break;
1010                 }
1011
1012                 wqe = get_srq_wqe_buf(srq, wqe_idx);
1013                 fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge);
1014                 fill_wqe_idx(srq, wqe_idx);
1015                 srq->wrid[wqe_idx] = wr->wr_id;
1016         }
1017
1018         if (likely(nreq)) {
1019                 srq_db.byte_4 =
1020                         cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
1021                                     (srq->srqn & V2_DB_BYTE_4_TAG_M));
1022                 srq_db.parameter =
1023                         cpu_to_le32(srq->idx_que.head & V2_DB_PARAMETER_IDX_M);
1024
1025                 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
1026         }
1027
1028         spin_unlock_irqrestore(&srq->lock, flags);
1029
1030         return ret;
1031 }
1032
1033 static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
1034                                       unsigned long instance_stage,
1035                                       unsigned long reset_stage)
1036 {
1037         /* When hardware reset has been completed once or more, we should stop
1038          * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
1039          * function, we should exit with error. If now at HNAE3_INIT_CLIENT
1040          * stage of soft reset process, we should exit with error, and then
1041          * HNAE3_INIT_CLIENT related process can rollback the operation like
1042          * notifing hardware to free resources, HNAE3_INIT_CLIENT related
1043          * process will exit with error to notify NIC driver to reschedule soft
1044          * reset process once again.
1045          */
1046         hr_dev->is_reset = true;
1047         hr_dev->dis_db = true;
1048
1049         if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
1050             instance_stage == HNS_ROCE_STATE_INIT)
1051                 return CMD_RST_PRC_EBUSY;
1052
1053         return CMD_RST_PRC_SUCCESS;
1054 }
1055
1056 static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
1057                                         unsigned long instance_stage,
1058                                         unsigned long reset_stage)
1059 {
1060         struct hns_roce_v2_priv *priv = hr_dev->priv;
1061         struct hnae3_handle *handle = priv->handle;
1062         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1063
1064         /* When hardware reset is detected, we should stop sending mailbox&cmq&
1065          * doorbell to hardware. If now in .init_instance() function, we should
1066          * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
1067          * process, we should exit with error, and then HNAE3_INIT_CLIENT
1068          * related process can rollback the operation like notifing hardware to
1069          * free resources, HNAE3_INIT_CLIENT related process will exit with
1070          * error to notify NIC driver to reschedule soft reset process once
1071          * again.
1072          */
1073         hr_dev->dis_db = true;
1074         if (!ops->get_hw_reset_stat(handle))
1075                 hr_dev->is_reset = true;
1076
1077         if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
1078             instance_stage == HNS_ROCE_STATE_INIT)
1079                 return CMD_RST_PRC_EBUSY;
1080
1081         return CMD_RST_PRC_SUCCESS;
1082 }
1083
1084 static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
1085 {
1086         struct hns_roce_v2_priv *priv = hr_dev->priv;
1087         struct hnae3_handle *handle = priv->handle;
1088         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1089
1090         /* When software reset is detected at .init_instance() function, we
1091          * should stop sending mailbox&cmq&doorbell to hardware, and exit
1092          * with error.
1093          */
1094         hr_dev->dis_db = true;
1095         if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
1096                 hr_dev->is_reset = true;
1097
1098         return CMD_RST_PRC_EBUSY;
1099 }
1100
1101 static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
1102 {
1103         struct hns_roce_v2_priv *priv = hr_dev->priv;
1104         struct hnae3_handle *handle = priv->handle;
1105         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1106         unsigned long instance_stage; /* the current instance stage */
1107         unsigned long reset_stage; /* the current reset stage */
1108         unsigned long reset_cnt;
1109         bool sw_resetting;
1110         bool hw_resetting;
1111
1112         if (hr_dev->is_reset)
1113                 return CMD_RST_PRC_SUCCESS;
1114
1115         /* Get information about reset from NIC driver or RoCE driver itself,
1116          * the meaning of the following variables from NIC driver are described
1117          * as below:
1118          * reset_cnt -- The count value of completed hardware reset.
1119          * hw_resetting -- Whether hardware device is resetting now.
1120          * sw_resetting -- Whether NIC's software reset process is running now.
1121          */
1122         instance_stage = handle->rinfo.instance_state;
1123         reset_stage = handle->rinfo.reset_state;
1124         reset_cnt = ops->ae_dev_reset_cnt(handle);
1125         hw_resetting = ops->get_cmdq_stat(handle);
1126         sw_resetting = ops->ae_dev_resetting(handle);
1127
1128         if (reset_cnt != hr_dev->reset_cnt)
1129                 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
1130                                                   reset_stage);
1131         else if (hw_resetting)
1132                 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
1133                                                     reset_stage);
1134         else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
1135                 return hns_roce_v2_cmd_sw_resetting(hr_dev);
1136
1137         return 0;
1138 }
1139
1140 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
1141                                    struct hns_roce_v2_cmq_ring *ring)
1142 {
1143         int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
1144
1145         ring->desc = kzalloc(size, GFP_KERNEL);
1146         if (!ring->desc)
1147                 return -ENOMEM;
1148
1149         ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
1150                                              DMA_BIDIRECTIONAL);
1151         if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
1152                 ring->desc_dma_addr = 0;
1153                 kfree(ring->desc);
1154                 ring->desc = NULL;
1155                 return -ENOMEM;
1156         }
1157
1158         return 0;
1159 }
1160
1161 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
1162                                    struct hns_roce_v2_cmq_ring *ring)
1163 {
1164         dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
1165                          ring->desc_num * sizeof(struct hns_roce_cmq_desc),
1166                          DMA_BIDIRECTIONAL);
1167
1168         ring->desc_dma_addr = 0;
1169         kfree(ring->desc);
1170 }
1171
1172 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
1173 {
1174         struct hns_roce_v2_priv *priv = hr_dev->priv;
1175         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
1176                                             &priv->cmq.csq : &priv->cmq.crq;
1177
1178         ring->flag = ring_type;
1179         ring->head = 0;
1180
1181         return hns_roce_alloc_cmq_desc(hr_dev, ring);
1182 }
1183
1184 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
1185 {
1186         struct hns_roce_v2_priv *priv = hr_dev->priv;
1187         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
1188                                             &priv->cmq.csq : &priv->cmq.crq;
1189         dma_addr_t dma = ring->desc_dma_addr;
1190
1191         if (ring_type == TYPE_CSQ) {
1192                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
1193                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
1194                            upper_32_bits(dma));
1195                 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
1196                            (u32)ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
1197                 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
1198                 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
1199         } else {
1200                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
1201                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
1202                            upper_32_bits(dma));
1203                 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
1204                            (u32)ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
1205                 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
1206                 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
1207         }
1208 }
1209
1210 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
1211 {
1212         struct hns_roce_v2_priv *priv = hr_dev->priv;
1213         int ret;
1214
1215         /* Setup the queue entries for command queue */
1216         priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
1217         priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;
1218
1219         /* Setup the lock for command queue */
1220         spin_lock_init(&priv->cmq.csq.lock);
1221         spin_lock_init(&priv->cmq.crq.lock);
1222
1223         /* Setup Tx write back timeout */
1224         priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
1225
1226         /* Init CSQ */
1227         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
1228         if (ret) {
1229                 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
1230                 return ret;
1231         }
1232
1233         /* Init CRQ */
1234         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
1235         if (ret) {
1236                 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
1237                 goto err_crq;
1238         }
1239
1240         /* Init CSQ REG */
1241         hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
1242
1243         /* Init CRQ REG */
1244         hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
1245
1246         return 0;
1247
1248 err_crq:
1249         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
1250
1251         return ret;
1252 }
1253
1254 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
1255 {
1256         struct hns_roce_v2_priv *priv = hr_dev->priv;
1257
1258         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
1259         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
1260 }
1261
1262 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
1263                                           enum hns_roce_opcode_type opcode,
1264                                           bool is_read)
1265 {
1266         memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
1267         desc->opcode = cpu_to_le16(opcode);
1268         desc->flag =
1269                 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
1270         if (is_read)
1271                 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
1272         else
1273                 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1274 }
1275
1276 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
1277 {
1278         u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_TAIL_REG);
1279         struct hns_roce_v2_priv *priv = hr_dev->priv;
1280
1281         return tail == priv->cmq.csq.head;
1282 }
1283
1284 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1285                                struct hns_roce_cmq_desc *desc, int num)
1286 {
1287         struct hns_roce_v2_priv *priv = hr_dev->priv;
1288         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1289         u32 timeout = 0;
1290         u16 desc_ret;
1291         u32 tail;
1292         int ret;
1293         int i;
1294
1295         spin_lock_bh(&csq->lock);
1296
1297         tail = csq->head;
1298
1299         for (i = 0; i < num; i++) {
1300                 csq->desc[csq->head++] = desc[i];
1301                 if (csq->head == csq->desc_num)
1302                         csq->head = 0;
1303         }
1304
1305         /* Write to hardware */
1306         roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, csq->head);
1307
1308         /* If the command is sync, wait for the firmware to write back,
1309          * if multi descriptors to be sent, use the first one to check
1310          */
1311         if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
1312                 do {
1313                         if (hns_roce_cmq_csq_done(hr_dev))
1314                                 break;
1315                         udelay(1);
1316                 } while (++timeout < priv->cmq.tx_timeout);
1317         }
1318
1319         if (hns_roce_cmq_csq_done(hr_dev)) {
1320                 for (ret = 0, i = 0; i < num; i++) {
1321                         /* check the result of hardware write back */
1322                         desc[i] = csq->desc[tail++];
1323                         if (tail == csq->desc_num)
1324                                 tail = 0;
1325
1326                         desc_ret = le16_to_cpu(desc[i].retval);
1327                         if (likely(desc_ret == CMD_EXEC_SUCCESS))
1328                                 continue;
1329
1330                         dev_err_ratelimited(hr_dev->dev,
1331                                             "Cmdq IO error, opcode = %x, return = %x\n",
1332                                             desc->opcode, desc_ret);
1333                         ret = -EIO;
1334                 }
1335         } else {
1336                 /* FW/HW reset or incorrect number of desc */
1337                 tail = roce_read(hr_dev, ROCEE_TX_CMQ_TAIL_REG);
1338                 dev_warn(hr_dev->dev, "CMDQ move tail from %d to %d\n",
1339                          csq->head, tail);
1340                 csq->head = tail;
1341
1342                 ret = -EAGAIN;
1343         }
1344
1345         spin_unlock_bh(&csq->lock);
1346
1347         return ret;
1348 }
1349
1350 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1351                              struct hns_roce_cmq_desc *desc, int num)
1352 {
1353         int retval;
1354         int ret;
1355
1356         ret = hns_roce_v2_rst_process_cmd(hr_dev);
1357         if (ret == CMD_RST_PRC_SUCCESS)
1358                 return 0;
1359         if (ret == CMD_RST_PRC_EBUSY)
1360                 return -EBUSY;
1361
1362         ret = __hns_roce_cmq_send(hr_dev, desc, num);
1363         if (ret) {
1364                 retval = hns_roce_v2_rst_process_cmd(hr_dev);
1365                 if (retval == CMD_RST_PRC_SUCCESS)
1366                         return 0;
1367                 else if (retval == CMD_RST_PRC_EBUSY)
1368                         return -EBUSY;
1369         }
1370
1371         return ret;
1372 }
1373
1374 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
1375 {
1376         struct hns_roce_query_version *resp;
1377         struct hns_roce_cmq_desc desc;
1378         int ret;
1379
1380         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1381         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1382         if (ret)
1383                 return ret;
1384
1385         resp = (struct hns_roce_query_version *)desc.data;
1386         hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
1387         hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1388
1389         return 0;
1390 }
1391
1392 static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
1393 {
1394         struct hns_roce_v2_priv *priv = hr_dev->priv;
1395         struct hnae3_handle *handle = priv->handle;
1396         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1397         unsigned long reset_cnt;
1398         bool sw_resetting;
1399         bool hw_resetting;
1400
1401         reset_cnt = ops->ae_dev_reset_cnt(handle);
1402         hw_resetting = ops->get_hw_reset_stat(handle);
1403         sw_resetting = ops->ae_dev_resetting(handle);
1404
1405         if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting)
1406                 return true;
1407
1408         return false;
1409 }
1410
1411 static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
1412                                       int flag)
1413 {
1414         struct hns_roce_v2_priv *priv = hr_dev->priv;
1415         struct hnae3_handle *handle = priv->handle;
1416         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1417         unsigned long instance_stage;
1418         unsigned long reset_cnt;
1419         unsigned long end;
1420         bool sw_resetting;
1421         bool hw_resetting;
1422
1423         instance_stage = handle->rinfo.instance_state;
1424         reset_cnt = ops->ae_dev_reset_cnt(handle);
1425         hw_resetting = ops->get_hw_reset_stat(handle);
1426         sw_resetting = ops->ae_dev_resetting(handle);
1427
1428         if (reset_cnt != hr_dev->reset_cnt) {
1429                 hr_dev->dis_db = true;
1430                 hr_dev->is_reset = true;
1431                 dev_info(hr_dev->dev, "Func clear success after reset.\n");
1432         } else if (hw_resetting) {
1433                 hr_dev->dis_db = true;
1434
1435                 dev_warn(hr_dev->dev,
1436                          "Func clear is pending, device in resetting state.\n");
1437                 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1438                 while (end) {
1439                         if (!ops->get_hw_reset_stat(handle)) {
1440                                 hr_dev->is_reset = true;
1441                                 dev_info(hr_dev->dev,
1442                                          "Func clear success after reset.\n");
1443                                 return;
1444                         }
1445                         msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1446                         end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1447                 }
1448
1449                 dev_warn(hr_dev->dev, "Func clear failed.\n");
1450         } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) {
1451                 hr_dev->dis_db = true;
1452
1453                 dev_warn(hr_dev->dev,
1454                          "Func clear is pending, device in resetting state.\n");
1455                 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1456                 while (end) {
1457                         if (ops->ae_dev_reset_cnt(handle) !=
1458                             hr_dev->reset_cnt) {
1459                                 hr_dev->is_reset = true;
1460                                 dev_info(hr_dev->dev,
1461                                          "Func clear success after sw reset\n");
1462                                 return;
1463                         }
1464                         msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1465                         end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1466                 }
1467
1468                 dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
1469         } else {
1470                 if (retval && !flag)
1471                         dev_warn(hr_dev->dev,
1472                                  "Func clear read failed, ret = %d.\n", retval);
1473
1474                 dev_warn(hr_dev->dev, "Func clear failed.\n");
1475         }
1476 }
1477 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1478 {
1479         bool fclr_write_fail_flag = false;
1480         struct hns_roce_func_clear *resp;
1481         struct hns_roce_cmq_desc desc;
1482         unsigned long end;
1483         int ret = 0;
1484
1485         if (hns_roce_func_clr_chk_rst(hr_dev))
1486                 goto out;
1487
1488         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1489         resp = (struct hns_roce_func_clear *)desc.data;
1490
1491         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1492         if (ret) {
1493                 fclr_write_fail_flag = true;
1494                 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
1495                          ret);
1496                 goto out;
1497         }
1498
1499         msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1500         end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1501         while (end) {
1502                 if (hns_roce_func_clr_chk_rst(hr_dev))
1503                         goto out;
1504                 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1505                 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1506
1507                 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1508                                               true);
1509
1510                 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1511                 if (ret)
1512                         continue;
1513
1514                 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
1515                         hr_dev->is_reset = true;
1516                         return;
1517                 }
1518         }
1519
1520 out:
1521         hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag);
1522 }
1523
1524 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1525 {
1526         struct hns_roce_query_fw_info *resp;
1527         struct hns_roce_cmq_desc desc;
1528         int ret;
1529
1530         hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1531         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1532         if (ret)
1533                 return ret;
1534
1535         resp = (struct hns_roce_query_fw_info *)desc.data;
1536         hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1537
1538         return 0;
1539 }
1540
1541 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1542 {
1543         struct hns_roce_cfg_global_param *req;
1544         struct hns_roce_cmq_desc desc;
1545
1546         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1547                                       false);
1548
1549         req = (struct hns_roce_cfg_global_param *)desc.data;
1550         memset(req, 0, sizeof(*req));
1551         roce_set_field(req->time_cfg_udp_port,
1552                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
1553                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
1554         roce_set_field(req->time_cfg_udp_port,
1555                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
1556                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S,
1557                        ROCE_V2_UDP_DPORT);
1558
1559         return hns_roce_cmq_send(hr_dev, &desc, 1);
1560 }
1561
1562 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1563 {
1564         struct hns_roce_cmq_desc desc[2];
1565         struct hns_roce_pf_res_a *req_a;
1566         struct hns_roce_pf_res_b *req_b;
1567         int ret;
1568
1569         hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_QUERY_PF_RES,
1570                                       true);
1571         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1572
1573         hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_QUERY_PF_RES,
1574                                       true);
1575
1576         ret = hns_roce_cmq_send(hr_dev, desc, 2);
1577         if (ret)
1578                 return ret;
1579
1580         req_a = (struct hns_roce_pf_res_a *)desc[0].data;
1581         req_b = (struct hns_roce_pf_res_b *)desc[1].data;
1582
1583         hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
1584                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_M,
1585                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_S);
1586         hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
1587                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
1588                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
1589         hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
1590                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_M,
1591                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_S);
1592         hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
1593                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_M,
1594                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_S);
1595
1596         hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1597                                              PF_RES_DATA_3_PF_SL_NUM_M,
1598                                              PF_RES_DATA_3_PF_SL_NUM_S);
1599         hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
1600                                              PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
1601                                              PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
1602
1603         hr_dev->caps.gmv_bt_num = roce_get_field(req_b->gmv_idx_num,
1604                                                  PF_RES_DATA_5_PF_GMV_BT_NUM_M,
1605                                                  PF_RES_DATA_5_PF_GMV_BT_NUM_S);
1606
1607         return 0;
1608 }
1609
1610 static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
1611 {
1612         struct hns_roce_pf_timer_res_a *req_a;
1613         struct hns_roce_cmq_desc desc;
1614         int ret;
1615
1616         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1617                                       true);
1618
1619         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1620         if (ret)
1621                 return ret;
1622
1623         req_a = (struct hns_roce_pf_timer_res_a *)desc.data;
1624
1625         hr_dev->caps.qpc_timer_bt_num =
1626                 roce_get_field(req_a->qpc_timer_bt_idx_num,
1627                                PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
1628                                PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
1629         hr_dev->caps.cqc_timer_bt_num =
1630                 roce_get_field(req_a->cqc_timer_bt_idx_num,
1631                                PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
1632                                PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
1633
1634         return 0;
1635 }
1636
1637 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id)
1638 {
1639         struct hns_roce_cmq_desc desc;
1640         struct hns_roce_vf_switch *swt;
1641         int ret;
1642
1643         swt = (struct hns_roce_vf_switch *)desc.data;
1644         hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1645         swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1646         roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M,
1647                        VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id);
1648         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1649         if (ret)
1650                 return ret;
1651
1652         desc.flag =
1653                 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
1654         desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1655         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
1656         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
1657         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
1658
1659         return hns_roce_cmq_send(hr_dev, &desc, 1);
1660 }
1661
1662 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1663 {
1664         struct hns_roce_cmq_desc desc[2];
1665         struct hns_roce_vf_res_a *req_a;
1666         struct hns_roce_vf_res_b *req_b;
1667
1668         req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1669         req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1670
1671         hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_ALLOC_VF_RES,
1672                                       false);
1673         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1674
1675         hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_ALLOC_VF_RES,
1676                                       false);
1677
1678         roce_set_field(req_a->vf_qpc_bt_idx_num,
1679                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1680                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1681         roce_set_field(req_a->vf_qpc_bt_idx_num,
1682                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1683                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, HNS_ROCE_VF_QPC_BT_NUM);
1684
1685         roce_set_field(req_a->vf_srqc_bt_idx_num,
1686                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1687                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1688         roce_set_field(req_a->vf_srqc_bt_idx_num,
1689                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1690                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1691                        HNS_ROCE_VF_SRQC_BT_NUM);
1692
1693         roce_set_field(req_a->vf_cqc_bt_idx_num,
1694                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1695                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1696         roce_set_field(req_a->vf_cqc_bt_idx_num,
1697                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1698                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, HNS_ROCE_VF_CQC_BT_NUM);
1699
1700         roce_set_field(req_a->vf_mpt_bt_idx_num,
1701                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1702                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1703         roce_set_field(req_a->vf_mpt_bt_idx_num,
1704                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1705                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, HNS_ROCE_VF_MPT_BT_NUM);
1706
1707         roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_IDX_M,
1708                        VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1709         roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_NUM_M,
1710                        VF_RES_A_DATA_5_VF_EQC_NUM_S, HNS_ROCE_VF_EQC_NUM);
1711
1712         roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1713                        VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1714         roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1715                        VF_RES_B_DATA_1_VF_SMAC_NUM_S, HNS_ROCE_VF_SMAC_NUM);
1716
1717         roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_IDX_M,
1718                        VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1719         roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_NUM_M,
1720                        VF_RES_B_DATA_2_VF_SGID_NUM_S, HNS_ROCE_VF_SGID_NUM);
1721
1722         roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_QID_IDX_M,
1723                        VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1724         roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_SL_NUM_M,
1725                        VF_RES_B_DATA_3_VF_SL_NUM_S, HNS_ROCE_VF_SL_NUM);
1726
1727         roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
1728                        VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
1729         roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
1730                        VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
1731                        HNS_ROCE_VF_SCCC_BT_NUM);
1732
1733         return hns_roce_cmq_send(hr_dev, desc, 2);
1734 }
1735
1736 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1737 {
1738         u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1739         u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1740         u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1741         u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
1742         u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
1743         struct hns_roce_cfg_bt_attr *req;
1744         struct hns_roce_cmq_desc desc;
1745
1746         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1747         req = (struct hns_roce_cfg_bt_attr *)desc.data;
1748         memset(req, 0, sizeof(*req));
1749
1750         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1751                        CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
1752                        hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
1753         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1754                        CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
1755                        hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
1756         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1757                        CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1758                        qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1759
1760         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1761                        CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
1762                        hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
1763         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1764                        CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
1765                        hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
1766         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1767                        CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1768                        srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1769
1770         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1771                        CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
1772                        hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
1773         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1774                        CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
1775                        hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
1776         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1777                        CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1778                        cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1779
1780         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1781                        CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
1782                        hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
1783         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1784                        CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
1785                        hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
1786         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1787                        CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1788                        mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1789
1790         roce_set_field(req->vf_sccc_cfg,
1791                        CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
1792                        CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
1793                        hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1794         roce_set_field(req->vf_sccc_cfg,
1795                        CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
1796                        CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
1797                        hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1798         roce_set_field(req->vf_sccc_cfg,
1799                        CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
1800                        CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
1801                        sccc_hop_num ==
1802                               HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
1803
1804         return hns_roce_cmq_send(hr_dev, &desc, 1);
1805 }
1806
1807 static void set_default_caps(struct hns_roce_dev *hr_dev)
1808 {
1809         struct hns_roce_caps *caps = &hr_dev->caps;
1810
1811         caps->num_qps           = HNS_ROCE_V2_MAX_QP_NUM;
1812         caps->max_wqes          = HNS_ROCE_V2_MAX_WQE_NUM;
1813         caps->num_cqs           = HNS_ROCE_V2_MAX_CQ_NUM;
1814         caps->num_srqs          = HNS_ROCE_V2_MAX_SRQ_NUM;
1815         caps->min_cqes          = HNS_ROCE_MIN_CQE_NUM;
1816         caps->max_cqes          = HNS_ROCE_V2_MAX_CQE_NUM;
1817         caps->max_sq_sg         = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1818         caps->max_extend_sg     = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1819         caps->max_rq_sg         = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1820         caps->max_sq_inline     = HNS_ROCE_V2_MAX_SQ_INLINE;
1821         caps->num_uars          = HNS_ROCE_V2_UAR_NUM;
1822         caps->phy_num_uars      = HNS_ROCE_V2_PHY_UAR_NUM;
1823         caps->num_aeq_vectors   = HNS_ROCE_V2_AEQE_VEC_NUM;
1824         caps->num_comp_vectors  = HNS_ROCE_V2_COMP_VEC_NUM;
1825         caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1826         caps->num_mtpts         = HNS_ROCE_V2_MAX_MTPT_NUM;
1827         caps->num_mtt_segs      = HNS_ROCE_V2_MAX_MTT_SEGS;
1828         caps->num_cqe_segs      = HNS_ROCE_V2_MAX_CQE_SEGS;
1829         caps->num_srqwqe_segs   = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
1830         caps->num_idx_segs      = HNS_ROCE_V2_MAX_IDX_SEGS;
1831         caps->num_pds           = HNS_ROCE_V2_MAX_PD_NUM;
1832         caps->max_qp_init_rdma  = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1833         caps->max_qp_dest_rdma  = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1834         caps->max_sq_desc_sz    = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1835         caps->max_rq_desc_sz    = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1836         caps->max_srq_desc_sz   = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1837         caps->qpc_sz            = HNS_ROCE_V2_QPC_SZ;
1838         caps->irrl_entry_sz     = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1839         caps->trrl_entry_sz     = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ;
1840         caps->cqc_entry_sz      = HNS_ROCE_V2_CQC_ENTRY_SZ;
1841         caps->srqc_entry_sz     = HNS_ROCE_V2_SRQC_ENTRY_SZ;
1842         caps->mtpt_entry_sz     = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1843         caps->mtt_entry_sz      = HNS_ROCE_V2_MTT_ENTRY_SZ;
1844         caps->idx_entry_sz      = HNS_ROCE_V2_IDX_ENTRY_SZ;
1845         caps->cqe_sz            = HNS_ROCE_V2_CQE_SIZE;
1846         caps->page_size_cap     = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1847         caps->reserved_lkey     = 0;
1848         caps->reserved_pds      = 0;
1849         caps->reserved_mrws     = 1;
1850         caps->reserved_uars     = 0;
1851         caps->reserved_cqs      = 0;
1852         caps->reserved_srqs     = 0;
1853         caps->reserved_qps      = HNS_ROCE_V2_RSV_QPS;
1854
1855         caps->qpc_ba_pg_sz      = 0;
1856         caps->qpc_buf_pg_sz     = 0;
1857         caps->qpc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1858         caps->srqc_ba_pg_sz     = 0;
1859         caps->srqc_buf_pg_sz    = 0;
1860         caps->srqc_hop_num      = HNS_ROCE_CONTEXT_HOP_NUM;
1861         caps->cqc_ba_pg_sz      = 0;
1862         caps->cqc_buf_pg_sz     = 0;
1863         caps->cqc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1864         caps->mpt_ba_pg_sz      = 0;
1865         caps->mpt_buf_pg_sz     = 0;
1866         caps->mpt_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1867         caps->mtt_ba_pg_sz      = 0;
1868         caps->mtt_buf_pg_sz     = 0;
1869         caps->mtt_hop_num       = HNS_ROCE_MTT_HOP_NUM;
1870         caps->wqe_sq_hop_num    = HNS_ROCE_SQWQE_HOP_NUM;
1871         caps->wqe_sge_hop_num   = HNS_ROCE_EXT_SGE_HOP_NUM;
1872         caps->wqe_rq_hop_num    = HNS_ROCE_RQWQE_HOP_NUM;
1873         caps->cqe_ba_pg_sz      = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
1874         caps->cqe_buf_pg_sz     = 0;
1875         caps->cqe_hop_num       = HNS_ROCE_CQE_HOP_NUM;
1876         caps->srqwqe_ba_pg_sz   = 0;
1877         caps->srqwqe_buf_pg_sz  = 0;
1878         caps->srqwqe_hop_num    = HNS_ROCE_SRQWQE_HOP_NUM;
1879         caps->idx_ba_pg_sz      = 0;
1880         caps->idx_buf_pg_sz     = 0;
1881         caps->idx_hop_num       = HNS_ROCE_IDX_HOP_NUM;
1882         caps->chunk_sz          = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1883
1884         caps->flags             = HNS_ROCE_CAP_FLAG_REREG_MR |
1885                                   HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1886                                   HNS_ROCE_CAP_FLAG_RECORD_DB |
1887                                   HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1888
1889         caps->pkey_table_len[0] = 1;
1890         caps->gid_table_len[0]  = HNS_ROCE_V2_GID_INDEX_NUM;
1891         caps->ceqe_depth        = HNS_ROCE_V2_COMP_EQE_NUM;
1892         caps->aeqe_depth        = HNS_ROCE_V2_ASYNC_EQE_NUM;
1893         caps->aeqe_size         = HNS_ROCE_AEQE_SIZE;
1894         caps->ceqe_size         = HNS_ROCE_CEQE_SIZE;
1895         caps->local_ca_ack_delay = 0;
1896         caps->max_mtu = IB_MTU_4096;
1897
1898         caps->max_srq_wrs       = HNS_ROCE_V2_MAX_SRQ_WR;
1899         caps->max_srq_sges      = HNS_ROCE_V2_MAX_SRQ_SGE;
1900
1901         caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
1902                        HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
1903                        HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1904
1905         caps->num_qpc_timer       = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1906         caps->qpc_timer_entry_sz  = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1907         caps->qpc_timer_ba_pg_sz  = 0;
1908         caps->qpc_timer_buf_pg_sz = 0;
1909         caps->qpc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;
1910         caps->num_cqc_timer       = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1911         caps->cqc_timer_entry_sz  = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1912         caps->cqc_timer_ba_pg_sz  = 0;
1913         caps->cqc_timer_buf_pg_sz = 0;
1914         caps->cqc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;
1915
1916         caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ;
1917         caps->sccc_ba_pg_sz       = 0;
1918         caps->sccc_buf_pg_sz      = 0;
1919         caps->sccc_hop_num        = HNS_ROCE_SCCC_HOP_NUM;
1920
1921         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
1922                 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
1923                 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
1924                 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
1925                 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
1926                 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
1927                 caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;
1928                 caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE /
1929                                                           caps->gmv_entry_sz);
1930                 caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;
1931                 caps->gmv_ba_pg_sz = 0;
1932                 caps->gmv_buf_pg_sz = 0;
1933                 caps->gid_table_len[0] = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /
1934                                          caps->gmv_entry_sz);
1935         }
1936 }
1937
1938 static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,
1939                        u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)
1940 {
1941         u64 obj_per_chunk;
1942         u64 bt_chunk_size = PAGE_SIZE;
1943         u64 buf_chunk_size = PAGE_SIZE;
1944         u64 obj_per_chunk_default = buf_chunk_size / obj_size;
1945
1946         *buf_page_size = 0;
1947         *bt_page_size = 0;
1948
1949         switch (hop_num) {
1950         case 3:
1951                 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1952                                 (bt_chunk_size / BA_BYTE_LEN) *
1953                                 (bt_chunk_size / BA_BYTE_LEN) *
1954                                  obj_per_chunk_default;
1955                 break;
1956         case 2:
1957                 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1958                                 (bt_chunk_size / BA_BYTE_LEN) *
1959                                  obj_per_chunk_default;
1960                 break;
1961         case 1:
1962                 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1963                                 obj_per_chunk_default;
1964                 break;
1965         case HNS_ROCE_HOP_NUM_0:
1966                 obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
1967                 break;
1968         default:
1969                 pr_err("table %u not support hop_num = %u!\n", hem_type,
1970                        hop_num);
1971                 return;
1972         }
1973
1974         if (hem_type >= HEM_TYPE_MTT)
1975                 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
1976         else
1977                 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
1978 }
1979
1980 static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
1981 {
1982         struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
1983         struct hns_roce_caps *caps = &hr_dev->caps;
1984         struct hns_roce_query_pf_caps_a *resp_a;
1985         struct hns_roce_query_pf_caps_b *resp_b;
1986         struct hns_roce_query_pf_caps_c *resp_c;
1987         struct hns_roce_query_pf_caps_d *resp_d;
1988         struct hns_roce_query_pf_caps_e *resp_e;
1989         int ctx_hop_num;
1990         int pbl_hop_num;
1991         int ret;
1992         int i;
1993
1994         for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
1995                 hns_roce_cmq_setup_basic_desc(&desc[i],
1996                                               HNS_ROCE_OPC_QUERY_PF_CAPS_NUM,
1997                                               true);
1998                 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
1999                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2000                 else
2001                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2002         }
2003
2004         ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
2005         if (ret)
2006                 return ret;
2007
2008         resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
2009         resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
2010         resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
2011         resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
2012         resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
2013
2014         caps->local_ca_ack_delay     = resp_a->local_ca_ack_delay;
2015         caps->max_sq_sg              = le16_to_cpu(resp_a->max_sq_sg);
2016         caps->max_sq_inline          = le16_to_cpu(resp_a->max_sq_inline);
2017         caps->max_rq_sg              = le16_to_cpu(resp_a->max_rq_sg);
2018         caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg);
2019         caps->max_extend_sg          = le32_to_cpu(resp_a->max_extend_sg);
2020         caps->num_qpc_timer          = le16_to_cpu(resp_a->num_qpc_timer);
2021         caps->num_cqc_timer          = le16_to_cpu(resp_a->num_cqc_timer);
2022         caps->max_srq_sges           = le16_to_cpu(resp_a->max_srq_sges);
2023         caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges);
2024         caps->num_aeq_vectors        = resp_a->num_aeq_vectors;
2025         caps->num_other_vectors      = resp_a->num_other_vectors;
2026         caps->max_sq_desc_sz         = resp_a->max_sq_desc_sz;
2027         caps->max_rq_desc_sz         = resp_a->max_rq_desc_sz;
2028         caps->max_srq_desc_sz        = resp_a->max_srq_desc_sz;
2029         caps->cqe_sz                 = HNS_ROCE_V2_CQE_SIZE;
2030
2031         caps->mtpt_entry_sz          = resp_b->mtpt_entry_sz;
2032         caps->irrl_entry_sz          = resp_b->irrl_entry_sz;
2033         caps->trrl_entry_sz          = resp_b->trrl_entry_sz;
2034         caps->cqc_entry_sz           = resp_b->cqc_entry_sz;
2035         caps->srqc_entry_sz          = resp_b->srqc_entry_sz;
2036         caps->idx_entry_sz           = resp_b->idx_entry_sz;
2037         caps->sccc_sz                = resp_b->sccc_sz;
2038         caps->max_mtu                = resp_b->max_mtu;
2039         caps->qpc_sz                 = HNS_ROCE_V2_QPC_SZ;
2040         caps->min_cqes               = resp_b->min_cqes;
2041         caps->min_wqes               = resp_b->min_wqes;
2042         caps->page_size_cap          = le32_to_cpu(resp_b->page_size_cap);
2043         caps->pkey_table_len[0]      = resp_b->pkey_table_len;
2044         caps->phy_num_uars           = resp_b->phy_num_uars;
2045         ctx_hop_num                  = resp_b->ctx_hop_num;
2046         pbl_hop_num                  = resp_b->pbl_hop_num;
2047
2048         caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds,
2049                                             V2_QUERY_PF_CAPS_C_NUM_PDS_M,
2050                                             V2_QUERY_PF_CAPS_C_NUM_PDS_S);
2051         caps->flags = roce_get_field(resp_c->cap_flags_num_pds,
2052                                      V2_QUERY_PF_CAPS_C_CAP_FLAGS_M,
2053                                      V2_QUERY_PF_CAPS_C_CAP_FLAGS_S);
2054         caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<
2055                        HNS_ROCE_CAP_FLAGS_EX_SHIFT;
2056
2057         caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs,
2058                                             V2_QUERY_PF_CAPS_C_NUM_CQS_M,
2059                                             V2_QUERY_PF_CAPS_C_NUM_CQS_S);
2060         caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs,
2061                                                 V2_QUERY_PF_CAPS_C_MAX_GID_M,
2062                                                 V2_QUERY_PF_CAPS_C_MAX_GID_S);
2063         caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth,
2064                                              V2_QUERY_PF_CAPS_C_CQ_DEPTH_M,
2065                                              V2_QUERY_PF_CAPS_C_CQ_DEPTH_S);
2066         caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws,
2067                                               V2_QUERY_PF_CAPS_C_NUM_MRWS_M,
2068                                               V2_QUERY_PF_CAPS_C_NUM_MRWS_S);
2069         caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps,
2070                                             V2_QUERY_PF_CAPS_C_NUM_QPS_M,
2071                                             V2_QUERY_PF_CAPS_C_NUM_QPS_S);
2072         caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps,
2073                                                 V2_QUERY_PF_CAPS_C_MAX_ORD_M,
2074                                                 V2_QUERY_PF_CAPS_C_MAX_ORD_S);
2075         caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
2076         caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
2077         caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs,
2078                                              V2_QUERY_PF_CAPS_D_NUM_SRQS_M,
2079                                              V2_QUERY_PF_CAPS_D_NUM_SRQS_S);
2080         caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
2081         caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth,
2082                                                V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M,
2083                                                V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S);
2084         caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth,
2085                                                 V2_QUERY_PF_CAPS_D_NUM_CEQS_M,
2086                                                 V2_QUERY_PF_CAPS_D_NUM_CEQS_S);
2087         caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth,
2088                                                V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M,
2089                                                V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S);
2090         caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
2091                                             V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M,
2092                                             V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S);
2093         caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
2094                                             V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M,
2095                                             V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S);
2096         caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds,
2097                                             V2_QUERY_PF_CAPS_D_RSV_PDS_M,
2098                                             V2_QUERY_PF_CAPS_D_RSV_PDS_S);
2099         caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds,
2100                                              V2_QUERY_PF_CAPS_D_NUM_UARS_M,
2101                                              V2_QUERY_PF_CAPS_D_NUM_UARS_S);
2102         caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps,
2103                                             V2_QUERY_PF_CAPS_D_RSV_QPS_M,
2104                                             V2_QUERY_PF_CAPS_D_RSV_QPS_S);
2105         caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps,
2106                                              V2_QUERY_PF_CAPS_D_RSV_UARS_M,
2107                                              V2_QUERY_PF_CAPS_D_RSV_UARS_S);
2108         caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
2109                                              V2_QUERY_PF_CAPS_E_RSV_MRWS_M,
2110                                              V2_QUERY_PF_CAPS_E_RSV_MRWS_S);
2111         caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
2112                                          V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M,
2113                                          V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S);
2114         caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs,
2115                                             V2_QUERY_PF_CAPS_E_RSV_CQS_M,
2116                                             V2_QUERY_PF_CAPS_E_RSV_CQS_S);
2117         caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs,
2118                                              V2_QUERY_PF_CAPS_E_RSV_SRQS_M,
2119                                              V2_QUERY_PF_CAPS_E_RSV_SRQS_S);
2120         caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey,
2121                                              V2_QUERY_PF_CAPS_E_RSV_LKEYS_M,
2122                                              V2_QUERY_PF_CAPS_E_RSV_LKEYS_S);
2123         caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
2124         caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
2125         caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
2126         caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
2127
2128         caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
2129         caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
2130         caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
2131         caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
2132         caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
2133         caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
2134         caps->mtt_ba_pg_sz = 0;
2135         caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
2136         caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
2137         caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
2138
2139         caps->qpc_hop_num = ctx_hop_num;
2140         caps->srqc_hop_num = ctx_hop_num;
2141         caps->cqc_hop_num = ctx_hop_num;
2142         caps->mpt_hop_num = ctx_hop_num;
2143         caps->mtt_hop_num = pbl_hop_num;
2144         caps->cqe_hop_num = pbl_hop_num;
2145         caps->srqwqe_hop_num = pbl_hop_num;
2146         caps->idx_hop_num = pbl_hop_num;
2147         caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
2148                                           V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M,
2149                                           V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S);
2150         caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
2151                                           V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M,
2152                                           V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S);
2153         caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
2154                                           V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M,
2155                                           V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S);
2156
2157         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2158                 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
2159                 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
2160                 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
2161                 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
2162                 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
2163                 caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;
2164                 caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE /
2165                                                     caps->gmv_entry_sz);
2166                 caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;
2167                 caps->gmv_ba_pg_sz = 0;
2168                 caps->gmv_buf_pg_sz = 0;
2169                 caps->gid_table_len[0] = caps->gmv_bt_num *
2170                                 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz);
2171         }
2172
2173         calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
2174                    caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
2175                    HEM_TYPE_QPC);
2176         calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
2177                    caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
2178                    HEM_TYPE_MTPT);
2179         calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
2180                    caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
2181                    HEM_TYPE_CQC);
2182         calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num,
2183                    caps->srqc_bt_num, &caps->srqc_buf_pg_sz,
2184                    &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);
2185
2186         caps->sccc_hop_num = ctx_hop_num;
2187         caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2188         caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2189
2190         calc_pg_sz(caps->num_qps, caps->sccc_sz,
2191                    caps->sccc_hop_num, caps->sccc_bt_num,
2192                    &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
2193                    HEM_TYPE_SCCC);
2194         calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
2195                    caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
2196                    &caps->cqc_timer_buf_pg_sz,
2197                    &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);
2198
2199         calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,
2200                    1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
2201         calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
2202                    caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
2203                    &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
2204         calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num,
2205                    1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX);
2206
2207         return 0;
2208 }
2209
2210 static int hns_roce_config_qpc_size(struct hns_roce_dev *hr_dev)
2211 {
2212         struct hns_roce_cmq_desc desc;
2213         struct hns_roce_cfg_entry_size *cfg_size =
2214                                   (struct hns_roce_cfg_entry_size *)desc.data;
2215
2216         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
2217                                       false);
2218
2219         cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_QPC_SIZE);
2220         cfg_size->size = cpu_to_le32(hr_dev->caps.qpc_sz);
2221
2222         return hns_roce_cmq_send(hr_dev, &desc, 1);
2223 }
2224
2225 static int hns_roce_config_sccc_size(struct hns_roce_dev *hr_dev)
2226 {
2227         struct hns_roce_cmq_desc desc;
2228         struct hns_roce_cfg_entry_size *cfg_size =
2229                                   (struct hns_roce_cfg_entry_size *)desc.data;
2230
2231         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
2232                                       false);
2233
2234         cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_SCCC_SIZE);
2235         cfg_size->size = cpu_to_le32(hr_dev->caps.sccc_sz);
2236
2237         return hns_roce_cmq_send(hr_dev, &desc, 1);
2238 }
2239
2240 static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)
2241 {
2242         int ret;
2243
2244         if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
2245                 return 0;
2246
2247         ret = hns_roce_config_qpc_size(hr_dev);
2248         if (ret) {
2249                 dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret);
2250                 return ret;
2251         }
2252
2253         ret = hns_roce_config_sccc_size(hr_dev);
2254         if (ret)
2255                 dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret);
2256
2257         return ret;
2258 }
2259
2260 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
2261 {
2262         struct hns_roce_caps *caps = &hr_dev->caps;
2263         int ret;
2264
2265         ret = hns_roce_cmq_query_hw_info(hr_dev);
2266         if (ret) {
2267                 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
2268                         ret);
2269                 return ret;
2270         }
2271
2272         ret = hns_roce_query_fw_ver(hr_dev);
2273         if (ret) {
2274                 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
2275                         ret);
2276                 return ret;
2277         }
2278
2279         ret = hns_roce_config_global_param(hr_dev);
2280         if (ret) {
2281                 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
2282                         ret);
2283                 return ret;
2284         }
2285
2286         /* Get pf resource owned by every pf */
2287         ret = hns_roce_query_pf_resource(hr_dev);
2288         if (ret) {
2289                 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
2290                         ret);
2291                 return ret;
2292         }
2293
2294         ret = hns_roce_query_pf_timer_resource(hr_dev);
2295         if (ret) {
2296                 dev_err(hr_dev->dev,
2297                         "failed to query pf timer resource, ret = %d.\n", ret);
2298                 return ret;
2299         }
2300
2301         ret = hns_roce_set_vf_switch_param(hr_dev, 0);
2302         if (ret) {
2303                 dev_err(hr_dev->dev,
2304                         "failed to set function switch param, ret = %d.\n",
2305                         ret);
2306                 return ret;
2307         }
2308
2309         hr_dev->vendor_part_id = hr_dev->pci_dev->device;
2310         hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
2311
2312         caps->pbl_ba_pg_sz      = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
2313         caps->pbl_buf_pg_sz     = 0;
2314         caps->pbl_hop_num       = HNS_ROCE_PBL_HOP_NUM;
2315         caps->eqe_ba_pg_sz      = 0;
2316         caps->eqe_buf_pg_sz     = 0;
2317         caps->eqe_hop_num       = HNS_ROCE_EQE_HOP_NUM;
2318         caps->tsq_buf_pg_sz     = 0;
2319
2320         ret = hns_roce_query_pf_caps(hr_dev);
2321         if (ret)
2322                 set_default_caps(hr_dev);
2323
2324         ret = hns_roce_alloc_vf_resource(hr_dev);
2325         if (ret) {
2326                 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
2327                         ret);
2328                 return ret;
2329         }
2330
2331         ret = hns_roce_v2_set_bt(hr_dev);
2332         if (ret) {
2333                 dev_err(hr_dev->dev,
2334                         "Configure bt attribute fail, ret = %d.\n", ret);
2335                 return ret;
2336         }
2337
2338         /* Configure the size of QPC, SCCC, etc. */
2339         ret = hns_roce_config_entry_size(hr_dev);
2340
2341         return ret;
2342 }
2343
2344 static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
2345                                       enum hns_roce_link_table_type type)
2346 {
2347         struct hns_roce_cmq_desc desc[2];
2348         struct hns_roce_cfg_llm_a *req_a =
2349                                 (struct hns_roce_cfg_llm_a *)desc[0].data;
2350         struct hns_roce_cfg_llm_b *req_b =
2351                                 (struct hns_roce_cfg_llm_b *)desc[1].data;
2352         struct hns_roce_v2_priv *priv = hr_dev->priv;
2353         struct hns_roce_link_table *link_tbl;
2354         struct hns_roce_link_table_entry *entry;
2355         enum hns_roce_opcode_type opcode;
2356         u32 page_num;
2357
2358         switch (type) {
2359         case TSQ_LINK_TABLE:
2360                 link_tbl = &priv->tsq;
2361                 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
2362                 break;
2363         case TPQ_LINK_TABLE:
2364                 link_tbl = &priv->tpq;
2365                 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
2366                 break;
2367         default:
2368                 return -EINVAL;
2369         }
2370
2371         page_num = link_tbl->npages;
2372         entry = link_tbl->table.buf;
2373
2374         hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
2375         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2376
2377         hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
2378
2379         req_a->base_addr_l = cpu_to_le32(link_tbl->table.map & 0xffffffff);
2380         req_a->base_addr_h = cpu_to_le32(link_tbl->table.map >> 32);
2381         roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_DEPTH_M,
2382                        CFG_LLM_QUE_DEPTH_S, link_tbl->npages);
2383         roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_PGSZ_M,
2384                        CFG_LLM_QUE_PGSZ_S, link_tbl->pg_sz);
2385         roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M,
2386                        CFG_LLM_INIT_EN_S, 1);
2387         req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
2388         req_a->head_ba_h_nxtptr = cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
2389         roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, CFG_LLM_HEAD_PTR_S,
2390                        0);
2391
2392         req_b->tail_ba_l = cpu_to_le32(entry[page_num - 1].blk_ba0);
2393         roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M,
2394                        CFG_LLM_TAIL_BA_H_S,
2395                        entry[page_num - 1].blk_ba1_nxt_ptr &
2396                        HNS_ROCE_LINK_TABLE_BA1_M);
2397         roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, CFG_LLM_TAIL_PTR_S,
2398                        (entry[page_num - 2].blk_ba1_nxt_ptr &
2399                         HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
2400                         HNS_ROCE_LINK_TABLE_NXT_PTR_S);
2401
2402         return hns_roce_cmq_send(hr_dev, desc, 2);
2403 }
2404
2405 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
2406                                     enum hns_roce_link_table_type type)
2407 {
2408         struct hns_roce_v2_priv *priv = hr_dev->priv;
2409         struct hns_roce_link_table *link_tbl;
2410         struct hns_roce_link_table_entry *entry;
2411         struct device *dev = hr_dev->dev;
2412         u32 buf_chk_sz;
2413         dma_addr_t t;
2414         int func_num = 1;
2415         u32 pg_num_a;
2416         u32 pg_num_b;
2417         u32 pg_num;
2418         u32 size;
2419         int i;
2420
2421         switch (type) {
2422         case TSQ_LINK_TABLE:
2423                 link_tbl = &priv->tsq;
2424                 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
2425                 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
2426                 pg_num_b = hr_dev->caps.sl_num * 4 + 2;
2427                 break;
2428         case TPQ_LINK_TABLE:
2429                 link_tbl = &priv->tpq;
2430                 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT);
2431                 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
2432                 pg_num_b = 2 * 4 * func_num + 2;
2433                 break;
2434         default:
2435                 return -EINVAL;
2436         }
2437
2438         pg_num = max(pg_num_a, pg_num_b);
2439         size = pg_num * sizeof(struct hns_roce_link_table_entry);
2440
2441         link_tbl->table.buf = dma_alloc_coherent(dev, size,
2442                                                  &link_tbl->table.map,
2443                                                  GFP_KERNEL);
2444         if (!link_tbl->table.buf)
2445                 goto out;
2446
2447         link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
2448                                     GFP_KERNEL);
2449         if (!link_tbl->pg_list)
2450                 goto err_kcalloc_failed;
2451
2452         entry = link_tbl->table.buf;
2453         for (i = 0; i < pg_num; ++i) {
2454                 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
2455                                                               &t, GFP_KERNEL);
2456                 if (!link_tbl->pg_list[i].buf)
2457                         goto err_alloc_buf_failed;
2458
2459                 link_tbl->pg_list[i].map = t;
2460
2461                 entry[i].blk_ba0 = (u32)(t >> 12);
2462                 entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44);
2463
2464                 if (i < (pg_num - 1))
2465                         entry[i].blk_ba1_nxt_ptr |=
2466                                 (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;
2467         }
2468         link_tbl->npages = pg_num;
2469         link_tbl->pg_sz = buf_chk_sz;
2470
2471         return hns_roce_config_link_table(hr_dev, type);
2472
2473 err_alloc_buf_failed:
2474         for (i -= 1; i >= 0; i--)
2475                 dma_free_coherent(dev, buf_chk_sz,
2476                                   link_tbl->pg_list[i].buf,
2477                                   link_tbl->pg_list[i].map);
2478         kfree(link_tbl->pg_list);
2479
2480 err_kcalloc_failed:
2481         dma_free_coherent(dev, size, link_tbl->table.buf,
2482                           link_tbl->table.map);
2483
2484 out:
2485         return -ENOMEM;
2486 }
2487
2488 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
2489                                      struct hns_roce_link_table *link_tbl)
2490 {
2491         struct device *dev = hr_dev->dev;
2492         int size;
2493         int i;
2494
2495         size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);
2496
2497         for (i = 0; i < link_tbl->npages; ++i)
2498                 if (link_tbl->pg_list[i].buf)
2499                         dma_free_coherent(dev, link_tbl->pg_sz,
2500                                           link_tbl->pg_list[i].buf,
2501                                           link_tbl->pg_list[i].map);
2502         kfree(link_tbl->pg_list);
2503
2504         dma_free_coherent(dev, size, link_tbl->table.buf,
2505                           link_tbl->table.map);
2506 }
2507
2508 static int get_hem_table(struct hns_roce_dev *hr_dev)
2509 {
2510         unsigned int qpc_count;
2511         unsigned int cqc_count;
2512         unsigned int gmv_count;
2513         int ret;
2514         int i;
2515
2516         /* Alloc memory for QPC Timer buffer space chunk */
2517         for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
2518              qpc_count++) {
2519                 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
2520                                          qpc_count);
2521                 if (ret) {
2522                         dev_err(hr_dev->dev, "QPC Timer get failed\n");
2523                         goto err_qpc_timer_failed;
2524                 }
2525         }
2526
2527         /* Alloc memory for CQC Timer buffer space chunk */
2528         for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
2529              cqc_count++) {
2530                 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
2531                                          cqc_count);
2532                 if (ret) {
2533                         dev_err(hr_dev->dev, "CQC Timer get failed\n");
2534                         goto err_cqc_timer_failed;
2535                 }
2536         }
2537
2538         /* Alloc memory for GMV(GID/MAC/VLAN) table buffer space chunk */
2539         for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num;
2540              gmv_count++) {
2541                 ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count);
2542                 if (ret) {
2543                         dev_err(hr_dev->dev,
2544                                 "failed to get gmv table, ret = %d.\n", ret);
2545                         goto err_gmv_failed;
2546                 }
2547         }
2548
2549         return 0;
2550
2551 err_gmv_failed:
2552         for (i = 0; i < gmv_count; i++)
2553                 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
2554
2555 err_cqc_timer_failed:
2556         for (i = 0; i < cqc_count; i++)
2557                 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2558
2559 err_qpc_timer_failed:
2560         for (i = 0; i < qpc_count; i++)
2561                 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2562
2563         return ret;
2564 }
2565
2566 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
2567 {
2568         struct hns_roce_v2_priv *priv = hr_dev->priv;
2569         int ret;
2570
2571         /* TSQ includes SQ doorbell and ack doorbell */
2572         ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
2573         if (ret) {
2574                 dev_err(hr_dev->dev, "failed to init TSQ, ret = %d.\n", ret);
2575                 return ret;
2576         }
2577
2578         ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
2579         if (ret) {
2580                 dev_err(hr_dev->dev, "failed to init TPQ, ret = %d.\n", ret);
2581                 goto err_tpq_init_failed;
2582         }
2583
2584         ret = get_hem_table(hr_dev);
2585         if (ret)
2586                 goto err_get_hem_table_failed;
2587
2588         return 0;
2589
2590 err_get_hem_table_failed:
2591         hns_roce_free_link_table(hr_dev, &priv->tpq);
2592
2593 err_tpq_init_failed:
2594         hns_roce_free_link_table(hr_dev, &priv->tsq);
2595
2596         return ret;
2597 }
2598
2599 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
2600 {
2601         struct hns_roce_v2_priv *priv = hr_dev->priv;
2602
2603         hns_roce_function_clear(hr_dev);
2604
2605         hns_roce_free_link_table(hr_dev, &priv->tpq);
2606         hns_roce_free_link_table(hr_dev, &priv->tsq);
2607 }
2608
2609 static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
2610 {
2611         struct hns_roce_cmq_desc desc;
2612         struct hns_roce_mbox_status *mb_st =
2613                                        (struct hns_roce_mbox_status *)desc.data;
2614         int status;
2615
2616         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
2617
2618         status = hns_roce_cmq_send(hr_dev, &desc, 1);
2619         if (status)
2620                 return status;
2621
2622         return le32_to_cpu(mb_st->mb_status_hw_run);
2623 }
2624
2625 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
2626 {
2627         u32 status = hns_roce_query_mbox_status(hr_dev);
2628
2629         return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
2630 }
2631
2632 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
2633 {
2634         u32 status = hns_roce_query_mbox_status(hr_dev);
2635
2636         return status & HNS_ROCE_HW_MB_STATUS_MASK;
2637 }
2638
2639 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
2640                               u64 out_param, u32 in_modifier, u8 op_modifier,
2641                               u16 op, u16 token, int event)
2642 {
2643         struct hns_roce_cmq_desc desc;
2644         struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
2645
2646         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
2647
2648         mb->in_param_l = cpu_to_le32(in_param);
2649         mb->in_param_h = cpu_to_le32(in_param >> 32);
2650         mb->out_param_l = cpu_to_le32(out_param);
2651         mb->out_param_h = cpu_to_le32(out_param >> 32);
2652         mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
2653         mb->token_event_en = cpu_to_le32(event << 16 | token);
2654
2655         return hns_roce_cmq_send(hr_dev, &desc, 1);
2656 }
2657
2658 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
2659                                  u64 out_param, u32 in_modifier, u8 op_modifier,
2660                                  u16 op, u16 token, int event)
2661 {
2662         struct device *dev = hr_dev->dev;
2663         unsigned long end;
2664         int ret;
2665
2666         end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
2667         while (hns_roce_v2_cmd_pending(hr_dev)) {
2668                 if (time_after(jiffies, end)) {
2669                         dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
2670                                 (int)end);
2671                         return -EAGAIN;
2672                 }
2673                 cond_resched();
2674         }
2675
2676         ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
2677                                  op_modifier, op, token, event);
2678         if (ret)
2679                 dev_err(dev, "Post mailbox fail(%d)\n", ret);
2680
2681         return ret;
2682 }
2683
2684 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
2685                                 unsigned int timeout)
2686 {
2687         struct device *dev = hr_dev->dev;
2688         unsigned long end;
2689         u32 status;
2690
2691         end = msecs_to_jiffies(timeout) + jiffies;
2692         while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
2693                 cond_resched();
2694
2695         if (hns_roce_v2_cmd_pending(hr_dev)) {
2696                 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
2697                 return -ETIMEDOUT;
2698         }
2699
2700         status = hns_roce_v2_cmd_complete(hr_dev);
2701         if (status != 0x1) {
2702                 if (status == CMD_RST_PRC_EBUSY)
2703                         return status;
2704
2705                 dev_err(dev, "mailbox status 0x%x!\n", status);
2706                 return -EBUSY;
2707         }
2708
2709         return 0;
2710 }
2711
2712 static void copy_gid(void *dest, const union ib_gid *gid)
2713 {
2714 #define GID_SIZE 4
2715         const union ib_gid *src = gid;
2716         __le32 (*p)[GID_SIZE] = dest;
2717         int i;
2718
2719         if (!gid)
2720                 src = &zgid;
2721
2722         for (i = 0; i < GID_SIZE; i++)
2723                 (*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]);
2724 }
2725
2726 static int config_sgid_table(struct hns_roce_dev *hr_dev,
2727                              int gid_index, const union ib_gid *gid,
2728                              enum hns_roce_sgid_type sgid_type)
2729 {
2730         struct hns_roce_cmq_desc desc;
2731         struct hns_roce_cfg_sgid_tb *sgid_tb =
2732                                     (struct hns_roce_cfg_sgid_tb *)desc.data;
2733
2734         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
2735
2736         roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M,
2737                        CFG_SGID_TB_TABLE_IDX_S, gid_index);
2738         roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M,
2739                        CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
2740
2741         copy_gid(&sgid_tb->vf_sgid_l, gid);
2742
2743         return hns_roce_cmq_send(hr_dev, &desc, 1);
2744 }
2745
2746 static int config_gmv_table(struct hns_roce_dev *hr_dev,
2747                             int gid_index, const union ib_gid *gid,
2748                             enum hns_roce_sgid_type sgid_type,
2749                             const struct ib_gid_attr *attr)
2750 {
2751         struct hns_roce_cmq_desc desc[2];
2752         struct hns_roce_cfg_gmv_tb_a *tb_a =
2753                                 (struct hns_roce_cfg_gmv_tb_a *)desc[0].data;
2754         struct hns_roce_cfg_gmv_tb_b *tb_b =
2755                                 (struct hns_roce_cfg_gmv_tb_b *)desc[1].data;
2756
2757         u16 vlan_id = VLAN_CFI_MASK;
2758         u8 mac[ETH_ALEN] = {};
2759         int ret;
2760
2761         if (gid) {
2762                 ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac);
2763                 if (ret)
2764                         return ret;
2765         }
2766
2767         hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false);
2768         desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2769
2770         hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false);
2771
2772         copy_gid(&tb_a->vf_sgid_l, gid);
2773
2774         roce_set_field(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_SGID_TYPE_M,
2775                        CFG_GMV_TB_VF_SGID_TYPE_S, sgid_type);
2776         roce_set_bit(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_VLAN_EN_S,
2777                      vlan_id < VLAN_CFI_MASK);
2778         roce_set_field(tb_a->vf_sgid_type_vlan, CFG_GMV_TB_VF_VLAN_ID_M,
2779                        CFG_GMV_TB_VF_VLAN_ID_S, vlan_id);
2780
2781         tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac);
2782         roce_set_field(tb_b->vf_smac_h, CFG_GMV_TB_SMAC_H_M,
2783                        CFG_GMV_TB_SMAC_H_S, *(u16 *)&mac[4]);
2784
2785         roce_set_field(tb_b->table_idx_rsv, CFG_GMV_TB_SGID_IDX_M,
2786                        CFG_GMV_TB_SGID_IDX_S, gid_index);
2787
2788         return hns_roce_cmq_send(hr_dev, desc, 2);
2789 }
2790
2791 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
2792                                int gid_index, const union ib_gid *gid,
2793                                const struct ib_gid_attr *attr)
2794 {
2795         enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
2796         int ret;
2797
2798         if (gid) {
2799                 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2800                         if (ipv6_addr_v4mapped((void *)gid))
2801                                 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2802                         else
2803                                 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2804                 } else if (attr->gid_type == IB_GID_TYPE_ROCE) {
2805                         sgid_type = GID_TYPE_FLAG_ROCE_V1;
2806                 }
2807         }
2808
2809         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
2810                 ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr);
2811         else
2812                 ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2813
2814         if (ret)
2815                 ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n",
2816                           ret);
2817
2818         return ret;
2819 }
2820
2821 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
2822                                u8 *addr)
2823 {
2824         struct hns_roce_cmq_desc desc;
2825         struct hns_roce_cfg_smac_tb *smac_tb =
2826                                     (struct hns_roce_cfg_smac_tb *)desc.data;
2827         u16 reg_smac_h;
2828         u32 reg_smac_l;
2829
2830         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
2831
2832         reg_smac_l = *(u32 *)(&addr[0]);
2833         reg_smac_h = *(u16 *)(&addr[4]);
2834
2835         roce_set_field(smac_tb->tb_idx_rsv, CFG_SMAC_TB_IDX_M,
2836                        CFG_SMAC_TB_IDX_S, phy_port);
2837         roce_set_field(smac_tb->vf_smac_h_rsv, CFG_SMAC_TB_VF_SMAC_H_M,
2838                        CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
2839         smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
2840
2841         return hns_roce_cmq_send(hr_dev, &desc, 1);
2842 }
2843
2844 static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
2845                         struct hns_roce_v2_mpt_entry *mpt_entry,
2846                         struct hns_roce_mr *mr)
2847 {
2848         u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
2849         struct ib_device *ibdev = &hr_dev->ib_dev;
2850         dma_addr_t pbl_ba;
2851         int i, count;
2852
2853         count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
2854                                   ARRAY_SIZE(pages), &pbl_ba);
2855         if (count < 1) {
2856                 ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
2857                           count);
2858                 return -ENOBUFS;
2859         }
2860
2861         /* Aligned to the hardware address access unit */
2862         for (i = 0; i < count; i++)
2863                 pages[i] >>= 6;
2864
2865         mpt_entry->pbl_size = cpu_to_le32(mr->npages);
2866         mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3);
2867         roce_set_field(mpt_entry->byte_48_mode_ba,
2868                        V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
2869                        upper_32_bits(pbl_ba >> 3));
2870
2871         mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
2872         roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
2873                        V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
2874
2875         mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
2876         roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
2877                        V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
2878         roce_set_field(mpt_entry->byte_64_buf_pa1,
2879                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2880                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2881                        to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
2882
2883         return 0;
2884 }
2885
2886 static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
2887                                   void *mb_buf, struct hns_roce_mr *mr,
2888                                   unsigned long mtpt_idx)
2889 {
2890         struct hns_roce_v2_mpt_entry *mpt_entry;
2891         int ret;
2892
2893         mpt_entry = mb_buf;
2894         memset(mpt_entry, 0, sizeof(*mpt_entry));
2895
2896         hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);
2897         hr_reg_write(mpt_entry, MPT_PD, mr->pd);
2898         hr_reg_enable(mpt_entry, MPT_L_INV_EN);
2899
2900         hr_reg_write(mpt_entry, MPT_BIND_EN,
2901                      !!(mr->access & IB_ACCESS_MW_BIND));
2902         hr_reg_write(mpt_entry, MPT_ATOMIC_EN,
2903                      !!(mr->access & IB_ACCESS_REMOTE_ATOMIC));
2904         hr_reg_write(mpt_entry, MPT_RR_EN,
2905                      !!(mr->access & IB_ACCESS_REMOTE_READ));
2906         hr_reg_write(mpt_entry, MPT_RW_EN,
2907                      !!(mr->access & IB_ACCESS_REMOTE_WRITE));
2908         hr_reg_write(mpt_entry, MPT_LW_EN,
2909                      !!((mr->access & IB_ACCESS_LOCAL_WRITE)));
2910
2911         mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
2912         mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
2913         mpt_entry->lkey = cpu_to_le32(mr->key);
2914         mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
2915         mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
2916
2917         if (mr->type != MR_TYPE_MR)
2918                 hr_reg_enable(mpt_entry, MPT_PA);
2919
2920         if (mr->type == MR_TYPE_DMA)
2921                 return 0;
2922
2923         if (mr->pbl_hop_num != HNS_ROCE_HOP_NUM_0)
2924                 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr->pbl_hop_num);
2925
2926         hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
2927                      to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
2928         hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD);
2929
2930         ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
2931
2932         return ret;
2933 }
2934
2935 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
2936                                         struct hns_roce_mr *mr, int flags,
2937                                         void *mb_buf)
2938 {
2939         struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
2940         u32 mr_access_flags = mr->access;
2941         int ret = 0;
2942
2943         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2944                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2945
2946         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2947                        V2_MPT_BYTE_4_PD_S, mr->pd);
2948
2949         if (flags & IB_MR_REREG_ACCESS) {
2950                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2951                              V2_MPT_BYTE_8_BIND_EN_S,
2952                              (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
2953                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2954                              V2_MPT_BYTE_8_ATOMIC_EN_S,
2955                              mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
2956                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2957                              mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
2958                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2959                              mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
2960                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2961                              mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
2962         }
2963
2964         if (flags & IB_MR_REREG_TRANS) {
2965                 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
2966                 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
2967                 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
2968                 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
2969
2970                 ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
2971         }
2972
2973         return ret;
2974 }
2975
2976 static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev,
2977                                        void *mb_buf, struct hns_roce_mr *mr)
2978 {
2979         struct ib_device *ibdev = &hr_dev->ib_dev;
2980         struct hns_roce_v2_mpt_entry *mpt_entry;
2981         dma_addr_t pbl_ba = 0;
2982
2983         mpt_entry = mb_buf;
2984         memset(mpt_entry, 0, sizeof(*mpt_entry));
2985
2986         if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) {
2987                 ibdev_err(ibdev, "failed to find frmr mtr.\n");
2988                 return -ENOBUFS;
2989         }
2990
2991         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2992                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2993         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2994                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
2995         roce_set_field(mpt_entry->byte_4_pd_hop_st,
2996                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2997                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2998                        to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
2999         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
3000                        V2_MPT_BYTE_4_PD_S, mr->pd);
3001
3002         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
3003         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
3004         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
3005
3006         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
3007         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
3008         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
3009         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
3010
3011         mpt_entry->pbl_size = cpu_to_le32(mr->npages);
3012
3013         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3));
3014         roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
3015                        V2_MPT_BYTE_48_PBL_BA_H_S,
3016                        upper_32_bits(pbl_ba >> 3));
3017
3018         roce_set_field(mpt_entry->byte_64_buf_pa1,
3019                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
3020                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
3021                        to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
3022
3023         return 0;
3024 }
3025
3026 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
3027 {
3028         struct hns_roce_v2_mpt_entry *mpt_entry;
3029
3030         mpt_entry = mb_buf;
3031         memset(mpt_entry, 0, sizeof(*mpt_entry));
3032
3033         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
3034                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
3035         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
3036                        V2_MPT_BYTE_4_PD_S, mw->pdn);
3037         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
3038                        V2_MPT_BYTE_4_PBL_HOP_NUM_S,
3039                        mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
3040                                                                mw->pbl_hop_num);
3041         roce_set_field(mpt_entry->byte_4_pd_hop_st,
3042                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
3043                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
3044                        mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
3045
3046         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
3047         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
3048         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 1);
3049
3050         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
3051         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
3052         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
3053         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
3054                      mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
3055
3056         roce_set_field(mpt_entry->byte_64_buf_pa1,
3057                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
3058                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
3059                        mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
3060
3061         mpt_entry->lkey = cpu_to_le32(mw->rkey);
3062
3063         return 0;
3064 }
3065
3066 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
3067 {
3068         return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
3069 }
3070
3071 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n)
3072 {
3073         struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
3074
3075         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
3076         return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
3077                 !!(n & hr_cq->cq_depth)) ? cqe : NULL;
3078 }
3079
3080 static inline void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 ci)
3081 {
3082         *hr_cq->set_ci_db = ci & V2_CQ_DB_PARAMETER_CONS_IDX_M;
3083 }
3084
3085 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3086                                    struct hns_roce_srq *srq)
3087 {
3088         struct hns_roce_v2_cqe *cqe, *dest;
3089         u32 prod_index;
3090         int nfreed = 0;
3091         int wqe_index;
3092         u8 owner_bit;
3093
3094         for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
3095              ++prod_index) {
3096                 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
3097                         break;
3098         }
3099
3100         /*
3101          * Now backwards through the CQ, removing CQ entries
3102          * that match our QP by overwriting them with next entries.
3103          */
3104         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
3105                 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
3106                 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
3107                                     V2_CQE_BYTE_16_LCL_QPN_S) &
3108                                     HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
3109                         if (srq &&
3110                             roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
3111                                 wqe_index = roce_get_field(cqe->byte_4,
3112                                                      V2_CQE_BYTE_4_WQE_INDX_M,
3113                                                      V2_CQE_BYTE_4_WQE_INDX_S);
3114                                 hns_roce_free_srq_wqe(srq, wqe_index);
3115                         }
3116                         ++nfreed;
3117                 } else if (nfreed) {
3118                         dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
3119                                           hr_cq->ib_cq.cqe);
3120                         owner_bit = roce_get_bit(dest->byte_4,
3121                                                  V2_CQE_BYTE_4_OWNER_S);
3122                         memcpy(dest, cqe, sizeof(*cqe));
3123                         roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
3124                                      owner_bit);
3125                 }
3126         }
3127
3128         if (nfreed) {
3129                 hr_cq->cons_index += nfreed;
3130                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
3131         }
3132 }
3133
3134 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3135                                  struct hns_roce_srq *srq)
3136 {
3137         spin_lock_irq(&hr_cq->lock);
3138         __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
3139         spin_unlock_irq(&hr_cq->lock);
3140 }
3141
3142 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
3143                                   struct hns_roce_cq *hr_cq, void *mb_buf,
3144                                   u64 *mtts, dma_addr_t dma_handle)
3145 {
3146         struct hns_roce_v2_cq_context *cq_context;
3147
3148         cq_context = mb_buf;
3149         memset(cq_context, 0, sizeof(*cq_context));
3150
3151         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
3152                        V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
3153         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
3154                        V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
3155         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
3156                        V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth));
3157         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
3158                        V2_CQC_BYTE_4_CEQN_S, hr_cq->vector);
3159
3160         roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
3161                        V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
3162
3163         roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQE_SIZE_M,
3164                        V2_CQC_BYTE_8_CQE_SIZE_S, hr_cq->cqe_size ==
3165                        HNS_ROCE_V3_CQE_SIZE ? 1 : 0);
3166
3167         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
3168                 hr_reg_enable(cq_context, CQC_STASH);
3169
3170         cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
3171
3172         roce_set_field(cq_context->byte_16_hop_addr,
3173                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
3174                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
3175                        upper_32_bits(to_hr_hw_page_addr(mtts[0])));
3176         roce_set_field(cq_context->byte_16_hop_addr,
3177                        V2_CQC_BYTE_16_CQE_HOP_NUM_M,
3178                        V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
3179                        HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
3180
3181         cq_context->cqe_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
3182         roce_set_field(cq_context->byte_24_pgsz_addr,
3183                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
3184                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
3185                        upper_32_bits(to_hr_hw_page_addr(mtts[1])));
3186         roce_set_field(cq_context->byte_24_pgsz_addr,
3187                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
3188                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
3189                        to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
3190         roce_set_field(cq_context->byte_24_pgsz_addr,
3191                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
3192                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
3193                        to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));
3194
3195         cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);
3196
3197         roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
3198                        V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
3199
3200         roce_set_bit(cq_context->byte_44_db_record,
3201                      V2_CQC_BYTE_44_DB_RECORD_EN_S,
3202                      (hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB) ? 1 : 0);
3203
3204         roce_set_field(cq_context->byte_44_db_record,
3205                        V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
3206                        V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
3207                        ((u32)hr_cq->db.dma) >> 1);
3208         cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32);
3209
3210         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
3211                        V2_CQC_BYTE_56_CQ_MAX_CNT_M,
3212                        V2_CQC_BYTE_56_CQ_MAX_CNT_S,
3213                        HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
3214         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
3215                        V2_CQC_BYTE_56_CQ_PERIOD_M,
3216                        V2_CQC_BYTE_56_CQ_PERIOD_S,
3217                        HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
3218 }
3219
3220 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
3221                                      enum ib_cq_notify_flags flags)
3222 {
3223         struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3224         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3225         u32 notification_flag;
3226         __le32 doorbell[2];
3227
3228         doorbell[0] = 0;
3229         doorbell[1] = 0;
3230
3231         notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
3232                              V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
3233         /*
3234          * flags = 0; Notification Flag = 1, next
3235          * flags = 1; Notification Flag = 0, solocited
3236          */
3237         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
3238                        hr_cq->cqn);
3239         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
3240                        HNS_ROCE_V2_CQ_DB_NTR);
3241         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
3242                        V2_CQ_DB_PARAMETER_CONS_IDX_S, hr_cq->cons_index);
3243         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
3244                        V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
3245         roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
3246                      notification_flag);
3247
3248         hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
3249
3250         return 0;
3251 }
3252
3253 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
3254                                                     struct hns_roce_qp **cur_qp,
3255                                                     struct ib_wc *wc)
3256 {
3257         struct hns_roce_rinl_sge *sge_list;
3258         u32 wr_num, wr_cnt, sge_num;
3259         u32 sge_cnt, data_len, size;
3260         void *wqe_buf;
3261
3262         wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
3263                                 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
3264         wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
3265
3266         sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
3267         sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
3268         wqe_buf = hns_roce_get_recv_wqe(*cur_qp, wr_cnt);
3269         data_len = wc->byte_len;
3270
3271         for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
3272                 size = min(sge_list[sge_cnt].len, data_len);
3273                 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
3274
3275                 data_len -= size;
3276                 wqe_buf += size;
3277         }
3278
3279         if (unlikely(data_len)) {
3280                 wc->status = IB_WC_LOC_LEN_ERR;
3281                 return -EAGAIN;
3282         }
3283
3284         return 0;
3285 }
3286
3287 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
3288                    int num_entries, struct ib_wc *wc)
3289 {
3290         unsigned int left;
3291         int npolled = 0;
3292
3293         left = wq->head - wq->tail;
3294         if (left == 0)
3295                 return 0;
3296
3297         left = min_t(unsigned int, (unsigned int)num_entries, left);
3298         while (npolled < left) {
3299                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3300                 wc->status = IB_WC_WR_FLUSH_ERR;
3301                 wc->vendor_err = 0;
3302                 wc->qp = &hr_qp->ibqp;
3303
3304                 wq->tail++;
3305                 wc++;
3306                 npolled++;
3307         }
3308
3309         return npolled;
3310 }
3311
3312 static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,
3313                                   struct ib_wc *wc)
3314 {
3315         struct hns_roce_qp *hr_qp;
3316         int npolled = 0;
3317
3318         list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
3319                 npolled += sw_comp(hr_qp, &hr_qp->sq,
3320                                    num_entries - npolled, wc + npolled);
3321                 if (npolled >= num_entries)
3322                         goto out;
3323         }
3324
3325         list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
3326                 npolled += sw_comp(hr_qp, &hr_qp->rq,
3327                                    num_entries - npolled, wc + npolled);
3328                 if (npolled >= num_entries)
3329                         goto out;
3330         }
3331
3332 out:
3333         return npolled;
3334 }
3335
3336 static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
3337                            struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe,
3338                            struct ib_wc *wc)
3339 {
3340         static const struct {
3341                 u32 cqe_status;
3342                 enum ib_wc_status wc_status;
3343         } map[] = {
3344                 { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },
3345                 { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },
3346                 { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
3347                 { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },
3348                 { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
3349                 { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },
3350                 { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
3351                 { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
3352                 { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
3353                 { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
3354                 { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },
3355                 { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,
3356                   IB_WC_RETRY_EXC_ERR },
3357                 { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
3358                 { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
3359                 { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
3360         };
3361
3362         u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
3363                                         V2_CQE_BYTE_4_STATUS_S);
3364         int i;
3365
3366         wc->status = IB_WC_GENERAL_ERR;
3367         for (i = 0; i < ARRAY_SIZE(map); i++)
3368                 if (cqe_status == map[i].cqe_status) {
3369                         wc->status = map[i].wc_status;
3370                         break;
3371                 }
3372
3373         if (likely(wc->status == IB_WC_SUCCESS ||
3374                    wc->status == IB_WC_WR_FLUSH_ERR))
3375                 return;
3376
3377         ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status);
3378         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe,
3379                        cq->cqe_size, false);
3380
3381         /*
3382          * For hns ROCEE, GENERAL_ERR is an error type that is not defined in
3383          * the standard protocol, the driver must ignore it and needn't to set
3384          * the QP to an error state.
3385          */
3386         if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
3387                 return;
3388
3389         /*
3390          * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets
3391          * into errored mode. Hence, as a workaround to this hardware
3392          * limitation, driver needs to assist in flushing. But the flushing
3393          * operation uses mailbox to convey the QP state to the hardware and
3394          * which can sleep due to the mutex protection around the mailbox calls.
3395          * Hence, use the deferred flush for now. Once wc error detected, the
3396          * flushing operation is needed.
3397          */
3398         if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
3399                 init_flush_work(hr_dev, qp);
3400 }
3401
3402 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
3403                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
3404 {
3405         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3406         struct hns_roce_srq *srq = NULL;
3407         struct hns_roce_v2_cqe *cqe;
3408         struct hns_roce_qp *hr_qp;
3409         struct hns_roce_wq *wq;
3410         int is_send;
3411         u16 wqe_ctr;
3412         u32 opcode;
3413         u32 qpn;
3414         int ret;
3415
3416         /* Find cqe according to consumer index */
3417         cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
3418         if (!cqe)
3419                 return -EAGAIN;
3420
3421         ++hr_cq->cons_index;
3422         /* Memory barrier */
3423         rmb();
3424
3425         /* 0->SQ, 1->RQ */
3426         is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
3427
3428         qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
3429                                 V2_CQE_BYTE_16_LCL_QPN_S);
3430
3431         if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
3432                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
3433                 if (unlikely(!hr_qp)) {
3434                         ibdev_err(&hr_dev->ib_dev,
3435                                   "CQ %06lx with entry for unknown QPN %06x\n",
3436                                   hr_cq->cqn, qpn & HNS_ROCE_V2_CQE_QPN_MASK);
3437                         return -EINVAL;
3438                 }
3439                 *cur_qp = hr_qp;
3440         }
3441
3442         wc->qp = &(*cur_qp)->ibqp;
3443         wc->vendor_err = 0;
3444
3445         if (is_send) {
3446                 wq = &(*cur_qp)->sq;
3447                 if ((*cur_qp)->sq_signal_bits) {
3448                         /*
3449                          * If sg_signal_bit is 1,
3450                          * firstly tail pointer updated to wqe
3451                          * which current cqe correspond to
3452                          */
3453                         wqe_ctr = (u16)roce_get_field(cqe->byte_4,
3454                                                       V2_CQE_BYTE_4_WQE_INDX_M,
3455                                                       V2_CQE_BYTE_4_WQE_INDX_S);
3456                         wq->tail += (wqe_ctr - (u16)wq->tail) &
3457                                     (wq->wqe_cnt - 1);
3458                 }
3459
3460                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3461                 ++wq->tail;
3462         } else if ((*cur_qp)->ibqp.srq) {
3463                 srq = to_hr_srq((*cur_qp)->ibqp.srq);
3464                 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
3465                                               V2_CQE_BYTE_4_WQE_INDX_M,
3466                                               V2_CQE_BYTE_4_WQE_INDX_S);
3467                 wc->wr_id = srq->wrid[wqe_ctr];
3468                 hns_roce_free_srq_wqe(srq, wqe_ctr);
3469         } else {
3470                 /* Update tail pointer, record wr_id */
3471                 wq = &(*cur_qp)->rq;
3472                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3473                 ++wq->tail;
3474         }
3475
3476         get_cqe_status(hr_dev, *cur_qp, hr_cq, cqe, wc);
3477         if (unlikely(wc->status != IB_WC_SUCCESS))
3478                 return 0;
3479
3480         if (is_send) {
3481                 wc->wc_flags = 0;
3482                 /* SQ corresponding to CQE */
3483                 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
3484                                        V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
3485                 case HNS_ROCE_V2_WQE_OP_SEND:
3486                         wc->opcode = IB_WC_SEND;
3487                         break;
3488                 case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV:
3489                         wc->opcode = IB_WC_SEND;
3490                         break;
3491                 case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
3492                         wc->opcode = IB_WC_SEND;
3493                         wc->wc_flags |= IB_WC_WITH_IMM;
3494                         break;
3495                 case HNS_ROCE_V2_WQE_OP_RDMA_READ:
3496                         wc->opcode = IB_WC_RDMA_READ;
3497                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3498                         break;
3499                 case HNS_ROCE_V2_WQE_OP_RDMA_WRITE:
3500                         wc->opcode = IB_WC_RDMA_WRITE;
3501                         break;
3502                 case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
3503                         wc->opcode = IB_WC_RDMA_WRITE;
3504                         wc->wc_flags |= IB_WC_WITH_IMM;
3505                         break;
3506                 case HNS_ROCE_V2_WQE_OP_LOCAL_INV:
3507                         wc->opcode = IB_WC_LOCAL_INV;
3508                         wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3509                         break;
3510                 case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
3511                         wc->opcode = IB_WC_COMP_SWAP;
3512                         wc->byte_len  = 8;
3513                         break;
3514                 case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
3515                         wc->opcode = IB_WC_FETCH_ADD;
3516                         wc->byte_len  = 8;
3517                         break;
3518                 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
3519                         wc->opcode = IB_WC_MASKED_COMP_SWAP;
3520                         wc->byte_len  = 8;
3521                         break;
3522                 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
3523                         wc->opcode = IB_WC_MASKED_FETCH_ADD;
3524                         wc->byte_len  = 8;
3525                         break;
3526                 case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR:
3527                         wc->opcode = IB_WC_REG_MR;
3528                         break;
3529                 case HNS_ROCE_V2_WQE_OP_BIND_MW:
3530                         wc->opcode = IB_WC_REG_MR;
3531                         break;
3532                 default:
3533                         wc->status = IB_WC_GENERAL_ERR;
3534                         break;
3535                 }
3536         } else {
3537                 /* RQ correspond to CQE */
3538                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3539
3540                 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
3541                                         V2_CQE_BYTE_4_OPCODE_S);
3542                 switch (opcode & 0x1f) {
3543                 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
3544                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3545                         wc->wc_flags = IB_WC_WITH_IMM;
3546                         wc->ex.imm_data =
3547                                 cpu_to_be32(le32_to_cpu(cqe->immtdata));
3548                         break;
3549                 case HNS_ROCE_V2_OPCODE_SEND:
3550                         wc->opcode = IB_WC_RECV;
3551                         wc->wc_flags = 0;
3552                         break;
3553                 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
3554                         wc->opcode = IB_WC_RECV;
3555                         wc->wc_flags = IB_WC_WITH_IMM;
3556                         wc->ex.imm_data =
3557                                 cpu_to_be32(le32_to_cpu(cqe->immtdata));
3558                         break;
3559                 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
3560                         wc->opcode = IB_WC_RECV;
3561                         wc->wc_flags = IB_WC_WITH_INVALIDATE;
3562                         wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
3563                         break;
3564                 default:
3565                         wc->status = IB_WC_GENERAL_ERR;
3566                         break;
3567                 }
3568
3569                 if ((wc->qp->qp_type == IB_QPT_RC ||
3570                      wc->qp->qp_type == IB_QPT_UC) &&
3571                     (opcode == HNS_ROCE_V2_OPCODE_SEND ||
3572                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
3573                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
3574                     (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
3575                         ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
3576                         if (unlikely(ret))
3577                                 return -EAGAIN;
3578                 }
3579
3580                 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
3581                                             V2_CQE_BYTE_32_SL_S);
3582                 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
3583                                                 V2_CQE_BYTE_32_RMT_QPN_M,
3584                                                 V2_CQE_BYTE_32_RMT_QPN_S);
3585                 wc->slid = 0;
3586                 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
3587                                               V2_CQE_BYTE_32_GRH_S) ?
3588                                               IB_WC_GRH : 0);
3589                 wc->port_num = roce_get_field(cqe->byte_32,
3590                                 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
3591                 wc->pkey_index = 0;
3592
3593                 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
3594                         wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
3595                                                           V2_CQE_BYTE_28_VID_M,
3596                                                           V2_CQE_BYTE_28_VID_S);
3597                         wc->wc_flags |= IB_WC_WITH_VLAN;
3598                 } else {
3599                         wc->vlan_id = 0xffff;
3600                 }
3601
3602                 wc->network_hdr_type = roce_get_field(cqe->byte_28,
3603                                                     V2_CQE_BYTE_28_PORT_TYPE_M,
3604                                                     V2_CQE_BYTE_28_PORT_TYPE_S);
3605         }
3606
3607         return 0;
3608 }
3609
3610 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
3611                                struct ib_wc *wc)
3612 {
3613         struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3614         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3615         struct hns_roce_qp *cur_qp = NULL;
3616         unsigned long flags;
3617         int npolled;
3618
3619         spin_lock_irqsave(&hr_cq->lock, flags);
3620
3621         /*
3622          * When the device starts to reset, the state is RST_DOWN. At this time,
3623          * there may still be some valid CQEs in the hardware that are not
3624          * polled. Therefore, it is not allowed to switch to the software mode
3625          * immediately. When the state changes to UNINIT, CQE no longer exists
3626          * in the hardware, and then switch to software mode.
3627          */
3628         if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {
3629                 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);
3630                 goto out;
3631         }
3632
3633         for (npolled = 0; npolled < num_entries; ++npolled) {
3634                 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
3635                         break;
3636         }
3637
3638         if (npolled)
3639                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
3640
3641 out:
3642         spin_unlock_irqrestore(&hr_cq->lock, flags);
3643
3644         return npolled;
3645 }
3646
3647 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
3648                               int step_idx)
3649 {
3650         int op;
3651
3652         if (type == HEM_TYPE_SCCC && step_idx)
3653                 return -EINVAL;
3654
3655         switch (type) {
3656         case HEM_TYPE_QPC:
3657                 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
3658                 break;
3659         case HEM_TYPE_MTPT:
3660                 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
3661                 break;
3662         case HEM_TYPE_CQC:
3663                 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
3664                 break;
3665         case HEM_TYPE_SRQC:
3666                 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
3667                 break;
3668         case HEM_TYPE_SCCC:
3669                 op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
3670                 break;
3671         case HEM_TYPE_QPC_TIMER:
3672                 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
3673                 break;
3674         case HEM_TYPE_CQC_TIMER:
3675                 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
3676                 break;
3677         default:
3678                 dev_warn(hr_dev->dev,
3679                          "table %u not to be written by mailbox!\n", type);
3680                 return -EINVAL;
3681         }
3682
3683         return op + step_idx;
3684 }
3685
3686 static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba,
3687                          u32 hem_type, int step_idx)
3688 {
3689         struct hns_roce_cmd_mailbox *mailbox;
3690         struct hns_roce_cmq_desc desc;
3691         struct hns_roce_cfg_gmv_bt *gmv_bt =
3692                                 (struct hns_roce_cfg_gmv_bt *)desc.data;
3693         int ret;
3694         int op;
3695
3696         if (hem_type == HEM_TYPE_GMV) {
3697                 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT,
3698                                               false);
3699
3700                 gmv_bt->gmv_ba_l = cpu_to_le32(bt_ba >> HNS_HW_PAGE_SHIFT);
3701                 gmv_bt->gmv_ba_h = cpu_to_le32(bt_ba >> (HNS_HW_PAGE_SHIFT +
3702                                                          32));
3703                 gmv_bt->gmv_bt_idx = cpu_to_le32(obj /
3704                         (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz));
3705
3706                 return hns_roce_cmq_send(hr_dev, &desc, 1);
3707         }
3708
3709         op = get_op_for_set_hem(hr_dev, hem_type, step_idx);
3710         if (op < 0)
3711                 return 0;
3712
3713         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3714         if (IS_ERR(mailbox))
3715                 return PTR_ERR(mailbox);
3716
3717         ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
3718                                 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
3719
3720         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3721
3722         return ret;
3723 }
3724
3725 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
3726                                struct hns_roce_hem_table *table, int obj,
3727                                int step_idx)
3728 {
3729         struct hns_roce_hem_iter iter;
3730         struct hns_roce_hem_mhop mhop;
3731         struct hns_roce_hem *hem;
3732         unsigned long mhop_obj = obj;
3733         int i, j, k;
3734         int ret = 0;
3735         u64 hem_idx = 0;
3736         u64 l1_idx = 0;
3737         u64 bt_ba = 0;
3738         u32 chunk_ba_num;
3739         u32 hop_num;
3740
3741         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3742                 return 0;
3743
3744         hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
3745         i = mhop.l0_idx;
3746         j = mhop.l1_idx;
3747         k = mhop.l2_idx;
3748         hop_num = mhop.hop_num;
3749         chunk_ba_num = mhop.bt_chunk_size / 8;
3750
3751         if (hop_num == 2) {
3752                 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
3753                           k;
3754                 l1_idx = i * chunk_ba_num + j;
3755         } else if (hop_num == 1) {
3756                 hem_idx = i * chunk_ba_num + j;
3757         } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
3758                 hem_idx = i;
3759         }
3760
3761         if (table->type == HEM_TYPE_SCCC)
3762                 obj = mhop.l0_idx;
3763
3764         if (check_whether_last_step(hop_num, step_idx)) {
3765                 hem = table->hem[hem_idx];
3766                 for (hns_roce_hem_first(hem, &iter);
3767                      !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
3768                         bt_ba = hns_roce_hem_addr(&iter);
3769                         ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type,
3770                                             step_idx);
3771                 }
3772         } else {
3773                 if (step_idx == 0)
3774                         bt_ba = table->bt_l0_dma_addr[i];
3775                 else if (step_idx == 1 && hop_num == 2)
3776                         bt_ba = table->bt_l1_dma_addr[l1_idx];
3777
3778                 ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
3779         }
3780
3781         return ret;
3782 }
3783
3784 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
3785                                  struct hns_roce_hem_table *table, int obj,
3786                                  int step_idx)
3787 {
3788         struct device *dev = hr_dev->dev;
3789         struct hns_roce_cmd_mailbox *mailbox;
3790         int ret;
3791         u16 op = 0xff;
3792
3793         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3794                 return 0;
3795
3796         switch (table->type) {
3797         case HEM_TYPE_QPC:
3798                 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
3799                 break;
3800         case HEM_TYPE_MTPT:
3801                 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
3802                 break;
3803         case HEM_TYPE_CQC:
3804                 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
3805                 break;
3806         case HEM_TYPE_SRQC:
3807                 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
3808                 break;
3809         case HEM_TYPE_SCCC:
3810         case HEM_TYPE_QPC_TIMER:
3811         case HEM_TYPE_CQC_TIMER:
3812         case HEM_TYPE_GMV:
3813                 return 0;
3814         default:
3815                 dev_warn(dev, "table %u not to be destroyed by mailbox!\n",
3816                          table->type);
3817                 return 0;
3818         }
3819
3820         op += step_idx;
3821
3822         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3823         if (IS_ERR(mailbox))
3824                 return PTR_ERR(mailbox);
3825
3826         /* configure the tag and op */
3827         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
3828                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
3829
3830         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3831         return ret;
3832 }
3833
3834 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
3835                                  struct hns_roce_v2_qp_context *context,
3836                                  struct hns_roce_v2_qp_context *qpc_mask,
3837                                  struct hns_roce_qp *hr_qp)
3838 {
3839         struct hns_roce_cmd_mailbox *mailbox;
3840         int qpc_size;
3841         int ret;
3842
3843         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3844         if (IS_ERR(mailbox))
3845                 return PTR_ERR(mailbox);
3846
3847         /* The qpc size of HIP08 is only 256B, which is half of HIP09 */
3848         qpc_size = hr_dev->caps.qpc_sz;
3849         memcpy(mailbox->buf, context, qpc_size);
3850         memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);
3851
3852         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
3853                                 HNS_ROCE_CMD_MODIFY_QPC,
3854                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
3855
3856         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3857
3858         return ret;
3859 }
3860
3861 static void set_access_flags(struct hns_roce_qp *hr_qp,
3862                              struct hns_roce_v2_qp_context *context,
3863                              struct hns_roce_v2_qp_context *qpc_mask,
3864                              const struct ib_qp_attr *attr, int attr_mask)
3865 {
3866         u8 dest_rd_atomic;
3867         u32 access_flags;
3868
3869         dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
3870                          attr->max_dest_rd_atomic : hr_qp->resp_depth;
3871
3872         access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
3873                        attr->qp_access_flags : hr_qp->atomic_rd_en;
3874
3875         if (!dest_rd_atomic)
3876                 access_flags &= IB_ACCESS_REMOTE_WRITE;
3877
3878         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3879                      !!(access_flags & IB_ACCESS_REMOTE_READ));
3880         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
3881
3882         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3883                      !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3884         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
3885
3886         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3887                      !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3888         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
3889         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S,
3890                      !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3891         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 0);
3892 }
3893
3894 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
3895                             struct hns_roce_v2_qp_context *context,
3896                             struct hns_roce_v2_qp_context *qpc_mask)
3897 {
3898         roce_set_field(context->byte_4_sqpn_tst,
3899                        V2_QPC_BYTE_4_SGE_SHIFT_M, V2_QPC_BYTE_4_SGE_SHIFT_S,
3900                        to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,
3901                                                hr_qp->sge.sge_shift));
3902
3903         roce_set_field(context->byte_20_smac_sgid_idx,
3904                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3905                        ilog2(hr_qp->sq.wqe_cnt));
3906
3907         roce_set_field(context->byte_20_smac_sgid_idx,
3908                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3909                        ilog2(hr_qp->rq.wqe_cnt));
3910 }
3911
3912 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
3913                                     const struct ib_qp_attr *attr,
3914                                     int attr_mask,
3915                                     struct hns_roce_v2_qp_context *context,
3916                                     struct hns_roce_v2_qp_context *qpc_mask)
3917 {
3918         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3919         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3920
3921         /*
3922          * In v2 engine, software pass context and context mask to hardware
3923          * when modifying qp. If software need modify some fields in context,
3924          * we should set all bits of the relevant fields in context mask to
3925          * 0 at the same time, else set them to 0x1.
3926          */
3927         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3928                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3929
3930         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3931                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3932
3933         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3934                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3935
3936         roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3937                        V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
3938
3939         set_qpc_wqe_cnt(hr_qp, context, qpc_mask);
3940
3941         /* No VLAN need to set 0xFFF */
3942         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3943                        V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
3944
3945         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
3946                 roce_set_bit(context->byte_68_rq_db,
3947                              V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
3948
3949         roce_set_field(context->byte_68_rq_db,
3950                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3951                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
3952                        ((u32)hr_qp->rdb.dma) >> 1);
3953         context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32);
3954
3955         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
3956                     (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
3957
3958         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3959                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3960         if (ibqp->srq) {
3961                 roce_set_field(context->byte_76_srqn_op_en,
3962                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3963                                to_hr_srq(ibqp->srq)->srqn);
3964                 roce_set_bit(context->byte_76_srqn_op_en,
3965                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
3966         }
3967
3968         roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
3969
3970         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3971                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3972
3973         if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ)
3974                 return;
3975
3976         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
3977                 hr_reg_enable(&context->ext, QPCEX_STASH);
3978 }
3979
3980 static void modify_qp_init_to_init(struct ib_qp *ibqp,
3981                                    const struct ib_qp_attr *attr, int attr_mask,
3982                                    struct hns_roce_v2_qp_context *context,
3983                                    struct hns_roce_v2_qp_context *qpc_mask)
3984 {
3985         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3986
3987         /*
3988          * In v2 engine, software pass context and context mask to hardware
3989          * when modifying qp. If software need modify some fields in context,
3990          * we should set all bits of the relevant fields in context mask to
3991          * 0 at the same time, else set them to 0x1.
3992          */
3993         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3994                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3995         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3996                        V2_QPC_BYTE_4_TST_S, 0);
3997
3998         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3999                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
4000         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
4001                        V2_QPC_BYTE_16_PD_S, 0);
4002
4003         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
4004                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
4005         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
4006                        V2_QPC_BYTE_80_RX_CQN_S, 0);
4007
4008         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
4009                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
4010         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
4011                        V2_QPC_BYTE_252_TX_CQN_S, 0);
4012
4013         if (ibqp->srq) {
4014                 roce_set_bit(context->byte_76_srqn_op_en,
4015                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
4016                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
4017                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
4018                 roce_set_field(context->byte_76_srqn_op_en,
4019                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
4020                                to_hr_srq(ibqp->srq)->srqn);
4021                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
4022                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
4023         }
4024
4025         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
4026                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
4027         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
4028                        V2_QPC_BYTE_4_SQPN_S, 0);
4029
4030         if (attr_mask & IB_QP_DEST_QPN) {
4031                 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
4032                                V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
4033                 roce_set_field(qpc_mask->byte_56_dqpn_err,
4034                                V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
4035         }
4036 }
4037
4038 static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
4039                             struct hns_roce_qp *hr_qp,
4040                             struct hns_roce_v2_qp_context *context,
4041                             struct hns_roce_v2_qp_context *qpc_mask)
4042 {
4043         u64 mtts[MTT_MIN_COUNT] = { 0 };
4044         u64 wqe_sge_ba;
4045         int count;
4046
4047         /* Search qp buf's mtts */
4048         count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
4049                                   MTT_MIN_COUNT, &wqe_sge_ba);
4050         if (hr_qp->rq.wqe_cnt && count < 1) {
4051                 ibdev_err(&hr_dev->ib_dev,
4052                           "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn);
4053                 return -EINVAL;
4054         }
4055
4056         context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
4057         qpc_mask->wqe_sge_ba = 0;
4058
4059         /*
4060          * In v2 engine, software pass context and context mask to hardware
4061          * when modifying qp. If software need modify some fields in context,
4062          * we should set all bits of the relevant fields in context mask to
4063          * 0 at the same time, else set them to 0x1.
4064          */
4065         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
4066                        V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
4067         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
4068                        V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
4069
4070         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
4071                        V2_QPC_BYTE_12_SQ_HOP_NUM_S,
4072                        to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,
4073                                         hr_qp->sq.wqe_cnt));
4074         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
4075                        V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
4076
4077         roce_set_field(context->byte_20_smac_sgid_idx,
4078                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
4079                        V2_QPC_BYTE_20_SGE_HOP_NUM_S,
4080                        to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,
4081                                         hr_qp->sge.sge_cnt));
4082         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4083                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
4084                        V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
4085
4086         roce_set_field(context->byte_20_smac_sgid_idx,
4087                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
4088                        V2_QPC_BYTE_20_RQ_HOP_NUM_S,
4089                        to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,
4090                                         hr_qp->rq.wqe_cnt));
4091
4092         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4093                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
4094                        V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
4095
4096         roce_set_field(context->byte_16_buf_ba_pg_sz,
4097                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
4098                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
4099                        to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));
4100         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
4101                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
4102                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
4103
4104         roce_set_field(context->byte_16_buf_ba_pg_sz,
4105                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
4106                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
4107                        to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));
4108         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
4109                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
4110                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
4111
4112         context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
4113         qpc_mask->rq_cur_blk_addr = 0;
4114
4115         roce_set_field(context->byte_92_srq_info,
4116                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
4117                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
4118                        upper_32_bits(to_hr_hw_page_addr(mtts[0])));
4119         roce_set_field(qpc_mask->byte_92_srq_info,
4120                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
4121                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
4122
4123         context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
4124         qpc_mask->rq_nxt_blk_addr = 0;
4125
4126         roce_set_field(context->byte_104_rq_sge,
4127                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
4128                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
4129                        upper_32_bits(to_hr_hw_page_addr(mtts[1])));
4130         roce_set_field(qpc_mask->byte_104_rq_sge,
4131                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
4132                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
4133
4134         roce_set_field(context->byte_84_rq_ci_pi,
4135                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4136                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
4137         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4138                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4139                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
4140
4141         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4142                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
4143                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
4144
4145         return 0;
4146 }
4147
4148 static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
4149                             struct hns_roce_qp *hr_qp,
4150                             struct hns_roce_v2_qp_context *context,
4151                             struct hns_roce_v2_qp_context *qpc_mask)
4152 {
4153         struct ib_device *ibdev = &hr_dev->ib_dev;
4154         u64 sge_cur_blk = 0;
4155         u64 sq_cur_blk = 0;
4156         int count;
4157
4158         /* search qp buf's mtts */
4159         count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
4160         if (count < 1) {
4161                 ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n",
4162                           hr_qp->qpn);
4163                 return -EINVAL;
4164         }
4165         if (hr_qp->sge.sge_cnt > 0) {
4166                 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
4167                                           hr_qp->sge.offset,
4168                                           &sge_cur_blk, 1, NULL);
4169                 if (count < 1) {
4170                         ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
4171                                   hr_qp->qpn);
4172                         return -EINVAL;
4173                 }
4174         }
4175
4176         /*
4177          * In v2 engine, software pass context and context mask to hardware
4178          * when modifying qp. If software need modify some fields in context,
4179          * we should set all bits of the relevant fields in context mask to
4180          * 0 at the same time, else set them to 0x1.
4181          */
4182         context->sq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
4183         roce_set_field(context->byte_168_irrl_idx,
4184                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
4185                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
4186                        upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4187         qpc_mask->sq_cur_blk_addr = 0;
4188         roce_set_field(qpc_mask->byte_168_irrl_idx,
4189                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
4190                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
4191
4192         context->sq_cur_sge_blk_addr =
4193                 cpu_to_le32(to_hr_hw_page_addr(sge_cur_blk));
4194         roce_set_field(context->byte_184_irrl_idx,
4195                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
4196                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
4197                        upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
4198         qpc_mask->sq_cur_sge_blk_addr = 0;
4199         roce_set_field(qpc_mask->byte_184_irrl_idx,
4200                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
4201                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
4202
4203         context->rx_sq_cur_blk_addr =
4204                 cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
4205         roce_set_field(context->byte_232_irrl_sge,
4206                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
4207                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
4208                        upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4209         qpc_mask->rx_sq_cur_blk_addr = 0;
4210         roce_set_field(qpc_mask->byte_232_irrl_sge,
4211                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
4212                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
4213
4214         return 0;
4215 }
4216
4217 static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,
4218                                   const struct ib_qp_attr *attr)
4219 {
4220         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
4221                 return IB_MTU_4096;
4222
4223         return attr->path_mtu;
4224 }
4225
4226 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
4227                                  const struct ib_qp_attr *attr, int attr_mask,
4228                                  struct hns_roce_v2_qp_context *context,
4229                                  struct hns_roce_v2_qp_context *qpc_mask)
4230 {
4231         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4232         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4233         struct ib_device *ibdev = &hr_dev->ib_dev;
4234         dma_addr_t trrl_ba;
4235         dma_addr_t irrl_ba;
4236         enum ib_mtu mtu;
4237         u8 lp_pktn_ini;
4238         u64 *mtts;
4239         u8 *dmac;
4240         u8 *smac;
4241         int port;
4242         int ret;
4243
4244         ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);
4245         if (ret) {
4246                 ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);
4247                 return ret;
4248         }
4249
4250         /* Search IRRL's mtts */
4251         mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
4252                                    hr_qp->qpn, &irrl_ba);
4253         if (!mtts) {
4254                 ibdev_err(ibdev, "failed to find qp irrl_table.\n");
4255                 return -EINVAL;
4256         }
4257
4258         /* Search TRRL's mtts */
4259         mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
4260                                    hr_qp->qpn, &trrl_ba);
4261         if (!mtts) {
4262                 ibdev_err(ibdev, "failed to find qp trrl_table.\n");
4263                 return -EINVAL;
4264         }
4265
4266         if (attr_mask & IB_QP_ALT_PATH) {
4267                 ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",
4268                           attr_mask);
4269                 return -EINVAL;
4270         }
4271
4272         roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
4273                        V2_QPC_BYTE_132_TRRL_BA_S, trrl_ba >> 4);
4274         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
4275                        V2_QPC_BYTE_132_TRRL_BA_S, 0);
4276         context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4));
4277         qpc_mask->trrl_ba = 0;
4278         roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
4279                        V2_QPC_BYTE_140_TRRL_BA_S,
4280                        (u32)(trrl_ba >> (32 + 16 + 4)));
4281         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
4282                        V2_QPC_BYTE_140_TRRL_BA_S, 0);
4283
4284         context->irrl_ba = cpu_to_le32(irrl_ba >> 6);
4285         qpc_mask->irrl_ba = 0;
4286         roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
4287                        V2_QPC_BYTE_208_IRRL_BA_S,
4288                        irrl_ba >> (32 + 6));
4289         roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
4290                        V2_QPC_BYTE_208_IRRL_BA_S, 0);
4291
4292         roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
4293         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
4294
4295         roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
4296                      hr_qp->sq_signal_bits);
4297         roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
4298                      0);
4299
4300         port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
4301
4302         smac = (u8 *)hr_dev->dev_addr[port];
4303         dmac = (u8 *)attr->ah_attr.roce.dmac;
4304         /* when dmac equals smac or loop_idc is 1, it should loopback */
4305         if (ether_addr_equal_unaligned(dmac, smac) ||
4306             hr_dev->loop_idc == 0x1) {
4307                 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
4308                 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
4309         }
4310
4311         if (attr_mask & IB_QP_DEST_QPN) {
4312                 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
4313                                V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
4314                 roce_set_field(qpc_mask->byte_56_dqpn_err,
4315                                V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
4316         }
4317
4318         memcpy(&(context->dmac), dmac, sizeof(u32));
4319         roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
4320                        V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
4321         qpc_mask->dmac = 0;
4322         roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
4323                        V2_QPC_BYTE_52_DMAC_S, 0);
4324
4325         mtu = get_mtu(ibqp, attr);
4326         hr_qp->path_mtu = mtu;
4327
4328         if (attr_mask & IB_QP_PATH_MTU) {
4329                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
4330                                V2_QPC_BYTE_24_MTU_S, mtu);
4331                 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
4332                                V2_QPC_BYTE_24_MTU_S, 0);
4333         }
4334
4335 #define MAX_LP_MSG_LEN 65536
4336         /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 64KB */
4337         lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / ib_mtu_enum_to_int(mtu));
4338
4339         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
4340                        V2_QPC_BYTE_56_LP_PKTN_INI_S, lp_pktn_ini);
4341         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
4342                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
4343
4344         /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */
4345         roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
4346                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, lp_pktn_ini);
4347         roce_set_field(qpc_mask->byte_172_sq_psn,
4348                        V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
4349                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
4350
4351         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4352                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
4353         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
4354                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
4355         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4356                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
4357                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
4358
4359         context->rq_rnr_timer = 0;
4360         qpc_mask->rq_rnr_timer = 0;
4361
4362         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
4363                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
4364         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
4365                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
4366
4367         /* rocee send 2^lp_sgen_ini segs every time */
4368         roce_set_field(context->byte_168_irrl_idx,
4369                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
4370                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
4371         roce_set_field(qpc_mask->byte_168_irrl_idx,
4372                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
4373                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
4374
4375         return 0;
4376 }
4377
4378 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
4379                                 const struct ib_qp_attr *attr, int attr_mask,
4380                                 struct hns_roce_v2_qp_context *context,
4381                                 struct hns_roce_v2_qp_context *qpc_mask)
4382 {
4383         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4384         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4385         struct ib_device *ibdev = &hr_dev->ib_dev;
4386         int ret;
4387
4388         /* Not support alternate path and path migration */
4389         if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) {
4390                 ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
4391                 return -EINVAL;
4392         }
4393
4394         ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
4395         if (ret) {
4396                 ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);
4397                 return ret;
4398         }
4399
4400         /*
4401          * Set some fields in context to zero, Because the default values
4402          * of all fields in context are zero, we need not set them to 0 again.
4403          * but we should set the relevant fields of context mask to 0.
4404          */
4405         roce_set_field(qpc_mask->byte_232_irrl_sge,
4406                        V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
4407                        V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
4408
4409         roce_set_field(qpc_mask->byte_240_irrl_tail,
4410                        V2_QPC_BYTE_240_RX_ACK_MSN_M,
4411                        V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
4412
4413         roce_set_field(qpc_mask->byte_248_ack_psn,
4414                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
4415                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
4416         roce_set_bit(qpc_mask->byte_248_ack_psn,
4417                      V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);
4418         roce_set_field(qpc_mask->byte_248_ack_psn,
4419                        V2_QPC_BYTE_248_IRRL_PSN_M,
4420                        V2_QPC_BYTE_248_IRRL_PSN_S, 0);
4421
4422         roce_set_field(qpc_mask->byte_240_irrl_tail,
4423                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
4424                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
4425
4426         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4427                        V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
4428                        V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
4429
4430         roce_set_bit(qpc_mask->byte_248_ack_psn,
4431                      V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
4432
4433         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
4434                        V2_QPC_BYTE_212_CHECK_FLG_S, 0);
4435
4436         roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4437                        V2_QPC_BYTE_212_LSN_S, 0x100);
4438         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4439                        V2_QPC_BYTE_212_LSN_S, 0);
4440
4441         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
4442                        V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
4443
4444         return 0;
4445 }
4446
4447 static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
4448 {
4449         if (!fl)
4450                 fl = rdma_calc_flow_label(lqpn, rqpn);
4451
4452         return rdma_flow_label_to_udp_sport(fl);
4453 }
4454
4455 static int hns_roce_v2_set_path(struct ib_qp *ibqp,
4456                                 const struct ib_qp_attr *attr,
4457                                 int attr_mask,
4458                                 struct hns_roce_v2_qp_context *context,
4459                                 struct hns_roce_v2_qp_context *qpc_mask)
4460 {
4461         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4462         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4463         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4464         struct ib_device *ibdev = &hr_dev->ib_dev;
4465         const struct ib_gid_attr *gid_attr = NULL;
4466         int is_roce_protocol;
4467         u16 vlan_id = 0xffff;
4468         bool is_udp = false;
4469         u8 ib_port;
4470         u8 hr_port;
4471         int ret;
4472
4473         ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
4474         hr_port = ib_port - 1;
4475         is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4476                            rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4477
4478         if (is_roce_protocol) {
4479                 gid_attr = attr->ah_attr.grh.sgid_attr;
4480                 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
4481                 if (ret)
4482                         return ret;
4483
4484                 if (gid_attr)
4485                         is_udp = (gid_attr->gid_type ==
4486                                  IB_GID_TYPE_ROCE_UDP_ENCAP);
4487         }
4488
4489         /* Only HIP08 needs to set the vlan_en bits in QPC */
4490         if (vlan_id < VLAN_N_VID &&
4491             hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
4492                 roce_set_bit(context->byte_76_srqn_op_en,
4493                              V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
4494                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
4495                              V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
4496                 roce_set_bit(context->byte_168_irrl_idx,
4497                              V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
4498                 roce_set_bit(qpc_mask->byte_168_irrl_idx,
4499                              V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
4500         }
4501
4502         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
4503                        V2_QPC_BYTE_24_VLAN_ID_S, vlan_id);
4504         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
4505                        V2_QPC_BYTE_24_VLAN_ID_S, 0);
4506
4507         if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
4508                 ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
4509                           grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
4510                 return -EINVAL;
4511         }
4512
4513         if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
4514                 ibdev_err(ibdev, "ah attr is not RDMA roce type\n");
4515                 return -EINVAL;
4516         }
4517
4518         roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4519                        V2_QPC_BYTE_52_UDPSPN_S,
4520                        is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num,
4521                                               attr->dest_qp_num) : 0);
4522
4523         roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4524                        V2_QPC_BYTE_52_UDPSPN_S, 0);
4525
4526         roce_set_field(context->byte_20_smac_sgid_idx,
4527                        V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
4528                        grh->sgid_index);
4529
4530         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4531                        V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
4532
4533         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4534                        V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
4535         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4536                        V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
4537
4538         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4539                        V2_QPC_BYTE_24_TC_S, get_tclass(&attr->ah_attr.grh));
4540         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4541                        V2_QPC_BYTE_24_TC_S, 0);
4542
4543         roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4544                        V2_QPC_BYTE_28_FL_S, grh->flow_label);
4545         roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4546                        V2_QPC_BYTE_28_FL_S, 0);
4547         memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4548         memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
4549
4550         hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4551         if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) {
4552                 ibdev_err(ibdev,
4553                           "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n",
4554                           hr_qp->sl, MAX_SERVICE_LEVEL);
4555                 return -EINVAL;
4556         }
4557
4558         roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4559                        V2_QPC_BYTE_28_SL_S, hr_qp->sl);
4560         roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4561                        V2_QPC_BYTE_28_SL_S, 0);
4562
4563         return 0;
4564 }
4565
4566 static bool check_qp_state(enum ib_qp_state cur_state,
4567                            enum ib_qp_state new_state)
4568 {
4569         static const bool sm[][IB_QPS_ERR + 1] = {
4570                 [IB_QPS_RESET] = { [IB_QPS_RESET] = true,
4571                                    [IB_QPS_INIT] = true },
4572                 [IB_QPS_INIT] = { [IB_QPS_RESET] = true,
4573                                   [IB_QPS_INIT] = true,
4574                                   [IB_QPS_RTR] = true,
4575                                   [IB_QPS_ERR] = true },
4576                 [IB_QPS_RTR] = { [IB_QPS_RESET] = true,
4577                                  [IB_QPS_RTS] = true,
4578                                  [IB_QPS_ERR] = true },
4579                 [IB_QPS_RTS] = { [IB_QPS_RESET] = true,
4580                                  [IB_QPS_RTS] = true,
4581                                  [IB_QPS_ERR] = true },
4582                 [IB_QPS_SQD] = {},
4583                 [IB_QPS_SQE] = {},
4584                 [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
4585         };
4586
4587         return sm[cur_state][new_state];
4588 }
4589
4590 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
4591                                       const struct ib_qp_attr *attr,
4592                                       int attr_mask,
4593                                       enum ib_qp_state cur_state,
4594                                       enum ib_qp_state new_state,
4595                                       struct hns_roce_v2_qp_context *context,
4596                                       struct hns_roce_v2_qp_context *qpc_mask)
4597 {
4598         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4599         int ret = 0;
4600
4601         if (!check_qp_state(cur_state, new_state)) {
4602                 ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");
4603                 return -EINVAL;
4604         }
4605
4606         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4607                 memset(qpc_mask, 0, hr_dev->caps.qpc_sz);
4608                 modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
4609                                         qpc_mask);
4610         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4611                 modify_qp_init_to_init(ibqp, attr, attr_mask, context,
4612                                        qpc_mask);
4613         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4614                 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
4615                                             qpc_mask);
4616         } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4617                 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
4618                                            qpc_mask);
4619         }
4620
4621         return ret;
4622 }
4623
4624 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
4625                                       const struct ib_qp_attr *attr,
4626                                       int attr_mask,
4627                                       struct hns_roce_v2_qp_context *context,
4628                                       struct hns_roce_v2_qp_context *qpc_mask)
4629 {
4630         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4631         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4632         int ret = 0;
4633
4634         if (attr_mask & IB_QP_AV) {
4635                 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
4636                                            qpc_mask);
4637                 if (ret)
4638                         return ret;
4639         }
4640
4641         if (attr_mask & IB_QP_TIMEOUT) {
4642                 if (attr->timeout < 31) {
4643                         roce_set_field(context->byte_28_at_fl,
4644                                        V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4645                                        attr->timeout);
4646                         roce_set_field(qpc_mask->byte_28_at_fl,
4647                                        V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4648                                        0);
4649                 } else {
4650                         ibdev_warn(&hr_dev->ib_dev,
4651                                    "Local ACK timeout shall be 0 to 30.\n");
4652                 }
4653         }
4654
4655         if (attr_mask & IB_QP_RETRY_CNT) {
4656                 roce_set_field(context->byte_212_lsn,
4657                                V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4658                                V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
4659                                attr->retry_cnt);
4660                 roce_set_field(qpc_mask->byte_212_lsn,
4661                                V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4662                                V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
4663
4664                 roce_set_field(context->byte_212_lsn,
4665                                V2_QPC_BYTE_212_RETRY_CNT_M,
4666                                V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
4667                 roce_set_field(qpc_mask->byte_212_lsn,
4668                                V2_QPC_BYTE_212_RETRY_CNT_M,
4669                                V2_QPC_BYTE_212_RETRY_CNT_S, 0);
4670         }
4671
4672         if (attr_mask & IB_QP_RNR_RETRY) {
4673                 roce_set_field(context->byte_244_rnr_rxack,
4674                                V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4675                                V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
4676                 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4677                                V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4678                                V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
4679
4680                 roce_set_field(context->byte_244_rnr_rxack,
4681                                V2_QPC_BYTE_244_RNR_CNT_M,
4682                                V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
4683                 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4684                                V2_QPC_BYTE_244_RNR_CNT_M,
4685                                V2_QPC_BYTE_244_RNR_CNT_S, 0);
4686         }
4687
4688         /* RC&UC&UD required attr */
4689         if (attr_mask & IB_QP_SQ_PSN) {
4690                 roce_set_field(context->byte_172_sq_psn,
4691                                V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4692                                V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
4693                 roce_set_field(qpc_mask->byte_172_sq_psn,
4694                                V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4695                                V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
4696
4697                 roce_set_field(context->byte_196_sq_psn,
4698                                V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4699                                V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
4700                 roce_set_field(qpc_mask->byte_196_sq_psn,
4701                                V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4702                                V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
4703
4704                 roce_set_field(context->byte_220_retry_psn_msn,
4705                                V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4706                                V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
4707                 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4708                                V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4709                                V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
4710
4711                 roce_set_field(context->byte_224_retry_msg,
4712                                V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4713                                V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
4714                                attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
4715                 roce_set_field(qpc_mask->byte_224_retry_msg,
4716                                V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4717                                V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
4718
4719                 roce_set_field(context->byte_224_retry_msg,
4720                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4721                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
4722                                attr->sq_psn);
4723                 roce_set_field(qpc_mask->byte_224_retry_msg,
4724                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4725                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
4726
4727                 roce_set_field(context->byte_244_rnr_rxack,
4728                                V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4729                                V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
4730                 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4731                                V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4732                                V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
4733         }
4734
4735         if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
4736              attr->max_dest_rd_atomic) {
4737                 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4738                                V2_QPC_BYTE_140_RR_MAX_S,
4739                                fls(attr->max_dest_rd_atomic - 1));
4740                 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4741                                V2_QPC_BYTE_140_RR_MAX_S, 0);
4742         }
4743
4744         if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
4745                 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
4746                                V2_QPC_BYTE_208_SR_MAX_S,
4747                                fls(attr->max_rd_atomic - 1));
4748                 roce_set_field(qpc_mask->byte_208_irrl,
4749                                V2_QPC_BYTE_208_SR_MAX_M,
4750                                V2_QPC_BYTE_208_SR_MAX_S, 0);
4751         }
4752
4753         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
4754                 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
4755
4756         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
4757                 roce_set_field(context->byte_80_rnr_rx_cqn,
4758                                V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4759                                V2_QPC_BYTE_80_MIN_RNR_TIME_S,
4760                                attr->min_rnr_timer);
4761                 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
4762                                V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4763                                V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
4764         }
4765
4766         /* RC&UC required attr */
4767         if (attr_mask & IB_QP_RQ_PSN) {
4768                 roce_set_field(context->byte_108_rx_reqepsn,
4769                                V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4770                                V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
4771                 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4772                                V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4773                                V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
4774
4775                 roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
4776                                V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
4777                 roce_set_field(qpc_mask->byte_152_raq,
4778                                V2_QPC_BYTE_152_RAQ_PSN_M,
4779                                V2_QPC_BYTE_152_RAQ_PSN_S, 0);
4780         }
4781
4782         if (attr_mask & IB_QP_QKEY) {
4783                 context->qkey_xrcd = cpu_to_le32(attr->qkey);
4784                 qpc_mask->qkey_xrcd = 0;
4785                 hr_qp->qkey = attr->qkey;
4786         }
4787
4788         return ret;
4789 }
4790
4791 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
4792                                           const struct ib_qp_attr *attr,
4793                                           int attr_mask)
4794 {
4795         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4796         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4797
4798         if (attr_mask & IB_QP_ACCESS_FLAGS)
4799                 hr_qp->atomic_rd_en = attr->qp_access_flags;
4800
4801         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4802                 hr_qp->resp_depth = attr->max_dest_rd_atomic;
4803         if (attr_mask & IB_QP_PORT) {
4804                 hr_qp->port = attr->port_num - 1;
4805                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
4806         }
4807 }
4808
4809 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
4810                                  const struct ib_qp_attr *attr,
4811                                  int attr_mask, enum ib_qp_state cur_state,
4812                                  enum ib_qp_state new_state)
4813 {
4814         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4815         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4816         struct hns_roce_v2_qp_context ctx[2];
4817         struct hns_roce_v2_qp_context *context = ctx;
4818         struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
4819         struct ib_device *ibdev = &hr_dev->ib_dev;
4820         unsigned long sq_flag = 0;
4821         unsigned long rq_flag = 0;
4822         int ret;
4823
4824         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
4825                 return -EOPNOTSUPP;
4826
4827         /*
4828          * In v2 engine, software pass context and context mask to hardware
4829          * when modifying qp. If software need modify some fields in context,
4830          * we should set all bits of the relevant fields in context mask to
4831          * 0 at the same time, else set them to 0x1.
4832          */
4833         memset(context, 0, hr_dev->caps.qpc_sz);
4834         memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
4835
4836         ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
4837                                          new_state, context, qpc_mask);
4838         if (ret)
4839                 goto out;
4840
4841         /* When QP state is err, SQ and RQ WQE should be flushed */
4842         if (new_state == IB_QPS_ERR) {
4843                 spin_lock_irqsave(&hr_qp->sq.lock, sq_flag);
4844                 hr_qp->state = IB_QPS_ERR;
4845                 roce_set_field(context->byte_160_sq_ci_pi,
4846                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4847                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
4848                                hr_qp->sq.head);
4849                 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
4850                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4851                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
4852                 spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
4853
4854                 if (!ibqp->srq) {
4855                         spin_lock_irqsave(&hr_qp->rq.lock, rq_flag);
4856                         roce_set_field(context->byte_84_rq_ci_pi,
4857                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4858                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
4859                                hr_qp->rq.head);
4860                         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4861                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4862                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
4863                         spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag);
4864                 }
4865         }
4866
4867         /* Configure the optional fields */
4868         ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
4869                                          qpc_mask);
4870         if (ret)
4871                 goto out;
4872
4873         roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
4874                      ibqp->srq ? 1 : 0);
4875         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4876                      V2_QPC_BYTE_108_INV_CREDIT_S, 0);
4877
4878         /* Every status migrate must change state */
4879         roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4880                        V2_QPC_BYTE_60_QP_ST_S, new_state);
4881         roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4882                        V2_QPC_BYTE_60_QP_ST_S, 0);
4883
4884         /* SW pass context to HW */
4885         ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);
4886         if (ret) {
4887                 ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret);
4888                 goto out;
4889         }
4890
4891         hr_qp->state = new_state;
4892
4893         hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
4894
4895         if (new_state == IB_QPS_RESET && !ibqp->uobject) {
4896                 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
4897                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
4898                 if (ibqp->send_cq != ibqp->recv_cq)
4899                         hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
4900                                              hr_qp->qpn, NULL);
4901
4902                 hr_qp->rq.head = 0;
4903                 hr_qp->rq.tail = 0;
4904                 hr_qp->sq.head = 0;
4905                 hr_qp->sq.tail = 0;
4906                 hr_qp->next_sge = 0;
4907                 if (hr_qp->rq.wqe_cnt)
4908                         *hr_qp->rdb.db_record = 0;
4909         }
4910
4911 out:
4912         return ret;
4913 }
4914
4915 static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
4916 {
4917         static const enum ib_qp_state map[] = {
4918                 [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,
4919                 [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,
4920                 [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,
4921                 [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,
4922                 [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,
4923                 [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,
4924                 [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,
4925                 [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD
4926         };
4927
4928         return (state < ARRAY_SIZE(map)) ? map[state] : -1;
4929 }
4930
4931 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
4932                                  struct hns_roce_qp *hr_qp,
4933                                  struct hns_roce_v2_qp_context *hr_context)
4934 {
4935         struct hns_roce_cmd_mailbox *mailbox;
4936         int ret;
4937
4938         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4939         if (IS_ERR(mailbox))
4940                 return PTR_ERR(mailbox);
4941
4942         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
4943                                 HNS_ROCE_CMD_QUERY_QPC,
4944                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
4945         if (ret)
4946                 goto out;
4947
4948         memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz);
4949
4950 out:
4951         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4952         return ret;
4953 }
4954
4955 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4956                                 int qp_attr_mask,
4957                                 struct ib_qp_init_attr *qp_init_attr)
4958 {
4959         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4960         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4961         struct hns_roce_v2_qp_context context = {};
4962         struct ib_device *ibdev = &hr_dev->ib_dev;
4963         int tmp_qp_state;
4964         int state;
4965         int ret;
4966
4967         memset(qp_attr, 0, sizeof(*qp_attr));
4968         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
4969
4970         mutex_lock(&hr_qp->mutex);
4971
4972         if (hr_qp->state == IB_QPS_RESET) {
4973                 qp_attr->qp_state = IB_QPS_RESET;
4974                 ret = 0;
4975                 goto done;
4976         }
4977
4978         ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
4979         if (ret) {
4980                 ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
4981                 ret = -EINVAL;
4982                 goto out;
4983         }
4984
4985         state = roce_get_field(context.byte_60_qpst_tempid,
4986                                V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
4987         tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
4988         if (tmp_qp_state == -1) {
4989                 ibdev_err(ibdev, "Illegal ib_qp_state\n");
4990                 ret = -EINVAL;
4991                 goto out;
4992         }
4993         hr_qp->state = (u8)tmp_qp_state;
4994         qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
4995         qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc,
4996                                                         V2_QPC_BYTE_24_MTU_M,
4997                                                         V2_QPC_BYTE_24_MTU_S);
4998         qp_attr->path_mig_state = IB_MIG_ARMED;
4999         qp_attr->ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;
5000         if (hr_qp->ibqp.qp_type == IB_QPT_UD)
5001                 qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);
5002
5003         qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn,
5004                                          V2_QPC_BYTE_108_RX_REQ_EPSN_M,
5005                                          V2_QPC_BYTE_108_RX_REQ_EPSN_S);
5006         qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn,
5007                                               V2_QPC_BYTE_172_SQ_CUR_PSN_M,
5008                                               V2_QPC_BYTE_172_SQ_CUR_PSN_S);
5009         qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err,
5010                                                   V2_QPC_BYTE_56_DQPN_M,
5011                                                   V2_QPC_BYTE_56_DQPN_S);
5012         qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en,
5013                                     V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) |
5014                                     ((roce_get_bit(context.byte_76_srqn_op_en,
5015                                     V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) |
5016                                     ((roce_get_bit(context.byte_76_srqn_op_en,
5017                                     V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
5018
5019         if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
5020             hr_qp->ibqp.qp_type == IB_QPT_UC) {
5021                 struct ib_global_route *grh =
5022                                 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
5023
5024                 rdma_ah_set_sl(&qp_attr->ah_attr,
5025                                roce_get_field(context.byte_28_at_fl,
5026                                               V2_QPC_BYTE_28_SL_M,
5027                                               V2_QPC_BYTE_28_SL_S));
5028                 grh->flow_label = roce_get_field(context.byte_28_at_fl,
5029                                                  V2_QPC_BYTE_28_FL_M,
5030                                                  V2_QPC_BYTE_28_FL_S);
5031                 grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx,
5032                                                  V2_QPC_BYTE_20_SGID_IDX_M,
5033                                                  V2_QPC_BYTE_20_SGID_IDX_S);
5034                 grh->hop_limit = roce_get_field(context.byte_24_mtu_tc,
5035                                                 V2_QPC_BYTE_24_HOP_LIMIT_M,
5036                                                 V2_QPC_BYTE_24_HOP_LIMIT_S);
5037                 grh->traffic_class = roce_get_field(context.byte_24_mtu_tc,
5038                                                     V2_QPC_BYTE_24_TC_M,
5039                                                     V2_QPC_BYTE_24_TC_S);
5040
5041                 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
5042         }
5043
5044         qp_attr->port_num = hr_qp->port + 1;
5045         qp_attr->sq_draining = 0;
5046         qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl,
5047                                                      V2_QPC_BYTE_208_SR_MAX_M,
5048                                                      V2_QPC_BYTE_208_SR_MAX_S);
5049         qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq,
5050                                                      V2_QPC_BYTE_140_RR_MAX_M,
5051                                                      V2_QPC_BYTE_140_RR_MAX_S);
5052         qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn,
5053                                                  V2_QPC_BYTE_80_MIN_RNR_TIME_M,
5054                                                  V2_QPC_BYTE_80_MIN_RNR_TIME_S);
5055         qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl,
5056                                               V2_QPC_BYTE_28_AT_M,
5057                                               V2_QPC_BYTE_28_AT_S);
5058         qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn,
5059                                             V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
5060                                             V2_QPC_BYTE_212_RETRY_NUM_INIT_S);
5061         qp_attr->rnr_retry = roce_get_field(context.byte_244_rnr_rxack,
5062                                             V2_QPC_BYTE_244_RNR_NUM_INIT_M,
5063                                             V2_QPC_BYTE_244_RNR_NUM_INIT_S);
5064
5065 done:
5066         qp_attr->cur_qp_state = qp_attr->qp_state;
5067         qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
5068         qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
5069
5070         if (!ibqp->uobject) {
5071                 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
5072                 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
5073         } else {
5074                 qp_attr->cap.max_send_wr = 0;
5075                 qp_attr->cap.max_send_sge = 0;
5076         }
5077
5078         qp_init_attr->cap = qp_attr->cap;
5079         qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits;
5080
5081 out:
5082         mutex_unlock(&hr_qp->mutex);
5083         return ret;
5084 }
5085
5086 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
5087                                          struct hns_roce_qp *hr_qp,
5088                                          struct ib_udata *udata)
5089 {
5090         struct ib_device *ibdev = &hr_dev->ib_dev;
5091         struct hns_roce_cq *send_cq, *recv_cq;
5092         unsigned long flags;
5093         int ret = 0;
5094
5095         if ((hr_qp->ibqp.qp_type == IB_QPT_RC ||
5096              hr_qp->ibqp.qp_type == IB_QPT_UD) &&
5097            hr_qp->state != IB_QPS_RESET) {
5098                 /* Modify qp to reset before destroying qp */
5099                 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
5100                                             hr_qp->state, IB_QPS_RESET);
5101                 if (ret)
5102                         ibdev_err(ibdev,
5103                                   "failed to modify QP to RST, ret = %d.\n",
5104                                   ret);
5105         }
5106
5107         send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
5108         recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
5109
5110         spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
5111         hns_roce_lock_cqs(send_cq, recv_cq);
5112
5113         if (!udata) {
5114                 if (recv_cq)
5115                         __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,
5116                                                (hr_qp->ibqp.srq ?
5117                                                 to_hr_srq(hr_qp->ibqp.srq) :
5118                                                 NULL));
5119
5120                 if (send_cq && send_cq != recv_cq)
5121                         __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
5122
5123         }
5124
5125         hns_roce_qp_remove(hr_dev, hr_qp);
5126
5127         hns_roce_unlock_cqs(send_cq, recv_cq);
5128         spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
5129
5130         return ret;
5131 }
5132
5133 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
5134 {
5135         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5136         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5137         int ret;
5138
5139         ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
5140         if (ret)
5141                 ibdev_err(&hr_dev->ib_dev,
5142                           "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",
5143                           hr_qp->qpn, ret);
5144
5145         hns_roce_qp_destroy(hr_dev, hr_qp, udata);
5146
5147         return 0;
5148 }
5149
5150 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
5151                                             struct hns_roce_qp *hr_qp)
5152 {
5153         struct ib_device *ibdev = &hr_dev->ib_dev;
5154         struct hns_roce_sccc_clr_done *resp;
5155         struct hns_roce_sccc_clr *clr;
5156         struct hns_roce_cmq_desc desc;
5157         int ret, i;
5158
5159         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
5160                 return 0;
5161
5162         mutex_lock(&hr_dev->qp_table.scc_mutex);
5163
5164         /* set scc ctx clear done flag */
5165         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
5166         ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
5167         if (ret) {
5168                 ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);
5169                 goto out;
5170         }
5171
5172         /* clear scc context */
5173         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
5174         clr = (struct hns_roce_sccc_clr *)desc.data;
5175         clr->qpn = cpu_to_le32(hr_qp->qpn);
5176         ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
5177         if (ret) {
5178                 ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);
5179                 goto out;
5180         }
5181
5182         /* query scc context clear is done or not */
5183         resp = (struct hns_roce_sccc_clr_done *)desc.data;
5184         for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
5185                 hns_roce_cmq_setup_basic_desc(&desc,
5186                                               HNS_ROCE_OPC_QUERY_SCCC, true);
5187                 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5188                 if (ret) {
5189                         ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",
5190                                   ret);
5191                         goto out;
5192                 }
5193
5194                 if (resp->clr_done)
5195                         goto out;
5196
5197                 msleep(20);
5198         }
5199
5200         ibdev_err(ibdev, "Query SCC clr done flag overtime.\n");
5201         ret = -ETIMEDOUT;
5202
5203 out:
5204         mutex_unlock(&hr_dev->qp_table.scc_mutex);
5205         return ret;
5206 }
5207
5208 #define DMA_IDX_SHIFT 3
5209 #define DMA_WQE_SHIFT 3
5210
5211 static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq,
5212                                               struct hns_roce_srq_context *ctx)
5213 {
5214         struct hns_roce_idx_que *idx_que = &srq->idx_que;
5215         struct ib_device *ibdev = srq->ibsrq.device;
5216         struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5217         u64 mtts_idx[MTT_MIN_COUNT] = {};
5218         dma_addr_t dma_handle_idx = 0;
5219         int ret;
5220
5221         /* Get physical address of idx que buf */
5222         ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx,
5223                                 ARRAY_SIZE(mtts_idx), &dma_handle_idx);
5224         if (ret < 1) {
5225                 ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",
5226                           ret);
5227                 return -ENOBUFS;
5228         }
5229
5230         hr_reg_write(ctx, SRQC_IDX_HOP_NUM,
5231                      to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt));
5232
5233         hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT);
5234         hr_reg_write(ctx, SRQC_IDX_BT_BA_H,
5235                      upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT));
5236
5237         hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ,
5238                      to_hr_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift));
5239         hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ,
5240                      to_hr_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift));
5241
5242         hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L,
5243                      to_hr_hw_page_addr(mtts_idx[0]));
5244         hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H,
5245                      upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));
5246
5247         hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L,
5248                      to_hr_hw_page_addr(mtts_idx[1]));
5249         hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H,
5250                      upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));
5251
5252         return 0;
5253 }
5254
5255 static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf)
5256 {
5257         struct ib_device *ibdev = srq->ibsrq.device;
5258         struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5259         struct hns_roce_srq_context *ctx = mb_buf;
5260         u64 mtts_wqe[MTT_MIN_COUNT] = {};
5261         dma_addr_t dma_handle_wqe = 0;
5262         int ret;
5263
5264         memset(ctx, 0, sizeof(*ctx));
5265
5266         /* Get the physical address of srq buf */
5267         ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,
5268                                 ARRAY_SIZE(mtts_wqe), &dma_handle_wqe);
5269         if (ret < 1) {
5270                 ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",
5271                           ret);
5272                 return -ENOBUFS;
5273         }
5274
5275         hr_reg_write(ctx, SRQC_SRQ_ST, 1);
5276         hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn);
5277         hr_reg_write(ctx, SRQC_SRQN, srq->srqn);
5278         hr_reg_write(ctx, SRQC_XRCD, 0);
5279         hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn);
5280         hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt));
5281         hr_reg_write(ctx, SRQC_RQWS,
5282                      srq->max_gs <= 0 ? 0 : fls(srq->max_gs - 1));
5283
5284         hr_reg_write(ctx, SRQC_WQE_HOP_NUM,
5285                      to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,
5286                                       srq->wqe_cnt));
5287
5288         hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT);
5289         hr_reg_write(ctx, SRQC_WQE_BT_BA_H,
5290                      upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT));
5291
5292         hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ,
5293                      to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));
5294         hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ,
5295                      to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));
5296
5297         return hns_roce_v2_write_srqc_index_queue(srq, ctx);
5298 }
5299
5300 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
5301                                   struct ib_srq_attr *srq_attr,
5302                                   enum ib_srq_attr_mask srq_attr_mask,
5303                                   struct ib_udata *udata)
5304 {
5305         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5306         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5307         struct hns_roce_srq_context *srq_context;
5308         struct hns_roce_srq_context *srqc_mask;
5309         struct hns_roce_cmd_mailbox *mailbox;
5310         int ret;
5311
5312         /* Resizing SRQs is not supported yet */
5313         if (srq_attr_mask & IB_SRQ_MAX_WR)
5314                 return -EINVAL;
5315
5316         if (srq_attr_mask & IB_SRQ_LIMIT) {
5317                 if (srq_attr->srq_limit > srq->wqe_cnt)
5318                         return -EINVAL;
5319
5320                 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5321                 if (IS_ERR(mailbox))
5322                         return PTR_ERR(mailbox);
5323
5324                 srq_context = mailbox->buf;
5325                 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
5326
5327                 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
5328
5329                 roce_set_field(srq_context->byte_8_limit_wl,
5330                                SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5331                                SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
5332                 roce_set_field(srqc_mask->byte_8_limit_wl,
5333                                SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5334                                SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
5335
5336                 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
5337                                         HNS_ROCE_CMD_MODIFY_SRQC,
5338                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5339                 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5340                 if (ret) {
5341                         ibdev_err(&hr_dev->ib_dev,
5342                                   "failed to handle cmd of modifying SRQ, ret = %d.\n",
5343                                   ret);
5344                         return ret;
5345                 }
5346         }
5347
5348         return 0;
5349 }
5350
5351 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
5352 {
5353         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5354         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5355         struct hns_roce_srq_context *srq_context;
5356         struct hns_roce_cmd_mailbox *mailbox;
5357         int limit_wl;
5358         int ret;
5359
5360         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5361         if (IS_ERR(mailbox))
5362                 return PTR_ERR(mailbox);
5363
5364         srq_context = mailbox->buf;
5365         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
5366                                 HNS_ROCE_CMD_QUERY_SRQC,
5367                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
5368         if (ret) {
5369                 ibdev_err(&hr_dev->ib_dev,
5370                           "failed to process cmd of querying SRQ, ret = %d.\n",
5371                           ret);
5372                 goto out;
5373         }
5374
5375         limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
5376                                   SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5377                                   SRQC_BYTE_8_SRQ_LIMIT_WL_S);
5378
5379         attr->srq_limit = limit_wl;
5380         attr->max_wr = srq->wqe_cnt;
5381         attr->max_sge = srq->max_gs - srq->rsv_sge;
5382
5383 out:
5384         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5385         return ret;
5386 }
5387
5388 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
5389 {
5390         struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
5391         struct hns_roce_v2_cq_context *cq_context;
5392         struct hns_roce_cq *hr_cq = to_hr_cq(cq);
5393         struct hns_roce_v2_cq_context *cqc_mask;
5394         struct hns_roce_cmd_mailbox *mailbox;
5395         int ret;
5396
5397         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5398         if (IS_ERR(mailbox))
5399                 return PTR_ERR(mailbox);
5400
5401         cq_context = mailbox->buf;
5402         cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
5403
5404         memset(cqc_mask, 0xff, sizeof(*cqc_mask));
5405
5406         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
5407                        V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
5408                        cq_count);
5409         roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
5410                        V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
5411                        0);
5412         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
5413                        V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
5414                        cq_period);
5415         roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
5416                        V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
5417                        0);
5418
5419         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
5420                                 HNS_ROCE_CMD_MODIFY_CQC,
5421                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
5422         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5423         if (ret)
5424                 ibdev_err(&hr_dev->ib_dev,
5425                           "failed to process cmd when modifying CQ, ret = %d.\n",
5426                           ret);
5427
5428         return ret;
5429 }
5430
5431 static void hns_roce_irq_work_handle(struct work_struct *work)
5432 {
5433         struct hns_roce_work *irq_work =
5434                                 container_of(work, struct hns_roce_work, work);
5435         struct ib_device *ibdev = &irq_work->hr_dev->ib_dev;
5436
5437         switch (irq_work->event_type) {
5438         case HNS_ROCE_EVENT_TYPE_PATH_MIG:
5439                 ibdev_info(ibdev, "Path migrated succeeded.\n");
5440                 break;
5441         case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
5442                 ibdev_warn(ibdev, "Path migration failed.\n");
5443                 break;
5444         case HNS_ROCE_EVENT_TYPE_COMM_EST:
5445                 break;
5446         case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5447                 ibdev_warn(ibdev, "Send queue drained.\n");
5448                 break;
5449         case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
5450                 ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n",
5451                           irq_work->queue_num, irq_work->sub_type);
5452                 break;
5453         case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
5454                 ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n",
5455                           irq_work->queue_num);
5456                 break;
5457         case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
5458                 ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n",
5459                           irq_work->queue_num, irq_work->sub_type);
5460                 break;
5461         case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
5462                 ibdev_warn(ibdev, "SRQ limit reach.\n");
5463                 break;
5464         case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
5465                 ibdev_warn(ibdev, "SRQ last wqe reach.\n");
5466                 break;
5467         case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5468                 ibdev_err(ibdev, "SRQ catas error.\n");
5469                 break;
5470         case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5471                 ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num);
5472                 break;
5473         case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5474                 ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num);
5475                 break;
5476         case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5477                 ibdev_warn(ibdev, "DB overflow.\n");
5478                 break;
5479         case HNS_ROCE_EVENT_TYPE_FLR:
5480                 ibdev_warn(ibdev, "Function level reset.\n");
5481                 break;
5482         default:
5483                 break;
5484         }
5485
5486         kfree(irq_work);
5487 }
5488
5489 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
5490                                       struct hns_roce_eq *eq, u32 queue_num)
5491 {
5492         struct hns_roce_work *irq_work;
5493
5494         irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
5495         if (!irq_work)
5496                 return;
5497
5498         INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
5499         irq_work->hr_dev = hr_dev;
5500         irq_work->event_type = eq->event_type;
5501         irq_work->sub_type = eq->sub_type;
5502         irq_work->queue_num = queue_num;
5503         queue_work(hr_dev->irq_workq, &(irq_work->work));
5504 }
5505
5506 static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
5507 {
5508         struct hns_roce_dev *hr_dev = eq->hr_dev;
5509         __le32 doorbell[2] = {};
5510
5511         if (eq->type_flag == HNS_ROCE_AEQ) {
5512                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
5513                                HNS_ROCE_V2_EQ_DB_CMD_S,
5514                                eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
5515                                HNS_ROCE_EQ_DB_CMD_AEQ :
5516                                HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
5517         } else {
5518                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
5519                                HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
5520
5521                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
5522                                HNS_ROCE_V2_EQ_DB_CMD_S,
5523                                eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
5524                                HNS_ROCE_EQ_DB_CMD_CEQ :
5525                                HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
5526         }
5527
5528         roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
5529                        HNS_ROCE_V2_EQ_DB_PARA_S,
5530                        (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
5531
5532         hns_roce_write64(hr_dev, doorbell, eq->doorbell);
5533 }
5534
5535 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
5536 {
5537         struct hns_roce_aeqe *aeqe;
5538
5539         aeqe = hns_roce_buf_offset(eq->mtr.kmem,
5540                                    (eq->cons_index & (eq->entries - 1)) *
5541                                    eq->eqe_size);
5542
5543         return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
5544                 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
5545 }
5546
5547 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
5548                                struct hns_roce_eq *eq)
5549 {
5550         struct device *dev = hr_dev->dev;
5551         struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
5552         int aeqe_found = 0;
5553         int event_type;
5554         u32 queue_num;
5555         int sub_type;
5556
5557         while (aeqe) {
5558                 /* Make sure we read AEQ entry after we have checked the
5559                  * ownership bit
5560                  */
5561                 dma_rmb();
5562
5563                 event_type = roce_get_field(aeqe->asyn,
5564                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
5565                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
5566                 sub_type = roce_get_field(aeqe->asyn,
5567                                           HNS_ROCE_V2_AEQE_SUB_TYPE_M,
5568                                           HNS_ROCE_V2_AEQE_SUB_TYPE_S);
5569                 queue_num = roce_get_field(aeqe->event.queue_event.num,
5570                                            HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5571                                            HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5572
5573                 switch (event_type) {
5574                 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
5575                 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
5576                 case HNS_ROCE_EVENT_TYPE_COMM_EST:
5577                 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5578                 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
5579                 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
5580                 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
5581                 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
5582                         hns_roce_qp_event(hr_dev, queue_num, event_type);
5583                         break;
5584                 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
5585                 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5586                         hns_roce_srq_event(hr_dev, queue_num, event_type);
5587                         break;
5588                 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5589                 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5590                         hns_roce_cq_event(hr_dev, queue_num, event_type);
5591                         break;
5592                 case HNS_ROCE_EVENT_TYPE_MB:
5593                         hns_roce_cmd_event(hr_dev,
5594                                         le16_to_cpu(aeqe->event.cmd.token),
5595                                         aeqe->event.cmd.status,
5596                                         le64_to_cpu(aeqe->event.cmd.out_param));
5597                         break;
5598                 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5599                 case HNS_ROCE_EVENT_TYPE_FLR:
5600                         break;
5601                 default:
5602                         dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
5603                                 event_type, eq->eqn, eq->cons_index);
5604                         break;
5605                 }
5606
5607                 eq->event_type = event_type;
5608                 eq->sub_type = sub_type;
5609                 ++eq->cons_index;
5610                 aeqe_found = 1;
5611
5612                 hns_roce_v2_init_irq_work(hr_dev, eq, queue_num);
5613
5614                 aeqe = next_aeqe_sw_v2(eq);
5615         }
5616
5617         set_eq_cons_index_v2(eq);
5618         return aeqe_found;
5619 }
5620
5621 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
5622 {
5623         struct hns_roce_ceqe *ceqe;
5624
5625         ceqe = hns_roce_buf_offset(eq->mtr.kmem,
5626                                    (eq->cons_index & (eq->entries - 1)) *
5627                                    eq->eqe_size);
5628
5629         return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
5630                 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
5631 }
5632
5633 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
5634                                struct hns_roce_eq *eq)
5635 {
5636         struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
5637         int ceqe_found = 0;
5638         u32 cqn;
5639
5640         while (ceqe) {
5641                 /* Make sure we read CEQ entry after we have checked the
5642                  * ownership bit
5643                  */
5644                 dma_rmb();
5645
5646                 cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M,
5647                                      HNS_ROCE_V2_CEQE_COMP_CQN_S);
5648
5649                 hns_roce_cq_completion(hr_dev, cqn);
5650
5651                 ++eq->cons_index;
5652                 ceqe_found = 1;
5653
5654                 ceqe = next_ceqe_sw_v2(eq);
5655         }
5656
5657         set_eq_cons_index_v2(eq);
5658
5659         return ceqe_found;
5660 }
5661
5662 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
5663 {
5664         struct hns_roce_eq *eq = eq_ptr;
5665         struct hns_roce_dev *hr_dev = eq->hr_dev;
5666         int int_work;
5667
5668         if (eq->type_flag == HNS_ROCE_CEQ)
5669                 /* Completion event interrupt */
5670                 int_work = hns_roce_v2_ceq_int(hr_dev, eq);
5671         else
5672                 /* Asychronous event interrupt */
5673                 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
5674
5675         return IRQ_RETVAL(int_work);
5676 }
5677
5678 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
5679 {
5680         struct hns_roce_dev *hr_dev = dev_id;
5681         struct device *dev = hr_dev->dev;
5682         int int_work = 0;
5683         u32 int_st;
5684         u32 int_en;
5685
5686         /* Abnormal interrupt */
5687         int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
5688         int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
5689
5690         if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
5691                 struct pci_dev *pdev = hr_dev->pci_dev;
5692                 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5693                 const struct hnae3_ae_ops *ops = ae_dev->ops;
5694
5695                 dev_err(dev, "AEQ overflow!\n");
5696
5697                 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S;
5698                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5699
5700                 /* Set reset level for reset_event() */
5701                 if (ops->set_default_reset_request)
5702                         ops->set_default_reset_request(ae_dev,
5703                                                        HNAE3_FUNC_RESET);
5704                 if (ops->reset_event)
5705                         ops->reset_event(pdev, NULL);
5706
5707                 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5708                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5709
5710                 int_work = 1;
5711         } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
5712                 dev_err(dev, "BUS ERR!\n");
5713
5714                 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S;
5715                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5716
5717                 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5718                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5719
5720                 int_work = 1;
5721         } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
5722                 dev_err(dev, "OTHER ERR!\n");
5723
5724                 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S;
5725                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5726
5727                 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
5728                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5729
5730                 int_work = 1;
5731         } else
5732                 dev_err(dev, "There is no abnormal irq found!\n");
5733
5734         return IRQ_RETVAL(int_work);
5735 }
5736
5737 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
5738                                         int eq_num, int enable_flag)
5739 {
5740         int i;
5741
5742         if (enable_flag == EQ_ENABLE) {
5743                 for (i = 0; i < eq_num; i++)
5744                         roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5745                                    i * EQ_REG_OFFSET,
5746                                    HNS_ROCE_V2_VF_EVENT_INT_EN_M);
5747
5748                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5749                            HNS_ROCE_V2_VF_ABN_INT_EN_M);
5750                 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5751                            HNS_ROCE_V2_VF_ABN_INT_CFG_M);
5752         } else {
5753                 for (i = 0; i < eq_num; i++)
5754                         roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5755                                    i * EQ_REG_OFFSET,
5756                                    HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
5757
5758                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5759                            HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
5760                 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5761                            HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
5762         }
5763 }
5764
5765 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
5766 {
5767         struct device *dev = hr_dev->dev;
5768         int ret;
5769
5770         if (eqn < hr_dev->caps.num_comp_vectors)
5771                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5772                                         0, HNS_ROCE_CMD_DESTROY_CEQC,
5773                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5774         else
5775                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5776                                         0, HNS_ROCE_CMD_DESTROY_AEQC,
5777                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5778         if (ret)
5779                 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
5780 }
5781
5782 static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
5783 {
5784         hns_roce_mtr_destroy(hr_dev, &eq->mtr);
5785 }
5786
5787 static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
5788                       void *mb_buf)
5789 {
5790         u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
5791         struct hns_roce_eq_context *eqc;
5792         u64 bt_ba = 0;
5793         int count;
5794
5795         eqc = mb_buf;
5796         memset(eqc, 0, sizeof(struct hns_roce_eq_context));
5797
5798         /* init eqc */
5799         eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
5800         eq->cons_index = 0;
5801         eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
5802         eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
5803         eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
5804         eq->shift = ilog2((unsigned int)eq->entries);
5805
5806         /* if not multi-hop, eqe buffer only use one trunk */
5807         count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT,
5808                                   &bt_ba);
5809         if (count < 1) {
5810                 dev_err(hr_dev->dev, "failed to find EQE mtr\n");
5811                 return -ENOBUFS;
5812         }
5813
5814         /* set eqc state */
5815         roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S,
5816                        HNS_ROCE_V2_EQ_STATE_VALID);
5817
5818         /* set eqe hop num */
5819         roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M,
5820                        HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
5821
5822         /* set eqc over_ignore */
5823         roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M,
5824                        HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
5825
5826         /* set eqc coalesce */
5827         roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M,
5828                        HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
5829
5830         /* set eqc arm_state */
5831         roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M,
5832                        HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
5833
5834         /* set eqn */
5835         roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S,
5836                        eq->eqn);
5837
5838         /* set eqe_cnt */
5839         roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M,
5840                        HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT);
5841
5842         /* set eqe_ba_pg_sz */
5843         roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M,
5844                        HNS_ROCE_EQC_BA_PG_SZ_S,
5845                        to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));
5846
5847         /* set eqe_buf_pg_sz */
5848         roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M,
5849                        HNS_ROCE_EQC_BUF_PG_SZ_S,
5850                        to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));
5851
5852         /* set eq_producer_idx */
5853         roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M,
5854                        HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX);
5855
5856         /* set eq_max_cnt */
5857         roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M,
5858                        HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
5859
5860         /* set eq_period */
5861         roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M,
5862                        HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
5863
5864         /* set eqe_report_timer */
5865         roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M,
5866                        HNS_ROCE_EQC_REPORT_TIMER_S,
5867                        HNS_ROCE_EQ_INIT_REPORT_TIMER);
5868
5869         /* set bt_ba [34:3] */
5870         roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M,
5871                        HNS_ROCE_EQC_EQE_BA_L_S, bt_ba >> 3);
5872
5873         /* set bt_ba [64:35] */
5874         roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M,
5875                        HNS_ROCE_EQC_EQE_BA_H_S, bt_ba >> 35);
5876
5877         /* set eq shift */
5878         roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S,
5879                        eq->shift);
5880
5881         /* set eq MSI_IDX */
5882         roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M,
5883                        HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX);
5884
5885         /* set cur_eqe_ba [27:12] */
5886         roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M,
5887                        HNS_ROCE_EQC_CUR_EQE_BA_L_S, eqe_ba[0] >> 12);
5888
5889         /* set cur_eqe_ba [59:28] */
5890         roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M,
5891                        HNS_ROCE_EQC_CUR_EQE_BA_M_S, eqe_ba[0] >> 28);
5892
5893         /* set cur_eqe_ba [63:60] */
5894         roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M,
5895                        HNS_ROCE_EQC_CUR_EQE_BA_H_S, eqe_ba[0] >> 60);
5896
5897         /* set eq consumer idx */
5898         roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M,
5899                        HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX);
5900
5901         roce_set_field(eqc->byte_40, HNS_ROCE_EQC_NXT_EQE_BA_L_M,
5902                        HNS_ROCE_EQC_NXT_EQE_BA_L_S, eqe_ba[1] >> 12);
5903
5904         roce_set_field(eqc->byte_44, HNS_ROCE_EQC_NXT_EQE_BA_H_M,
5905                        HNS_ROCE_EQC_NXT_EQE_BA_H_S, eqe_ba[1] >> 44);
5906
5907         roce_set_field(eqc->byte_44, HNS_ROCE_EQC_EQE_SIZE_M,
5908                        HNS_ROCE_EQC_EQE_SIZE_S,
5909                        eq->eqe_size == HNS_ROCE_V3_EQE_SIZE ? 1 : 0);
5910
5911         return 0;
5912 }
5913
5914 static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
5915 {
5916         struct hns_roce_buf_attr buf_attr = {};
5917         int err;
5918
5919         if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)
5920                 eq->hop_num = 0;
5921         else
5922                 eq->hop_num = hr_dev->caps.eqe_hop_num;
5923
5924         buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + HNS_HW_PAGE_SHIFT;
5925         buf_attr.region[0].size = eq->entries * eq->eqe_size;
5926         buf_attr.region[0].hopnum = eq->hop_num;
5927         buf_attr.region_count = 1;
5928
5929         err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
5930                                   hr_dev->caps.eqe_ba_pg_sz +
5931                                   HNS_HW_PAGE_SHIFT, NULL, 0);
5932         if (err)
5933                 dev_err(hr_dev->dev, "Failed to alloc EQE mtr, err %d\n", err);
5934
5935         return err;
5936 }
5937
5938 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
5939                                  struct hns_roce_eq *eq,
5940                                  unsigned int eq_cmd)
5941 {
5942         struct hns_roce_cmd_mailbox *mailbox;
5943         int ret;
5944
5945         /* Allocate mailbox memory */
5946         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5947         if (IS_ERR_OR_NULL(mailbox))
5948                 return -ENOMEM;
5949
5950         ret = alloc_eq_buf(hr_dev, eq);
5951         if (ret)
5952                 goto free_cmd_mbox;
5953
5954         ret = config_eqc(hr_dev, eq, mailbox->buf);
5955         if (ret)
5956                 goto err_cmd_mbox;
5957
5958         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
5959                                 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
5960         if (ret) {
5961                 dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");
5962                 goto err_cmd_mbox;
5963         }
5964
5965         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5966
5967         return 0;
5968
5969 err_cmd_mbox:
5970         free_eq_buf(hr_dev, eq);
5971
5972 free_cmd_mbox:
5973         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5974
5975         return ret;
5976 }
5977
5978 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
5979                                   int comp_num, int aeq_num, int other_num)
5980 {
5981         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5982         int i, j;
5983         int ret;
5984
5985         for (i = 0; i < irq_num; i++) {
5986                 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5987                                                GFP_KERNEL);
5988                 if (!hr_dev->irq_names[i]) {
5989                         ret = -ENOMEM;
5990                         goto err_kzalloc_failed;
5991                 }
5992         }
5993
5994         /* irq contains: abnormal + AEQ + CEQ */
5995         for (j = 0; j < other_num; j++)
5996                 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
5997                          "hns-abn-%d", j);
5998
5999         for (j = other_num; j < (other_num + aeq_num); j++)
6000                 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6001                          "hns-aeq-%d", j - other_num);
6002
6003         for (j = (other_num + aeq_num); j < irq_num; j++)
6004                 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6005                          "hns-ceq-%d", j - other_num - aeq_num);
6006
6007         for (j = 0; j < irq_num; j++) {
6008                 if (j < other_num)
6009                         ret = request_irq(hr_dev->irq[j],
6010                                           hns_roce_v2_msix_interrupt_abn,
6011                                           0, hr_dev->irq_names[j], hr_dev);
6012
6013                 else if (j < (other_num + comp_num))
6014                         ret = request_irq(eq_table->eq[j - other_num].irq,
6015                                           hns_roce_v2_msix_interrupt_eq,
6016                                           0, hr_dev->irq_names[j + aeq_num],
6017                                           &eq_table->eq[j - other_num]);
6018                 else
6019                         ret = request_irq(eq_table->eq[j - other_num].irq,
6020                                           hns_roce_v2_msix_interrupt_eq,
6021                                           0, hr_dev->irq_names[j - comp_num],
6022                                           &eq_table->eq[j - other_num]);
6023                 if (ret) {
6024                         dev_err(hr_dev->dev, "Request irq error!\n");
6025                         goto err_request_failed;
6026                 }
6027         }
6028
6029         return 0;
6030
6031 err_request_failed:
6032         for (j -= 1; j >= 0; j--)
6033                 if (j < other_num)
6034                         free_irq(hr_dev->irq[j], hr_dev);
6035                 else
6036                         free_irq(eq_table->eq[j - other_num].irq,
6037                                  &eq_table->eq[j - other_num]);
6038
6039 err_kzalloc_failed:
6040         for (i -= 1; i >= 0; i--)
6041                 kfree(hr_dev->irq_names[i]);
6042
6043         return ret;
6044 }
6045
6046 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
6047 {
6048         int irq_num;
6049         int eq_num;
6050         int i;
6051
6052         eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6053         irq_num = eq_num + hr_dev->caps.num_other_vectors;
6054
6055         for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
6056                 free_irq(hr_dev->irq[i], hr_dev);
6057
6058         for (i = 0; i < eq_num; i++)
6059                 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
6060
6061         for (i = 0; i < irq_num; i++)
6062                 kfree(hr_dev->irq_names[i]);
6063 }
6064
6065 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
6066 {
6067         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6068         struct device *dev = hr_dev->dev;
6069         struct hns_roce_eq *eq;
6070         unsigned int eq_cmd;
6071         int irq_num;
6072         int eq_num;
6073         int other_num;
6074         int comp_num;
6075         int aeq_num;
6076         int i;
6077         int ret;
6078
6079         other_num = hr_dev->caps.num_other_vectors;
6080         comp_num = hr_dev->caps.num_comp_vectors;
6081         aeq_num = hr_dev->caps.num_aeq_vectors;
6082
6083         eq_num = comp_num + aeq_num;
6084         irq_num = eq_num + other_num;
6085
6086         eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
6087         if (!eq_table->eq)
6088                 return -ENOMEM;
6089
6090         /* create eq */
6091         for (i = 0; i < eq_num; i++) {
6092                 eq = &eq_table->eq[i];
6093                 eq->hr_dev = hr_dev;
6094                 eq->eqn = i;
6095                 if (i < comp_num) {
6096                         /* CEQ */
6097                         eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
6098                         eq->type_flag = HNS_ROCE_CEQ;
6099                         eq->entries = hr_dev->caps.ceqe_depth;
6100                         eq->eqe_size = hr_dev->caps.ceqe_size;
6101                         eq->irq = hr_dev->irq[i + other_num + aeq_num];
6102                         eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
6103                         eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
6104                 } else {
6105                         /* AEQ */
6106                         eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
6107                         eq->type_flag = HNS_ROCE_AEQ;
6108                         eq->entries = hr_dev->caps.aeqe_depth;
6109                         eq->eqe_size = hr_dev->caps.aeqe_size;
6110                         eq->irq = hr_dev->irq[i - comp_num + other_num];
6111                         eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
6112                         eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
6113                 }
6114
6115                 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
6116                 if (ret) {
6117                         dev_err(dev, "eq create failed.\n");
6118                         goto err_create_eq_fail;
6119                 }
6120         }
6121
6122         /* enable irq */
6123         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
6124
6125         ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num,
6126                                      aeq_num, other_num);
6127         if (ret) {
6128                 dev_err(dev, "Request irq failed.\n");
6129                 goto err_request_irq_fail;
6130         }
6131
6132         hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0);
6133         if (!hr_dev->irq_workq) {
6134                 dev_err(dev, "Create irq workqueue failed!\n");
6135                 ret = -ENOMEM;
6136                 goto err_create_wq_fail;
6137         }
6138
6139         return 0;
6140
6141 err_create_wq_fail:
6142         __hns_roce_free_irq(hr_dev);
6143
6144 err_request_irq_fail:
6145         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6146
6147 err_create_eq_fail:
6148         for (i -= 1; i >= 0; i--)
6149                 free_eq_buf(hr_dev, &eq_table->eq[i]);
6150         kfree(eq_table->eq);
6151
6152         return ret;
6153 }
6154
6155 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
6156 {
6157         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6158         int eq_num;
6159         int i;
6160
6161         eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6162
6163         /* Disable irq */
6164         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6165
6166         __hns_roce_free_irq(hr_dev);
6167
6168         for (i = 0; i < eq_num; i++) {
6169                 hns_roce_v2_destroy_eqc(hr_dev, i);
6170
6171                 free_eq_buf(hr_dev, &eq_table->eq[i]);
6172         }
6173
6174         kfree(eq_table->eq);
6175
6176         flush_workqueue(hr_dev->irq_workq);
6177         destroy_workqueue(hr_dev->irq_workq);
6178 }
6179
6180 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
6181         .query_cqc_info = hns_roce_v2_query_cqc_info,
6182 };
6183
6184 static const struct ib_device_ops hns_roce_v2_dev_ops = {
6185         .destroy_qp = hns_roce_v2_destroy_qp,
6186         .modify_cq = hns_roce_v2_modify_cq,
6187         .poll_cq = hns_roce_v2_poll_cq,
6188         .post_recv = hns_roce_v2_post_recv,
6189         .post_send = hns_roce_v2_post_send,
6190         .query_qp = hns_roce_v2_query_qp,
6191         .req_notify_cq = hns_roce_v2_req_notify_cq,
6192 };
6193
6194 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6195         .modify_srq = hns_roce_v2_modify_srq,
6196         .post_srq_recv = hns_roce_v2_post_srq_recv,
6197         .query_srq = hns_roce_v2_query_srq,
6198 };
6199
6200 static const struct hns_roce_hw hns_roce_hw_v2 = {
6201         .cmq_init = hns_roce_v2_cmq_init,
6202         .cmq_exit = hns_roce_v2_cmq_exit,
6203         .hw_profile = hns_roce_v2_profile,
6204         .hw_init = hns_roce_v2_init,
6205         .hw_exit = hns_roce_v2_exit,
6206         .post_mbox = hns_roce_v2_post_mbox,
6207         .chk_mbox = hns_roce_v2_chk_mbox,
6208         .rst_prc_mbox = hns_roce_v2_rst_process_cmd,
6209         .set_gid = hns_roce_v2_set_gid,
6210         .set_mac = hns_roce_v2_set_mac,
6211         .write_mtpt = hns_roce_v2_write_mtpt,
6212         .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
6213         .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
6214         .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
6215         .write_cqc = hns_roce_v2_write_cqc,
6216         .set_hem = hns_roce_v2_set_hem,
6217         .clear_hem = hns_roce_v2_clear_hem,
6218         .modify_qp = hns_roce_v2_modify_qp,
6219         .query_qp = hns_roce_v2_query_qp,
6220         .destroy_qp = hns_roce_v2_destroy_qp,
6221         .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
6222         .modify_cq = hns_roce_v2_modify_cq,
6223         .post_send = hns_roce_v2_post_send,
6224         .post_recv = hns_roce_v2_post_recv,
6225         .req_notify_cq = hns_roce_v2_req_notify_cq,
6226         .poll_cq = hns_roce_v2_poll_cq,
6227         .init_eq = hns_roce_v2_init_eq_table,
6228         .cleanup_eq = hns_roce_v2_cleanup_eq_table,
6229         .write_srqc = hns_roce_v2_write_srqc,
6230         .modify_srq = hns_roce_v2_modify_srq,
6231         .query_srq = hns_roce_v2_query_srq,
6232         .post_srq_recv = hns_roce_v2_post_srq_recv,
6233         .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6234         .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
6235 };
6236
6237 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6238         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6239         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
6240         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6241         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
6242         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6243         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
6244         /* required last entry */
6245         {0, }
6246 };
6247
6248 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6249
6250 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
6251                                   struct hnae3_handle *handle)
6252 {
6253         struct hns_roce_v2_priv *priv = hr_dev->priv;
6254         int i;
6255
6256         hr_dev->pci_dev = handle->pdev;
6257         hr_dev->dev = &handle->pdev->dev;
6258         hr_dev->hw = &hns_roce_hw_v2;
6259         hr_dev->dfx = &hns_roce_dfx_hw_v2;
6260         hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6261         hr_dev->odb_offset = hr_dev->sdb_offset;
6262
6263         /* Get info from NIC driver. */
6264         hr_dev->reg_base = handle->rinfo.roce_io_base;
6265         hr_dev->mem_base = handle->rinfo.roce_mem_base;
6266         hr_dev->caps.num_ports = 1;
6267         hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6268         hr_dev->iboe.phy_port[0] = 0;
6269
6270         addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6271                             hr_dev->iboe.netdevs[0]->dev_addr);
6272
6273         for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
6274                 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6275                                                 i + handle->rinfo.base_vector);
6276
6277         /* cmd issue mode: 0 is poll, 1 is event */
6278         hr_dev->cmd_mod = 1;
6279         hr_dev->loop_idc = 0;
6280
6281         hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6282         priv->handle = handle;
6283 }
6284
6285 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6286 {
6287         struct hns_roce_dev *hr_dev;
6288         int ret;
6289
6290         hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
6291         if (!hr_dev)
6292                 return -ENOMEM;
6293
6294         hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6295         if (!hr_dev->priv) {
6296                 ret = -ENOMEM;
6297                 goto error_failed_kzalloc;
6298         }
6299
6300         hns_roce_hw_v2_get_cfg(hr_dev, handle);
6301
6302         ret = hns_roce_init(hr_dev);
6303         if (ret) {
6304                 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6305                 goto error_failed_get_cfg;
6306         }
6307
6308         handle->priv = hr_dev;
6309
6310         return 0;
6311
6312 error_failed_get_cfg:
6313         kfree(hr_dev->priv);
6314
6315 error_failed_kzalloc:
6316         ib_dealloc_device(&hr_dev->ib_dev);
6317
6318         return ret;
6319 }
6320
6321 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6322                                            bool reset)
6323 {
6324         struct hns_roce_dev *hr_dev = handle->priv;
6325
6326         if (!hr_dev)
6327                 return;
6328
6329         handle->priv = NULL;
6330
6331         hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
6332         hns_roce_handle_device_err(hr_dev);
6333
6334         hns_roce_exit(hr_dev);
6335         kfree(hr_dev->priv);
6336         ib_dealloc_device(&hr_dev->ib_dev);
6337 }
6338
6339 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6340 {
6341         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
6342         const struct pci_device_id *id;
6343         struct device *dev = &handle->pdev->dev;
6344         int ret;
6345
6346         handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
6347
6348         if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
6349                 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6350                 goto reset_chk_err;
6351         }
6352
6353         id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
6354         if (!id)
6355                 return 0;
6356
6357         ret = __hns_roce_hw_v2_init_instance(handle);
6358         if (ret) {
6359                 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6360                 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
6361                 if (ops->ae_dev_resetting(handle) ||
6362                     ops->get_hw_reset_stat(handle))
6363                         goto reset_chk_err;
6364                 else
6365                         return ret;
6366         }
6367
6368         handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
6369
6370
6371         return 0;
6372
6373 reset_chk_err:
6374         dev_err(dev, "Device is busy in resetting state.\n"
6375                      "please retry later.\n");
6376
6377         return -EBUSY;
6378 }
6379
6380 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6381                                            bool reset)
6382 {
6383         if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
6384                 return;
6385
6386         handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
6387
6388         __hns_roce_hw_v2_uninit_instance(handle, reset);
6389
6390         handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6391 }
6392 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
6393 {
6394         struct hns_roce_dev *hr_dev;
6395
6396         if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
6397                 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6398                 return 0;
6399         }
6400
6401         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
6402         clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6403
6404         hr_dev = handle->priv;
6405         if (!hr_dev)
6406                 return 0;
6407
6408         hr_dev->is_reset = true;
6409         hr_dev->active = false;
6410         hr_dev->dis_db = true;
6411
6412         hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
6413
6414         return 0;
6415 }
6416
6417 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
6418 {
6419         struct device *dev = &handle->pdev->dev;
6420         int ret;
6421
6422         if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
6423                                &handle->rinfo.state)) {
6424                 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6425                 return 0;
6426         }
6427
6428         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
6429
6430         dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
6431         ret = __hns_roce_hw_v2_init_instance(handle);
6432         if (ret) {
6433                 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
6434                  * callback function, RoCE Engine reinitialize. If RoCE reinit
6435                  * failed, we should inform NIC driver.
6436                  */
6437                 handle->priv = NULL;
6438                 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
6439         } else {
6440                 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6441                 dev_info(dev, "Reset done, RoCE client reinit finished.\n");
6442         }
6443
6444         return ret;
6445 }
6446
6447 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
6448 {
6449         if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
6450                 return 0;
6451
6452         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
6453         dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
6454         msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
6455         __hns_roce_hw_v2_uninit_instance(handle, false);
6456
6457         return 0;
6458 }
6459
6460 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
6461                                        enum hnae3_reset_notify_type type)
6462 {
6463         int ret = 0;
6464
6465         switch (type) {
6466         case HNAE3_DOWN_CLIENT:
6467                 ret = hns_roce_hw_v2_reset_notify_down(handle);
6468                 break;
6469         case HNAE3_INIT_CLIENT:
6470                 ret = hns_roce_hw_v2_reset_notify_init(handle);
6471                 break;
6472         case HNAE3_UNINIT_CLIENT:
6473                 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
6474                 break;
6475         default:
6476                 break;
6477         }
6478
6479         return ret;
6480 }
6481
6482 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
6483         .init_instance = hns_roce_hw_v2_init_instance,
6484         .uninit_instance = hns_roce_hw_v2_uninit_instance,
6485         .reset_notify = hns_roce_hw_v2_reset_notify,
6486 };
6487
6488 static struct hnae3_client hns_roce_hw_v2_client = {
6489         .name = "hns_roce_hw_v2",
6490         .type = HNAE3_CLIENT_ROCE,
6491         .ops = &hns_roce_hw_v2_ops,
6492 };
6493
6494 static int __init hns_roce_hw_v2_init(void)
6495 {
6496         return hnae3_register_client(&hns_roce_hw_v2_client);
6497 }
6498
6499 static void __exit hns_roce_hw_v2_exit(void)
6500 {
6501         hnae3_unregister_client(&hns_roce_hw_v2_client);
6502 }
6503
6504 module_init(hns_roce_hw_v2_init);
6505 module_exit(hns_roce_hw_v2_exit);
6506
6507 MODULE_LICENSE("Dual BSD/GPL");
6508 MODULE_AUTHOR("Wei Hu <[email protected]>");
6509 MODULE_AUTHOR("Lijun Ou <[email protected]>");
6510 MODULE_AUTHOR("Shaobo Xu <[email protected]>");
6511 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");