drivers/infiniband/hw/irdma/type.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
   2 /* Copyright (c) 2015 - 2021 Intel Corporation */
   3 #ifndef IRDMA_TYPE_H
   4 #define IRDMA_TYPE_H
   5 #include "osdep.h"
   6 #include "irdma.h"
   7 #include "user.h"
   8 #include "hmc.h"
   9 #include "uda.h"
  10 #include "ws.h"
  11 #define IRDMA_DEBUG_ERR         "ERR"
  12 #define IRDMA_DEBUG_INIT        "INIT"
  13 #define IRDMA_DEBUG_DEV         "DEV"
  14 #define IRDMA_DEBUG_CM          "CM"
  15 #define IRDMA_DEBUG_VERBS       "VERBS"
  16 #define IRDMA_DEBUG_PUDA        "PUDA"
  17 #define IRDMA_DEBUG_ILQ         "ILQ"
  18 #define IRDMA_DEBUG_IEQ         "IEQ"
  19 #define IRDMA_DEBUG_QP          "QP"
  20 #define IRDMA_DEBUG_CQ          "CQ"
  21 #define IRDMA_DEBUG_MR          "MR"
  22 #define IRDMA_DEBUG_PBLE        "PBLE"
  23 #define IRDMA_DEBUG_WQE         "WQE"
  24 #define IRDMA_DEBUG_AEQ         "AEQ"
  25 #define IRDMA_DEBUG_CQP         "CQP"
  26 #define IRDMA_DEBUG_HMC         "HMC"
  27 #define IRDMA_DEBUG_USER        "USER"
  28 #define IRDMA_DEBUG_VIRT        "VIRT"
  29 #define IRDMA_DEBUG_DCB         "DCB"
  30 #define IRDMA_DEBUG_CQE         "CQE"
  31 #define IRDMA_DEBUG_CLNT        "CLNT"
  32 #define IRDMA_DEBUG_WS          "WS"
  33 #define IRDMA_DEBUG_STATS       "STATS"
  34
  35 enum irdma_page_size {
  36         IRDMA_PAGE_SIZE_4K = 0,
  37         IRDMA_PAGE_SIZE_2M,
  38         IRDMA_PAGE_SIZE_1G,
  39 };
  40
  41 enum irdma_hdrct_flags {
  42         DDP_LEN_FLAG  = 0x80,
  43         DDP_HDR_FLAG  = 0x40,
  44         RDMA_HDR_FLAG = 0x20,
  45 };
  46
  47 enum irdma_term_layers {
  48         LAYER_RDMA = 0,
  49         LAYER_DDP  = 1,
  50         LAYER_MPA  = 2,
  51 };
  52
  53 enum irdma_term_error_types {
  54         RDMAP_REMOTE_PROT = 1,
  55         RDMAP_REMOTE_OP   = 2,
  56         DDP_CATASTROPHIC  = 0,
  57         DDP_TAGGED_BUF    = 1,
  58         DDP_UNTAGGED_BUF  = 2,
  59         DDP_LLP           = 3,
  60 };
  61
  62 enum irdma_term_rdma_errors {
  63         RDMAP_INV_STAG            = 0x00,
  64         RDMAP_INV_BOUNDS          = 0x01,
  65         RDMAP_ACCESS              = 0x02,
  66         RDMAP_UNASSOC_STAG        = 0x03,
  67         RDMAP_TO_WRAP             = 0x04,
  68         RDMAP_INV_RDMAP_VER       = 0x05,
  69         RDMAP_UNEXPECTED_OP       = 0x06,
  70         RDMAP_CATASTROPHIC_LOCAL  = 0x07,
  71         RDMAP_CATASTROPHIC_GLOBAL = 0x08,
  72         RDMAP_CANT_INV_STAG       = 0x09,
  73         RDMAP_UNSPECIFIED         = 0xff,
  74 };
  75
  76 enum irdma_term_ddp_errors {
  77         DDP_CATASTROPHIC_LOCAL      = 0x00,
  78         DDP_TAGGED_INV_STAG         = 0x00,
  79         DDP_TAGGED_BOUNDS           = 0x01,
  80         DDP_TAGGED_UNASSOC_STAG     = 0x02,
  81         DDP_TAGGED_TO_WRAP          = 0x03,
  82         DDP_TAGGED_INV_DDP_VER      = 0x04,
  83         DDP_UNTAGGED_INV_QN         = 0x01,
  84         DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
  85         DDP_UNTAGGED_INV_MSN_RANGE  = 0x03,
  86         DDP_UNTAGGED_INV_MO         = 0x04,
  87         DDP_UNTAGGED_INV_TOO_LONG   = 0x05,
  88         DDP_UNTAGGED_INV_DDP_VER    = 0x06,
  89 };
  90
  91 enum irdma_term_mpa_errors {
  92         MPA_CLOSED  = 0x01,
  93         MPA_CRC     = 0x02,
  94         MPA_MARKER  = 0x03,
  95         MPA_REQ_RSP = 0x04,
  96 };
  97
  98 enum irdma_qp_event_type {
  99         IRDMA_QP_EVENT_CATASTROPHIC,
 100         IRDMA_QP_EVENT_ACCESS_ERR,
 101         IRDMA_QP_EVENT_REQ_ERR,
 102 };
 103
 104 enum irdma_hw_stats_index {
 105         /* gen1 - 32-bit */
 106         IRDMA_HW_STAT_INDEX_IP4RXDISCARD        = 0,
 107         IRDMA_HW_STAT_INDEX_IP4RXTRUNC          = 1,
 108         IRDMA_HW_STAT_INDEX_IP4TXNOROUTE        = 2,
 109         IRDMA_HW_STAT_INDEX_IP6RXDISCARD        = 3,
 110         IRDMA_HW_STAT_INDEX_IP6RXTRUNC          = 4,
 111         IRDMA_HW_STAT_INDEX_IP6TXNOROUTE        = 5,
 112         IRDMA_HW_STAT_INDEX_TCPRTXSEG           = 6,
 113         IRDMA_HW_STAT_INDEX_TCPRXOPTERR         = 7,
 114         IRDMA_HW_STAT_INDEX_TCPRXPROTOERR       = 8,
 115         IRDMA_HW_STAT_INDEX_RXVLANERR           = 9,
 116                 /* gen1 - 64-bit */
 117         IRDMA_HW_STAT_INDEX_IP4RXOCTS           = 10,
 118         IRDMA_HW_STAT_INDEX_IP4RXPKTS           = 11,
 119         IRDMA_HW_STAT_INDEX_IP4RXFRAGS          = 12,
 120         IRDMA_HW_STAT_INDEX_IP4RXMCPKTS         = 13,
 121         IRDMA_HW_STAT_INDEX_IP4TXOCTS           = 14,
 122         IRDMA_HW_STAT_INDEX_IP4TXPKTS           = 15,
 123         IRDMA_HW_STAT_INDEX_IP4TXFRAGS          = 16,
 124         IRDMA_HW_STAT_INDEX_IP4TXMCPKTS         = 17,
 125         IRDMA_HW_STAT_INDEX_IP6RXOCTS           = 18,
 126         IRDMA_HW_STAT_INDEX_IP6RXPKTS           = 19,
 127         IRDMA_HW_STAT_INDEX_IP6RXFRAGS          = 20,
 128         IRDMA_HW_STAT_INDEX_IP6RXMCPKTS         = 21,
 129         IRDMA_HW_STAT_INDEX_IP6TXOCTS           = 22,
 130         IRDMA_HW_STAT_INDEX_IP6TXPKTS           = 23,
 131         IRDMA_HW_STAT_INDEX_IP6TXFRAGS          = 24,
 132         IRDMA_HW_STAT_INDEX_IP6TXMCPKTS         = 25,
 133         IRDMA_HW_STAT_INDEX_TCPRXSEGS           = 26,
 134         IRDMA_HW_STAT_INDEX_TCPTXSEG            = 27,
 135         IRDMA_HW_STAT_INDEX_RDMARXRDS           = 28,
 136         IRDMA_HW_STAT_INDEX_RDMARXSNDS          = 29,
 137         IRDMA_HW_STAT_INDEX_RDMARXWRS           = 30,
 138         IRDMA_HW_STAT_INDEX_RDMATXRDS           = 31,
 139         IRDMA_HW_STAT_INDEX_RDMATXSNDS          = 32,
 140         IRDMA_HW_STAT_INDEX_RDMATXWRS           = 33,
 141         IRDMA_HW_STAT_INDEX_RDMAVBND            = 34,
 142         IRDMA_HW_STAT_INDEX_RDMAVINV            = 35,
 143         IRDMA_HW_STAT_INDEX_IP4RXMCOCTS         = 36,
 144         IRDMA_HW_STAT_INDEX_IP4TXMCOCTS         = 37,
 145         IRDMA_HW_STAT_INDEX_IP6RXMCOCTS         = 38,
 146         IRDMA_HW_STAT_INDEX_IP6TXMCOCTS         = 39,
 147         IRDMA_HW_STAT_INDEX_UDPRXPKTS           = 40,
 148         IRDMA_HW_STAT_INDEX_UDPTXPKTS           = 41,
 149         IRDMA_HW_STAT_INDEX_MAX_GEN_1           = 42, /* Must be same value as next entry */
 150         /* gen2 - 64-bit */
 151         IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS   = 42,
 152         /* gen2 - 32-bit */
 153         IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED      = 43,
 154         IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED      = 44,
 155         IRDMA_HW_STAT_INDEX_TXNPCNPSENT         = 45,
 156         IRDMA_HW_STAT_INDEX_MAX_GEN_2           = 46,
 157 };
 158
 159 enum irdma_feature_type {
 160         IRDMA_FEATURE_FW_INFO = 0,
 161         IRDMA_HW_VERSION_INFO = 1,
 162         IRDMA_QSETS_MAX       = 26,
 163         IRDMA_MAX_FEATURES, /* Must be last entry */
 164 };
 165
 166 enum irdma_sched_prio_type {
 167         IRDMA_PRIO_WEIGHTED_RR     = 1,
 168         IRDMA_PRIO_STRICT          = 2,
 169         IRDMA_PRIO_WEIGHTED_STRICT = 3,
 170 };
 171
 172 enum irdma_vm_vf_type {
 173         IRDMA_VF_TYPE = 0,
 174         IRDMA_VM_TYPE,
 175         IRDMA_PF_TYPE,
 176 };
 177
 178 enum irdma_cqp_hmc_profile {
 179         IRDMA_HMC_PROFILE_DEFAULT  = 1,
 180         IRDMA_HMC_PROFILE_FAVOR_VF = 2,
 181         IRDMA_HMC_PROFILE_EQUAL    = 3,
 182 };
 183
 184 enum irdma_quad_entry_type {
 185         IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,
 186         IRDMA_QHASH_TYPE_TCP_SYN,
 187         IRDMA_QHASH_TYPE_UDP_UNICAST,
 188         IRDMA_QHASH_TYPE_UDP_MCAST,
 189         IRDMA_QHASH_TYPE_ROCE_MCAST,
 190         IRDMA_QHASH_TYPE_ROCEV2_HW,
 191 };
 192
 193 enum irdma_quad_hash_manage_type {
 194         IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,
 195         IRDMA_QHASH_MANAGE_TYPE_ADD,
 196         IRDMA_QHASH_MANAGE_TYPE_MODIFY,
 197 };
 198
 199 enum irdma_syn_rst_handling {
 200         IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,
 201         IRDMA_SYN_RST_HANDLING_HW_TCP,
 202         IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,
 203         IRDMA_SYN_RST_HANDLING_FW_TCP,
 204 };
 205
 206 enum irdma_queue_type {
 207         IRDMA_QUEUE_TYPE_SQ_RQ = 0,
 208         IRDMA_QUEUE_TYPE_CQP,
 209 };
 210
 211 struct irdma_sc_dev;
 212 struct irdma_vsi_pestat;
 213
 214 struct irdma_dcqcn_cc_params {
 215         u8 cc_cfg_valid;
 216         u8 min_dec_factor;
 217         u8 min_rate;
 218         u8 dcqcn_f;
 219         u16 rai_factor;
 220         u16 hai_factor;
 221         u16 dcqcn_t;
 222         u32 dcqcn_b;
 223         u32 rreduce_mperiod;
 224 };
 225
 226 struct irdma_cqp_init_info {
 227         u64 cqp_compl_ctx;
 228         u64 host_ctx_pa;
 229         u64 sq_pa;
 230         struct irdma_sc_dev *dev;
 231         struct irdma_cqp_quanta *sq;
 232         struct irdma_dcqcn_cc_params dcqcn_params;
 233         __le64 *host_ctx;
 234         u64 *scratch_array;
 235         u32 sq_size;
 236         u16 hw_maj_ver;
 237         u16 hw_min_ver;
 238         u8 struct_ver;
 239         u8 hmc_profile;
 240         u8 ena_vf_count;
 241         u8 ceqs_per_vf;
 242         bool en_datacenter_tcp:1;
 243         bool disable_packed:1;
 244         bool rocev2_rto_policy:1;
 245         enum irdma_protocol_used protocol_used;
 246 };
 247
 248 struct irdma_terminate_hdr {
 249         u8 layer_etype;
 250         u8 error_code;
 251         u8 hdrct;
 252         u8 rsvd;
 253 };
 254
 255 struct irdma_cqp_sq_wqe {
 256         __le64 buf[IRDMA_CQP_WQE_SIZE];
 257 };
 258
 259 struct irdma_sc_aeqe {
 260         __le64 buf[IRDMA_AEQE_SIZE];
 261 };
 262
 263 struct irdma_ceqe {
 264         __le64 buf[IRDMA_CEQE_SIZE];
 265 };
 266
 267 struct irdma_cqp_ctx {
 268         __le64 buf[IRDMA_CQP_CTX_SIZE];
 269 };
 270
 271 struct irdma_cq_shadow_area {
 272         __le64 buf[IRDMA_SHADOW_AREA_SIZE];
 273 };
 274
 275 struct irdma_dev_hw_stats_offsets {
 276         u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
 277 };
 278
 279 struct irdma_dev_hw_stats {
 280         u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
 281 };
 282
 283 struct irdma_gather_stats {
 284         u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
 285 };
 286
 287 struct irdma_hw_stat_map {
 288         u16 byteoff;
 289         u8 bitoff;
 290         u64 bitmask;
 291 };
 292
 293 struct irdma_stats_gather_info {
 294         bool use_hmc_fcn_index:1;
 295         bool use_stats_inst:1;
 296         u8 hmc_fcn_index;
 297         u8 stats_inst_index;
 298         struct irdma_dma_mem stats_buff_mem;
 299         void *gather_stats_va;
 300         void *last_gather_stats_va;
 301 };
 302
 303 struct irdma_vsi_pestat {
 304         struct irdma_hw *hw;
 305         struct irdma_dev_hw_stats hw_stats;
 306         struct irdma_stats_gather_info gather_info;
 307         struct timer_list stats_timer;
 308         struct irdma_sc_vsi *vsi;
 309         struct irdma_dev_hw_stats last_hw_stats;
 310         spinlock_t lock; /* rdma stats lock */
 311 };
 312
 313 struct irdma_hw {
 314         u8 __iomem *hw_addr;
 315         u8 __iomem *priv_hw_addr;
 316         struct device *device;
 317         struct irdma_hmc_info hmc;
 318 };
 319
 320 struct irdma_pfpdu {
 321         struct list_head rxlist;
 322         u32 rcv_nxt;
 323         u32 fps;
 324         u32 max_fpdu_data;
 325         u32 nextseqnum;
 326         u32 rcv_start_seq;
 327         bool mode:1;
 328         bool mpa_crc_err:1;
 329         u8  marker_len;
 330         u64 total_ieq_bufs;
 331         u64 fpdu_processed;
 332         u64 bad_seq_num;
 333         u64 crc_err;
 334         u64 no_tx_bufs;
 335         u64 tx_err;
 336         u64 out_of_order;
 337         u64 pmode_count;
 338         struct irdma_sc_ah *ah;
 339         struct irdma_puda_buf *ah_buf;
 340         spinlock_t lock; /* fpdu processing lock */
 341         struct irdma_puda_buf *lastrcv_buf;
 342 };
 343
 344 struct irdma_sc_pd {
 345         struct irdma_sc_dev *dev;
 346         u32 pd_id;
 347         int abi_ver;
 348 };
 349
 350 struct irdma_cqp_quanta {
 351         __le64 elem[IRDMA_CQP_WQE_SIZE];
 352 };
 353
 354 struct irdma_sc_cqp {
 355         u32 size;
 356         u64 sq_pa;
 357         u64 host_ctx_pa;
 358         void *back_cqp;
 359         struct irdma_sc_dev *dev;
 360         int (*process_cqp_sds)(struct irdma_sc_dev *dev,
 361                                struct irdma_update_sds_info *info);
 362         struct irdma_dma_mem sdbuf;
 363         struct irdma_ring sq_ring;
 364         struct irdma_cqp_quanta *sq_base;
 365         struct irdma_dcqcn_cc_params dcqcn_params;
 366         __le64 *host_ctx;
 367         u64 *scratch_array;
 368         u64 requested_ops;
 369         atomic64_t completed_ops;
 370         u32 cqp_id;
 371         u32 sq_size;
 372         u32 hw_sq_size;
 373         u16 hw_maj_ver;
 374         u16 hw_min_ver;
 375         u8 struct_ver;
 376         u8 polarity;
 377         u8 hmc_profile;
 378         u8 ena_vf_count;
 379         u8 timeout_count;
 380         u8 ceqs_per_vf;
 381         bool en_datacenter_tcp:1;
 382         bool disable_packed:1;
 383         bool rocev2_rto_policy:1;
 384         enum irdma_protocol_used protocol_used;
 385 };
 386
 387 struct irdma_sc_aeq {
 388         u32 size;
 389         u64 aeq_elem_pa;
 390         struct irdma_sc_dev *dev;
 391         struct irdma_sc_aeqe *aeqe_base;
 392         void *pbl_list;
 393         u32 elem_cnt;
 394         struct irdma_ring aeq_ring;
 395         u8 pbl_chunk_size;
 396         u32 first_pm_pbl_idx;
 397         u32 msix_idx;
 398         u8 polarity;
 399         bool virtual_map:1;
 400 };
 401
 402 struct irdma_sc_ceq {
 403         u32 size;
 404         u64 ceq_elem_pa;
 405         struct irdma_sc_dev *dev;
 406         struct irdma_ceqe *ceqe_base;
 407         void *pbl_list;
 408         u32 ceq_id;
 409         u32 elem_cnt;
 410         struct irdma_ring ceq_ring;
 411         u8 pbl_chunk_size;
 412         u8 tph_val;
 413         u32 first_pm_pbl_idx;
 414         u8 polarity;
 415         struct irdma_sc_vsi *vsi;
 416         struct irdma_sc_cq **reg_cq;
 417         u32 reg_cq_size;
 418         spinlock_t req_cq_lock; /* protect access to reg_cq array */
 419         bool virtual_map:1;
 420         bool tph_en:1;
 421         bool itr_no_expire:1;
 422 };
 423
 424 struct irdma_sc_cq {
 425         struct irdma_cq_uk cq_uk;
 426         u64 cq_pa;
 427         u64 shadow_area_pa;
 428         struct irdma_sc_dev *dev;
 429         struct irdma_sc_vsi *vsi;
 430         void *pbl_list;
 431         void *back_cq;
 432         u32 ceq_id;
 433         u32 shadow_read_threshold;
 434         u8 pbl_chunk_size;
 435         u8 cq_type;
 436         u8 tph_val;
 437         u32 first_pm_pbl_idx;
 438         bool ceqe_mask:1;
 439         bool virtual_map:1;
 440         bool check_overflow:1;
 441         bool ceq_id_valid:1;
 442         bool tph_en;
 443 };
 444
 445 struct irdma_sc_qp {
 446         struct irdma_qp_uk qp_uk;
 447         u64 sq_pa;
 448         u64 rq_pa;
 449         u64 hw_host_ctx_pa;
 450         u64 shadow_area_pa;
 451         u64 q2_pa;
 452         struct irdma_sc_dev *dev;
 453         struct irdma_sc_vsi *vsi;
 454         struct irdma_sc_pd *pd;
 455         __le64 *hw_host_ctx;
 456         void *llp_stream_handle;
 457         struct irdma_pfpdu pfpdu;
 458         u32 ieq_qp;
 459         u8 *q2_buf;
 460         u64 qp_compl_ctx;
 461         u32 push_idx;
 462         u16 qs_handle;
 463         u16 push_offset;
 464         u8 flush_wqes_count;
 465         u8 sq_tph_val;
 466         u8 rq_tph_val;
 467         u8 qp_state;
 468         u8 hw_sq_size;
 469         u8 hw_rq_size;
 470         u8 src_mac_addr_idx;
 471         bool on_qoslist:1;
 472         bool ieq_pass_thru:1;
 473         bool sq_tph_en:1;
 474         bool rq_tph_en:1;
 475         bool rcv_tph_en:1;
 476         bool xmit_tph_en:1;
 477         bool virtual_map:1;
 478         bool flush_sq:1;
 479         bool flush_rq:1;
 480         bool sq_flush_code:1;
 481         bool rq_flush_code:1;
 482         enum irdma_flush_opcode flush_code;
 483         enum irdma_qp_event_type event_type;
 484         u8 term_flags;
 485         u8 user_pri;
 486         struct list_head list;
 487 };
 488
 489 struct irdma_stats_inst_info {
 490         bool use_hmc_fcn_index;
 491         u8 hmc_fn_id;
 492         u8 stats_idx;
 493 };
 494
 495 struct irdma_up_info {
 496         u8 map[8];
 497         u8 cnp_up_override;
 498         u8 hmc_fcn_idx;
 499         bool use_vlan:1;
 500         bool use_cnp_up_override:1;
 501 };
 502
 503 #define IRDMA_MAX_WS_NODES      0x3FF
 504 #define IRDMA_WS_NODE_INVALID   0xFFFF
 505
 506 struct irdma_ws_node_info {
 507         u16 id;
 508         u16 vsi;
 509         u16 parent_id;
 510         u16 qs_handle;
 511         bool type_leaf:1;
 512         bool enable:1;
 513         u8 prio_type;
 514         u8 tc;
 515         u8 weight;
 516 };
 517
 518 struct irdma_hmc_fpm_misc {
 519         u32 max_ceqs;
 520         u32 max_sds;
 521         u32 xf_block_size;
 522         u32 q1_block_size;
 523         u32 ht_multiplier;
 524         u32 timer_bucket;
 525         u32 rrf_block_size;
 526         u32 ooiscf_block_size;
 527 };
 528
 529 #define IRDMA_LEAF_DEFAULT_REL_BW               64
 530 #define IRDMA_PARENT_DEFAULT_REL_BW             1
 531
 532 struct irdma_qos {
 533         struct list_head qplist;
 534         struct mutex qos_mutex; /* protect QoS attributes per QoS level */
 535         u64 lan_qos_handle;
 536         u32 l2_sched_node_id;
 537         u16 qs_handle;
 538         u8 traffic_class;
 539         u8 rel_bw;
 540         u8 prio_type;
 541         bool valid;
 542 };
 543
 544 #define IRDMA_INVALID_STATS_IDX 0xff
 545 struct irdma_sc_vsi {
 546         u16 vsi_idx;
 547         struct irdma_sc_dev *dev;
 548         void *back_vsi;
 549         u32 ilq_count;
 550         struct irdma_virt_mem ilq_mem;
 551         struct irdma_puda_rsrc *ilq;
 552         u32 ieq_count;
 553         struct irdma_virt_mem ieq_mem;
 554         struct irdma_puda_rsrc *ieq;
 555         u32 exception_lan_q;
 556         u16 mtu;
 557         u16 vm_id;
 558         enum irdma_vm_vf_type vm_vf_type;
 559         bool stats_inst_alloc:1;
 560         bool tc_change_pending:1;
 561         struct irdma_vsi_pestat *pestat;
 562         atomic_t qp_suspend_reqs;
 563         int (*register_qset)(struct irdma_sc_vsi *vsi,
 564                              struct irdma_ws_node *tc_node);
 565         void (*unregister_qset)(struct irdma_sc_vsi *vsi,
 566                                 struct irdma_ws_node *tc_node);
 567         u8 qos_rel_bw;
 568         u8 qos_prio_type;
 569         u8 stats_idx;
 570         u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
 571         struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
 572         u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
 573         bool dscp_mode:1;
 574 };
 575
 576 struct irdma_sc_dev {
 577         struct list_head cqp_cmd_head; /* head of the CQP command list */
 578         spinlock_t cqp_lock; /* protect CQP list access */
 579         bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1];
 580         struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
 581         u64 fpm_query_buf_pa;
 582         u64 fpm_commit_buf_pa;
 583         __le64 *fpm_query_buf;
 584         __le64 *fpm_commit_buf;
 585         struct irdma_hw *hw;
 586         u8 __iomem *db_addr;
 587         u32 __iomem *wqe_alloc_db;
 588         u32 __iomem *cq_arm_db;
 589         u32 __iomem *aeq_alloc_db;
 590         u32 __iomem *cqp_db;
 591         u32 __iomem *cq_ack_db;
 592         u32 __iomem *ceq_itr_mask_db;
 593         u32 __iomem *aeq_itr_mask_db;
 594         u32 __iomem *hw_regs[IRDMA_MAX_REGS];
 595         u32 ceq_itr;   /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
 596         u64 hw_masks[IRDMA_MAX_MASKS];
 597         u64 hw_shifts[IRDMA_MAX_SHIFTS];
 598         const struct irdma_hw_stat_map *hw_stats_map;
 599         u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
 600         u64 feature_info[IRDMA_MAX_FEATURES];
 601         u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
 602         struct irdma_hw_attrs hw_attrs;
 603         struct irdma_hmc_info *hmc_info;
 604         struct irdma_sc_cqp *cqp;
 605         struct irdma_sc_aeq *aeq;
 606         struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];
 607         struct irdma_sc_cq *ccq;
 608         const struct irdma_irq_ops *irq_ops;
 609         struct irdma_hmc_fpm_misc hmc_fpm_misc;
 610         struct irdma_ws_node *ws_tree_root;
 611         struct mutex ws_mutex; /* ws tree mutex */
 612         u16 num_vfs;
 613         u8 hmc_fn_id;
 614         u8 vf_id;
 615         bool vchnl_up:1;
 616         bool ceq_valid:1;
 617         u8 pci_rev;
 618         int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
 619         void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
 620         void (*ws_reset)(struct irdma_sc_vsi *vsi);
 621 };
 622
 623 struct irdma_modify_cq_info {
 624         u64 cq_pa;
 625         struct irdma_cqe *cq_base;
 626         u32 cq_size;
 627         u32 shadow_read_threshold;
 628         u8 pbl_chunk_size;
 629         u32 first_pm_pbl_idx;
 630         bool virtual_map:1;
 631         bool check_overflow;
 632         bool cq_resize:1;
 633 };
 634
 635 struct irdma_create_qp_info {
 636         bool ord_valid:1;
 637         bool tcp_ctx_valid:1;
 638         bool cq_num_valid:1;
 639         bool arp_cache_idx_valid:1;
 640         bool mac_valid:1;
 641         bool force_lpb;
 642         u8 next_iwarp_state;
 643 };
 644
 645 struct irdma_modify_qp_info {
 646         u64 rx_win0;
 647         u64 rx_win1;
 648         u16 new_mss;
 649         u8 next_iwarp_state;
 650         u8 curr_iwarp_state;
 651         u8 termlen;
 652         bool ord_valid:1;
 653         bool tcp_ctx_valid:1;
 654         bool udp_ctx_valid:1;
 655         bool cq_num_valid:1;
 656         bool arp_cache_idx_valid:1;
 657         bool reset_tcp_conn:1;
 658         bool remove_hash_idx:1;
 659         bool dont_send_term:1;
 660         bool dont_send_fin:1;
 661         bool cached_var_valid:1;
 662         bool mss_change:1;
 663         bool force_lpb:1;
 664         bool mac_valid:1;
 665 };
 666
 667 struct irdma_ccq_cqe_info {
 668         struct irdma_sc_cqp *cqp;
 669         u64 scratch;
 670         u32 op_ret_val;
 671         u16 maj_err_code;
 672         u16 min_err_code;
 673         u8 op_code;
 674         bool error;
 675 };
 676
 677 struct irdma_dcb_app_info {
 678         u8 priority;
 679         u8 selector;
 680         u16 prot_id;
 681 };
 682
 683 struct irdma_qos_tc_info {
 684         u64 tc_ctx;
 685         u8 rel_bw;
 686         u8 prio_type;
 687         u8 egress_virt_up;
 688         u8 ingress_virt_up;
 689 };
 690
 691 struct irdma_l2params {
 692         struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];
 693         struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];
 694         u32 num_apps;
 695         u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
 696         u16 mtu;
 697         u8 up2tc[IRDMA_MAX_USER_PRIORITY];
 698         u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
 699         u8 num_tc;
 700         u8 vsi_rel_bw;
 701         u8 vsi_prio_type;
 702         bool mtu_changed:1;
 703         bool tc_changed:1;
 704         bool dscp_mode:1;
 705 };
 706
 707 struct irdma_vsi_init_info {
 708         struct irdma_sc_dev *dev;
 709         void *back_vsi;
 710         struct irdma_l2params *params;
 711         u16 exception_lan_q;
 712         u16 pf_data_vsi_num;
 713         enum irdma_vm_vf_type vm_vf_type;
 714         u16 vm_id;
 715         int (*register_qset)(struct irdma_sc_vsi *vsi,
 716                              struct irdma_ws_node *tc_node);
 717         void (*unregister_qset)(struct irdma_sc_vsi *vsi,
 718                                 struct irdma_ws_node *tc_node);
 719 };
 720
 721 struct irdma_vsi_stats_info {
 722         struct irdma_vsi_pestat *pestat;
 723         u8 fcn_id;
 724         bool alloc_stats_inst;
 725 };
 726
 727 struct irdma_device_init_info {
 728         u64 fpm_query_buf_pa;
 729         u64 fpm_commit_buf_pa;
 730         __le64 *fpm_query_buf;
 731         __le64 *fpm_commit_buf;
 732         struct irdma_hw *hw;
 733         void __iomem *bar0;
 734         u8 hmc_fn_id;
 735 };
 736
 737 struct irdma_ceq_init_info {
 738         u64 ceqe_pa;
 739         struct irdma_sc_dev *dev;
 740         u64 *ceqe_base;
 741         void *pbl_list;
 742         u32 elem_cnt;
 743         u32 ceq_id;
 744         bool virtual_map:1;
 745         bool tph_en:1;
 746         bool itr_no_expire:1;
 747         u8 pbl_chunk_size;
 748         u8 tph_val;
 749         u32 first_pm_pbl_idx;
 750         struct irdma_sc_vsi *vsi;
 751         struct irdma_sc_cq **reg_cq;
 752         u32 reg_cq_idx;
 753 };
 754
 755 struct irdma_aeq_init_info {
 756         u64 aeq_elem_pa;
 757         struct irdma_sc_dev *dev;
 758         u32 *aeqe_base;
 759         void *pbl_list;
 760         u32 elem_cnt;
 761         bool virtual_map;
 762         u8 pbl_chunk_size;
 763         u32 first_pm_pbl_idx;
 764         u32 msix_idx;
 765 };
 766
 767 struct irdma_ccq_init_info {
 768         u64 cq_pa;
 769         u64 shadow_area_pa;
 770         struct irdma_sc_dev *dev;
 771         struct irdma_cqe *cq_base;
 772         __le64 *shadow_area;
 773         void *pbl_list;
 774         u32 num_elem;
 775         u32 ceq_id;
 776         u32 shadow_read_threshold;
 777         bool ceqe_mask:1;
 778         bool ceq_id_valid:1;
 779         bool avoid_mem_cflct:1;
 780         bool virtual_map:1;
 781         bool tph_en:1;
 782         u8 tph_val;
 783         u8 pbl_chunk_size;
 784         u32 first_pm_pbl_idx;
 785         struct irdma_sc_vsi *vsi;
 786 };
 787
 788 struct irdma_udp_offload_info {
 789         bool ipv4:1;
 790         bool insert_vlan_tag:1;
 791         u8 ttl;
 792         u8 tos;
 793         u16 src_port;
 794         u16 dst_port;
 795         u32 dest_ip_addr[4];
 796         u32 snd_mss;
 797         u16 vlan_tag;
 798         u16 arp_idx;
 799         u32 flow_label;
 800         u8 udp_state;
 801         u32 psn_nxt;
 802         u32 lsn;
 803         u32 epsn;
 804         u32 psn_max;
 805         u32 psn_una;
 806         u32 local_ipaddr[4];
 807         u32 cwnd;
 808         u8 rexmit_thresh;
 809         u8 rnr_nak_thresh;
 810 };
 811
 812 struct irdma_roce_offload_info {
 813         u16 p_key;
 814         u16 err_rq_idx;
 815         u32 qkey;
 816         u32 dest_qp;
 817         u8 roce_tver;
 818         u8 ack_credits;
 819         u8 err_rq_idx_valid;
 820         u32 pd_id;
 821         u16 ord_size;
 822         u16 ird_size;
 823         bool is_qp1:1;
 824         bool udprivcq_en:1;
 825         bool dcqcn_en:1;
 826         bool rcv_no_icrc:1;
 827         bool wr_rdresp_en:1;
 828         bool bind_en:1;
 829         bool fast_reg_en:1;
 830         bool priv_mode_en:1;
 831         bool rd_en:1;
 832         bool timely_en:1;
 833         bool dctcp_en:1;
 834         bool fw_cc_enable:1;
 835         bool use_stats_inst:1;
 836         u16 t_high;
 837         u16 t_low;
 838         u8 last_byte_sent;
 839         u8 mac_addr[ETH_ALEN];
 840         u8 rtomin;
 841 };
 842
 843 struct irdma_iwarp_offload_info {
 844         u16 rcv_mark_offset;
 845         u16 snd_mark_offset;
 846         u8 ddp_ver;
 847         u8 rdmap_ver;
 848         u8 iwarp_mode;
 849         u16 err_rq_idx;
 850         u32 pd_id;
 851         u16 ord_size;
 852         u16 ird_size;
 853         bool ib_rd_en:1;
 854         bool align_hdrs:1;
 855         bool rcv_no_mpa_crc:1;
 856         bool err_rq_idx_valid:1;
 857         bool snd_mark_en:1;
 858         bool rcv_mark_en:1;
 859         bool wr_rdresp_en:1;
 860         bool bind_en:1;
 861         bool fast_reg_en:1;
 862         bool priv_mode_en:1;
 863         bool rd_en:1;
 864         bool timely_en:1;
 865         bool use_stats_inst:1;
 866         bool ecn_en:1;
 867         bool dctcp_en:1;
 868         u16 t_high;
 869         u16 t_low;
 870         u8 last_byte_sent;
 871         u8 mac_addr[ETH_ALEN];
 872         u8 rtomin;
 873 };
 874
 875 struct irdma_tcp_offload_info {
 876         bool ipv4:1;
 877         bool no_nagle:1;
 878         bool insert_vlan_tag:1;
 879         bool time_stamp:1;
 880         bool drop_ooo_seg:1;
 881         bool avoid_stretch_ack:1;
 882         bool wscale:1;
 883         bool ignore_tcp_opt:1;
 884         bool ignore_tcp_uns_opt:1;
 885         u8 cwnd_inc_limit;
 886         u8 dup_ack_thresh;
 887         u8 ttl;
 888         u8 src_mac_addr_idx;
 889         u8 tos;
 890         u16 src_port;
 891         u16 dst_port;
 892         u32 dest_ip_addr[4];
 893         //u32 dest_ip_addr0;
 894         //u32 dest_ip_addr1;
 895         //u32 dest_ip_addr2;
 896         //u32 dest_ip_addr3;
 897         u32 snd_mss;
 898         u16 syn_rst_handling;
 899         u16 vlan_tag;
 900         u16 arp_idx;
 901         u32 flow_label;
 902         u8 tcp_state;
 903         u8 snd_wscale;
 904         u8 rcv_wscale;
 905         u32 time_stamp_recent;
 906         u32 time_stamp_age;
 907         u32 snd_nxt;
 908         u32 snd_wnd;
 909         u32 rcv_nxt;
 910         u32 rcv_wnd;
 911         u32 snd_max;
 912         u32 snd_una;
 913         u32 srtt;
 914         u32 rtt_var;
 915         u32 ss_thresh;
 916         u32 cwnd;
 917         u32 snd_wl1;
 918         u32 snd_wl2;
 919         u32 max_snd_window;
 920         u8 rexmit_thresh;
 921         u32 local_ipaddr[4];
 922 };
 923
 924 struct irdma_qp_host_ctx_info {
 925         u64 qp_compl_ctx;
 926         union {
 927                 struct irdma_tcp_offload_info *tcp_info;
 928                 struct irdma_udp_offload_info *udp_info;
 929         };
 930         union {
 931                 struct irdma_iwarp_offload_info *iwarp_info;
 932                 struct irdma_roce_offload_info *roce_info;
 933         };
 934         u32 send_cq_num;
 935         u32 rcv_cq_num;
 936         u32 rem_endpoint_idx;
 937         u8 stats_idx;
 938         bool srq_valid:1;
 939         bool tcp_info_valid:1;
 940         bool iwarp_info_valid:1;
 941         bool stats_idx_valid:1;
 942         u8 user_pri;
 943 };
 944
 945 struct irdma_aeqe_info {
 946         u64 compl_ctx;
 947         u32 qp_cq_id;
 948         u16 ae_id;
 949         u16 wqe_idx;
 950         u8 tcp_state;
 951         u8 iwarp_state;
 952         bool qp:1;
 953         bool cq:1;
 954         bool sq:1;
 955         bool rq:1;
 956         bool in_rdrsp_wr:1;
 957         bool out_rdrsp:1;
 958         bool aeqe_overflow:1;
 959         u8 q2_data_written;
 960         u8 ae_src;
 961 };
 962
 963 struct irdma_allocate_stag_info {
 964         u64 total_len;
 965         u64 first_pm_pbl_idx;
 966         u32 chunk_size;
 967         u32 stag_idx;
 968         u32 page_size;
 969         u32 pd_id;
 970         u16 access_rights;
 971         bool remote_access:1;
 972         bool use_hmc_fcn_index:1;
 973         bool use_pf_rid:1;
 974         bool all_memory:1;
 975         u8 hmc_fcn_index;
 976 };
 977
 978 struct irdma_mw_alloc_info {
 979         u32 mw_stag_index;
 980         u32 page_size;
 981         u32 pd_id;
 982         bool remote_access:1;
 983         bool mw_wide:1;
 984         bool mw1_bind_dont_vldt_key:1;
 985 };
 986
 987 struct irdma_reg_ns_stag_info {
 988         u64 reg_addr_pa;
 989         u64 va;
 990         u64 total_len;
 991         u32 page_size;
 992         u32 chunk_size;
 993         u32 first_pm_pbl_index;
 994         enum irdma_addressing_type addr_type;
 995         irdma_stag_index stag_idx;
 996         u16 access_rights;
 997         u32 pd_id;
 998         irdma_stag_key stag_key;
 999         bool use_hmc_fcn_index:1;
1000         u8 hmc_fcn_index;
1001         bool use_pf_rid:1;
1002         bool all_memory:1;
1003 };
1004
1005 struct irdma_fast_reg_stag_info {
1006         u64 wr_id;
1007         u64 reg_addr_pa;
1008         u64 fbo;
1009         void *va;
1010         u64 total_len;
1011         u32 page_size;
1012         u32 chunk_size;
1013         u32 first_pm_pbl_index;
1014         enum irdma_addressing_type addr_type;
1015         irdma_stag_index stag_idx;
1016         u16 access_rights;
1017         u32 pd_id;
1018         irdma_stag_key stag_key;
1019         bool local_fence:1;
1020         bool read_fence:1;
1021         bool signaled:1;
1022         bool use_hmc_fcn_index:1;
1023         u8 hmc_fcn_index;
1024         bool use_pf_rid:1;
1025         bool defer_flag:1;
1026 };
1027
1028 struct irdma_dealloc_stag_info {
1029         u32 stag_idx;
1030         u32 pd_id;
1031         bool mr:1;
1032         bool dealloc_pbl:1;
1033 };
1034
1035 struct irdma_register_shared_stag {
1036         u64 va;
1037         enum irdma_addressing_type addr_type;
1038         irdma_stag_index new_stag_idx;
1039         irdma_stag_index parent_stag_idx;
1040         u32 access_rights;
1041         u32 pd_id;
1042         u32 page_size;
1043         irdma_stag_key new_stag_key;
1044 };
1045
1046 struct irdma_qp_init_info {
1047         struct irdma_qp_uk_init_info qp_uk_init_info;
1048         struct irdma_sc_pd *pd;
1049         struct irdma_sc_vsi *vsi;
1050         __le64 *host_ctx;
1051         u8 *q2;
1052         u64 sq_pa;
1053         u64 rq_pa;
1054         u64 host_ctx_pa;
1055         u64 q2_pa;
1056         u64 shadow_area_pa;
1057         u8 sq_tph_val;
1058         u8 rq_tph_val;
1059         bool sq_tph_en:1;
1060         bool rq_tph_en:1;
1061         bool rcv_tph_en:1;
1062         bool xmit_tph_en:1;
1063         bool virtual_map:1;
1064 };
1065
1066 struct irdma_cq_init_info {
1067         struct irdma_sc_dev *dev;
1068         u64 cq_base_pa;
1069         u64 shadow_area_pa;
1070         u32 ceq_id;
1071         u32 shadow_read_threshold;
1072         u8 pbl_chunk_size;
1073         u32 first_pm_pbl_idx;
1074         bool virtual_map:1;
1075         bool ceqe_mask:1;
1076         bool ceq_id_valid:1;
1077         bool tph_en:1;
1078         u8 tph_val;
1079         u8 type;
1080         struct irdma_cq_uk_init_info cq_uk_init_info;
1081         struct irdma_sc_vsi *vsi;
1082 };
1083
1084 struct irdma_upload_context_info {
1085         u64 buf_pa;
1086         u32 qp_id;
1087         u8 qp_type;
1088         bool freeze_qp:1;
1089         bool raw_format:1;
1090 };
1091
1092 struct irdma_local_mac_entry_info {
1093         u8 mac_addr[6];
1094         u16 entry_idx;
1095 };
1096
1097 struct irdma_add_arp_cache_entry_info {
1098         u8 mac_addr[ETH_ALEN];
1099         u32 reach_max;
1100         u16 arp_index;
1101         bool permanent;
1102 };
1103
1104 struct irdma_apbvt_info {
1105         u16 port;
1106         bool add;
1107 };
1108
1109 struct irdma_qhash_table_info {
1110         struct irdma_sc_vsi *vsi;
1111         enum irdma_quad_hash_manage_type manage;
1112         enum irdma_quad_entry_type entry_type;
1113         bool vlan_valid:1;
1114         bool ipv4_valid:1;
1115         u8 mac_addr[ETH_ALEN];
1116         u16 vlan_id;
1117         u8 user_pri;
1118         u32 qp_num;
1119         u32 dest_ip[4];
1120         u32 src_ip[4];
1121         u16 dest_port;
1122         u16 src_port;
1123 };
1124
1125 struct irdma_cqp_manage_push_page_info {
1126         u32 push_idx;
1127         u16 qs_handle;
1128         u8 free_page;
1129         u8 push_page_type;
1130 };
1131
1132 struct irdma_qp_flush_info {
1133         u16 sq_minor_code;
1134         u16 sq_major_code;
1135         u16 rq_minor_code;
1136         u16 rq_major_code;
1137         u16 ae_code;
1138         u8 ae_src;
1139         bool sq:1;
1140         bool rq:1;
1141         bool userflushcode:1;
1142         bool generate_ae:1;
1143 };
1144
1145 struct irdma_gen_ae_info {
1146         u16 ae_code;
1147         u8 ae_src;
1148 };
1149
1150 struct irdma_cqp_timeout {
1151         u64 compl_cqp_cmds;
1152         u32 count;
1153 };
1154
1155 struct irdma_irq_ops {
1156         void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable);
1157         void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
1158                               bool enable);
1159         void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);
1160         void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);
1161 };
1162
1163 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);
1164 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
1165                         bool check_overflow, bool post_sq);
1166 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
1167 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
1168                               struct irdma_ccq_cqe_info *info);
1169 int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
1170                       struct irdma_ccq_init_info *info);
1171
1172 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
1173 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
1174
1175 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
1176 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
1177                       struct irdma_ceq_init_info *info);
1178 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);
1179 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
1180
1181 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
1182                       struct irdma_aeq_init_info *info);
1183 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
1184                            struct irdma_aeqe_info *info);
1185 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);
1186
1187 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
1188                       int abi_ver);
1189 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);
1190 void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
1191                               struct irdma_sc_dev *dev);
1192 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
1193 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
1194 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
1195                       struct irdma_cqp_init_info *info);
1196 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
1197 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
1198                                   struct irdma_ccq_cqe_info *cmpl_info);
1199 int irdma_sc_fast_register(struct irdma_sc_qp *qp,
1200                            struct irdma_fast_reg_stag_info *info, bool post_sq);
1201 int irdma_sc_qp_create(struct irdma_sc_qp *qp,
1202                        struct irdma_create_qp_info *info, u64 scratch,
1203                        bool post_sq);
1204 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
1205                         bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
1206 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
1207                            struct irdma_qp_flush_info *info, u64 scratch,
1208                            bool post_sq);
1209 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
1210 int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
1211                        struct irdma_modify_qp_info *info, u64 scratch,
1212                        bool post_sq);
1213 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1214                         irdma_stag stag);
1215
1216 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
1217 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1218                         struct irdma_qp_host_ctx_info *info);
1219 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1220                              struct irdma_qp_host_ctx_info *info);
1221 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
1222 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
1223 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
1224 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
1225                                         u8 hmc_fn_id, bool post_sq,
1226                                         bool poll_registers);
1227
1228 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
1229 struct cqp_info {
1230         union {
1231                 struct {
1232                         struct irdma_sc_qp *qp;
1233                         struct irdma_create_qp_info info;
1234                         u64 scratch;
1235                 } qp_create;
1236
1237                 struct {
1238                         struct irdma_sc_qp *qp;
1239                         struct irdma_modify_qp_info info;
1240                         u64 scratch;
1241                 } qp_modify;
1242
1243                 struct {
1244                         struct irdma_sc_qp *qp;
1245                         u64 scratch;
1246                         bool remove_hash_idx;
1247                         bool ignore_mw_bnd;
1248                 } qp_destroy;
1249
1250                 struct {
1251                         struct irdma_sc_cq *cq;
1252                         u64 scratch;
1253                         bool check_overflow;
1254                 } cq_create;
1255
1256                 struct {
1257                         struct irdma_sc_cq *cq;
1258                         struct irdma_modify_cq_info info;
1259                         u64 scratch;
1260                 } cq_modify;
1261
1262                 struct {
1263                         struct irdma_sc_cq *cq;
1264                         u64 scratch;
1265                 } cq_destroy;
1266
1267                 struct {
1268                         struct irdma_sc_dev *dev;
1269                         struct irdma_allocate_stag_info info;
1270                         u64 scratch;
1271                 } alloc_stag;
1272
1273                 struct {
1274                         struct irdma_sc_dev *dev;
1275                         struct irdma_mw_alloc_info info;
1276                         u64 scratch;
1277                 } mw_alloc;
1278
1279                 struct {
1280                         struct irdma_sc_dev *dev;
1281                         struct irdma_reg_ns_stag_info info;
1282                         u64 scratch;
1283                 } mr_reg_non_shared;
1284
1285                 struct {
1286                         struct irdma_sc_dev *dev;
1287                         struct irdma_dealloc_stag_info info;
1288                         u64 scratch;
1289                 } dealloc_stag;
1290
1291                 struct {
1292                         struct irdma_sc_cqp *cqp;
1293                         struct irdma_add_arp_cache_entry_info info;
1294                         u64 scratch;
1295                 } add_arp_cache_entry;
1296
1297                 struct {
1298                         struct irdma_sc_cqp *cqp;
1299                         u64 scratch;
1300                         u16 arp_index;
1301                 } del_arp_cache_entry;
1302
1303                 struct {
1304                         struct irdma_sc_cqp *cqp;
1305                         struct irdma_local_mac_entry_info info;
1306                         u64 scratch;
1307                 } add_local_mac_entry;
1308
1309                 struct {
1310                         struct irdma_sc_cqp *cqp;
1311                         u64 scratch;
1312                         u8 entry_idx;
1313                         u8 ignore_ref_count;
1314                 } del_local_mac_entry;
1315
1316                 struct {
1317                         struct irdma_sc_cqp *cqp;
1318                         u64 scratch;
1319                 } alloc_local_mac_entry;
1320
1321                 struct {
1322                         struct irdma_sc_cqp *cqp;
1323                         struct irdma_cqp_manage_push_page_info info;
1324                         u64 scratch;
1325                 } manage_push_page;
1326
1327                 struct {
1328                         struct irdma_sc_dev *dev;
1329                         struct irdma_upload_context_info info;
1330                         u64 scratch;
1331                 } qp_upload_context;
1332
1333                 struct {
1334                         struct irdma_sc_dev *dev;
1335                         struct irdma_hmc_fcn_info info;
1336                         u64 scratch;
1337                 } manage_hmc_pm;
1338
1339                 struct {
1340                         struct irdma_sc_ceq *ceq;
1341                         u64 scratch;
1342                 } ceq_create;
1343
1344                 struct {
1345                         struct irdma_sc_ceq *ceq;
1346                         u64 scratch;
1347                 } ceq_destroy;
1348
1349                 struct {
1350                         struct irdma_sc_aeq *aeq;
1351                         u64 scratch;
1352                 } aeq_create;
1353
1354                 struct {
1355                         struct irdma_sc_aeq *aeq;
1356                         u64 scratch;
1357                 } aeq_destroy;
1358
1359                 struct {
1360                         struct irdma_sc_qp *qp;
1361                         struct irdma_qp_flush_info info;
1362                         u64 scratch;
1363                 } qp_flush_wqes;
1364
1365                 struct {
1366                         struct irdma_sc_qp *qp;
1367                         struct irdma_gen_ae_info info;
1368                         u64 scratch;
1369                 } gen_ae;
1370
1371                 struct {
1372                         struct irdma_sc_cqp *cqp;
1373                         void *fpm_val_va;
1374                         u64 fpm_val_pa;
1375                         u8 hmc_fn_id;
1376                         u64 scratch;
1377                 } query_fpm_val;
1378
1379                 struct {
1380                         struct irdma_sc_cqp *cqp;
1381                         void *fpm_val_va;
1382                         u64 fpm_val_pa;
1383                         u8 hmc_fn_id;
1384                         u64 scratch;
1385                 } commit_fpm_val;
1386
1387                 struct {
1388                         struct irdma_sc_cqp *cqp;
1389                         struct irdma_apbvt_info info;
1390                         u64 scratch;
1391                 } manage_apbvt_entry;
1392
1393                 struct {
1394                         struct irdma_sc_cqp *cqp;
1395                         struct irdma_qhash_table_info info;
1396                         u64 scratch;
1397                 } manage_qhash_table_entry;
1398
1399                 struct {
1400                         struct irdma_sc_dev *dev;
1401                         struct irdma_update_sds_info info;
1402                         u64 scratch;
1403                 } update_pe_sds;
1404
1405                 struct {
1406                         struct irdma_sc_cqp *cqp;
1407                         struct irdma_sc_qp *qp;
1408                         u64 scratch;
1409                 } suspend_resume;
1410
1411                 struct {
1412                         struct irdma_sc_cqp *cqp;
1413                         struct irdma_ah_info info;
1414                         u64 scratch;
1415                 } ah_create;
1416
1417                 struct {
1418                         struct irdma_sc_cqp *cqp;
1419                         struct irdma_ah_info info;
1420                         u64 scratch;
1421                 } ah_destroy;
1422
1423                 struct {
1424                         struct irdma_sc_cqp *cqp;
1425                         struct irdma_mcast_grp_info info;
1426                         u64 scratch;
1427                 } mc_create;
1428
1429                 struct {
1430                         struct irdma_sc_cqp *cqp;
1431                         struct irdma_mcast_grp_info info;
1432                         u64 scratch;
1433                 } mc_destroy;
1434
1435                 struct {
1436                         struct irdma_sc_cqp *cqp;
1437                         struct irdma_mcast_grp_info info;
1438                         u64 scratch;
1439                 } mc_modify;
1440
1441                 struct {
1442                         struct irdma_sc_cqp *cqp;
1443                         struct irdma_stats_inst_info info;
1444                         u64 scratch;
1445                 } stats_manage;
1446
1447                 struct {
1448                         struct irdma_sc_cqp *cqp;
1449                         struct irdma_stats_gather_info info;
1450                         u64 scratch;
1451                 } stats_gather;
1452
1453                 struct {
1454                         struct irdma_sc_cqp *cqp;
1455                         struct irdma_ws_node_info info;
1456                         u64 scratch;
1457                 } ws_node;
1458
1459                 struct {
1460                         struct irdma_sc_cqp *cqp;
1461                         struct irdma_up_info info;
1462                         u64 scratch;
1463                 } up_map;
1464
1465                 struct {
1466                         struct irdma_sc_cqp *cqp;
1467                         struct irdma_dma_mem query_buff_mem;
1468                         u64 scratch;
1469                 } query_rdma;
1470         } u;
1471 };
1472
1473 struct cqp_cmds_info {
1474         struct list_head cqp_cmd_entry;
1475         u8 cqp_cmd;
1476         u8 post_sq;
1477         struct cqp_info in;
1478 };
1479
1480 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
1481                                            u32 *wqe_idx);
1482
1483 /**
1484  * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
1485  * @cqp: struct for cqp hw
1486  * @scratch: private data for CQP WQE
1487  */
1488 static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)
1489 {
1490         u32 wqe_idx;
1491
1492         return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
1493 }
1494 #endif /* IRDMA_TYPE_H */