drivers/infiniband/hw/irdma/utils.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2015 - 2021 Intel Corporation */
   3 #include "main.h"
   4
   5 /**
   6  * irdma_arp_table -manage arp table
   7  * @rf: RDMA PCI function
   8  * @ip_addr: ip address for device
   9  * @ipv4: IPv4 flag
  10  * @mac_addr: mac address ptr
  11  * @action: modify, delete or add
  12  */
  13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
  14                     const u8 *mac_addr, u32 action)
  15 {
  16         unsigned long flags;
  17         int arp_index;
  18         u32 ip[4] = {};
  19
  20         if (ipv4)
  21                 ip[0] = *ip_addr;
  22         else
  23                 memcpy(ip, ip_addr, sizeof(ip));
  24
  25         spin_lock_irqsave(&rf->arp_lock, flags);
  26         for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
  27                 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
  28                         break;
  29         }
  30
  31         switch (action) {
  32         case IRDMA_ARP_ADD:
  33                 if (arp_index != rf->arp_table_size) {
  34                         arp_index = -1;
  35                         break;
  36                 }
  37
  38                 arp_index = 0;
  39                 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
  40                                      (u32 *)&arp_index, &rf->next_arp_index)) {
  41                         arp_index = -1;
  42                         break;
  43                 }
  44
  45                 memcpy(rf->arp_table[arp_index].ip_addr, ip,
  46                        sizeof(rf->arp_table[arp_index].ip_addr));
  47                 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
  48                 break;
  49         case IRDMA_ARP_RESOLVE:
  50                 if (arp_index == rf->arp_table_size)
  51                         arp_index = -1;
  52                 break;
  53         case IRDMA_ARP_DELETE:
  54                 if (arp_index == rf->arp_table_size) {
  55                         arp_index = -1;
  56                         break;
  57                 }
  58
  59                 memset(rf->arp_table[arp_index].ip_addr, 0,
  60                        sizeof(rf->arp_table[arp_index].ip_addr));
  61                 eth_zero_addr(rf->arp_table[arp_index].mac_addr);
  62                 irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
  63                 break;
  64         default:
  65                 arp_index = -1;
  66                 break;
  67         }
  68
  69         spin_unlock_irqrestore(&rf->arp_lock, flags);
  70         return arp_index;
  71 }
  72
  73 /**
  74  * irdma_add_arp - add a new arp entry if needed
  75  * @rf: RDMA function
  76  * @ip: IP address
  77  * @ipv4: IPv4 flag
  78  * @mac: MAC address
  79  */
  80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)
  81 {
  82         int arpidx;
  83
  84         arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
  85         if (arpidx >= 0) {
  86                 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
  87                         return arpidx;
  88
  89                 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
  90                                        ipv4, IRDMA_ARP_DELETE);
  91         }
  92
  93         irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
  94
  95         return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
  96 }
  97
  98 /**
  99  * wr32 - write 32 bits to hw register
 100  * @hw: hardware information including registers
 101  * @reg: register offset
 102  * @val: value to write to register
 103  */
 104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
 105 {
 106         writel(val, hw->hw_addr + reg);
 107 }
 108
 109 /**
 110  * rd32 - read a 32 bit hw register
 111  * @hw: hardware information including registers
 112  * @reg: register offset
 113  *
 114  * Return value of register content
 115  */
 116 inline u32 rd32(struct irdma_hw *hw, u32 reg)
 117 {
 118         return readl(hw->hw_addr + reg);
 119 }
 120
 121 /**
 122  * rd64 - read a 64 bit hw register
 123  * @hw: hardware information including registers
 124  * @reg: register offset
 125  *
 126  * Return value of register content
 127  */
 128 inline u64 rd64(struct irdma_hw *hw, u32 reg)
 129 {
 130         return readq(hw->hw_addr + reg);
 131 }
 132
 133 static void irdma_gid_change_event(struct ib_device *ibdev)
 134 {
 135         struct ib_event ib_event;
 136
 137         ib_event.event = IB_EVENT_GID_CHANGE;
 138         ib_event.device = ibdev;
 139         ib_event.element.port_num = 1;
 140         ib_dispatch_event(&ib_event);
 141 }
 142
 143 /**
 144  * irdma_inetaddr_event - system notifier for ipv4 addr events
 145  * @notifier: not used
 146  * @event: event for notifier
 147  * @ptr: if address
 148  */
 149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
 150                          void *ptr)
 151 {
 152         struct in_ifaddr *ifa = ptr;
 153         struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
 154         struct irdma_device *iwdev;
 155         struct ib_device *ibdev;
 156         u32 local_ipaddr;
 157
 158         real_dev = rdma_vlan_dev_real_dev(netdev);
 159         if (!real_dev)
 160                 real_dev = netdev;
 161
 162         ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 163         if (!ibdev)
 164                 return NOTIFY_DONE;
 165
 166         iwdev = to_iwdev(ibdev);
 167         local_ipaddr = ntohl(ifa->ifa_address);
 168         ibdev_dbg(&iwdev->ibdev,
 169                   "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
 170                   event, &local_ipaddr, real_dev->dev_addr);
 171         switch (event) {
 172         case NETDEV_DOWN:
 173                 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
 174                                        &local_ipaddr, true, IRDMA_ARP_DELETE);
 175                 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);
 176                 irdma_gid_change_event(&iwdev->ibdev);
 177                 break;
 178         case NETDEV_UP:
 179         case NETDEV_CHANGEADDR:
 180                 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);
 181                 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);
 182                 irdma_gid_change_event(&iwdev->ibdev);
 183                 break;
 184         default:
 185                 break;
 186         }
 187
 188         ib_device_put(ibdev);
 189
 190         return NOTIFY_DONE;
 191 }
 192
 193 /**
 194  * irdma_inet6addr_event - system notifier for ipv6 addr events
 195  * @notifier: not used
 196  * @event: event for notifier
 197  * @ptr: if address
 198  */
 199 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
 200                           void *ptr)
 201 {
 202         struct inet6_ifaddr *ifa = ptr;
 203         struct net_device *real_dev, *netdev = ifa->idev->dev;
 204         struct irdma_device *iwdev;
 205         struct ib_device *ibdev;
 206         u32 local_ipaddr6[4];
 207
 208         real_dev = rdma_vlan_dev_real_dev(netdev);
 209         if (!real_dev)
 210                 real_dev = netdev;
 211
 212         ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 213         if (!ibdev)
 214                 return NOTIFY_DONE;
 215
 216         iwdev = to_iwdev(ibdev);
 217         irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
 218         ibdev_dbg(&iwdev->ibdev,
 219                   "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
 220                   event, local_ipaddr6, real_dev->dev_addr);
 221         switch (event) {
 222         case NETDEV_DOWN:
 223                 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
 224                                        local_ipaddr6, false, IRDMA_ARP_DELETE);
 225                 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);
 226                 irdma_gid_change_event(&iwdev->ibdev);
 227                 break;
 228         case NETDEV_UP:
 229         case NETDEV_CHANGEADDR:
 230                 irdma_add_arp(iwdev->rf, local_ipaddr6, false,
 231                               real_dev->dev_addr);
 232                 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);
 233                 irdma_gid_change_event(&iwdev->ibdev);
 234                 break;
 235         default:
 236                 break;
 237         }
 238
 239         ib_device_put(ibdev);
 240
 241         return NOTIFY_DONE;
 242 }
 243
 244 /**
 245  * irdma_net_event - system notifier for net events
 246  * @notifier: not used
 247  * @event: event for notifier
 248  * @ptr: neighbor
 249  */
 250 int irdma_net_event(struct notifier_block *notifier, unsigned long event,
 251                     void *ptr)
 252 {
 253         struct neighbour *neigh = ptr;
 254         struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;
 255         struct irdma_device *iwdev;
 256         struct ib_device *ibdev;
 257         __be32 *p;
 258         u32 local_ipaddr[4] = {};
 259         bool ipv4 = true;
 260
 261         switch (event) {
 262         case NETEVENT_NEIGH_UPDATE:
 263                 real_dev = rdma_vlan_dev_real_dev(netdev);
 264                 if (!real_dev)
 265                         real_dev = netdev;
 266                 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 267                 if (!ibdev)
 268                         return NOTIFY_DONE;
 269
 270                 iwdev = to_iwdev(ibdev);
 271                 p = (__be32 *)neigh->primary_key;
 272                 if (neigh->tbl->family == AF_INET6) {
 273                         ipv4 = false;
 274                         irdma_copy_ip_ntohl(local_ipaddr, p);
 275                 } else {
 276                         local_ipaddr[0] = ntohl(*p);
 277                 }
 278
 279                 ibdev_dbg(&iwdev->ibdev,
 280                           "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
 281                           iwdev->netdev, neigh->nud_state, local_ipaddr,
 282                           neigh->ha);
 283
 284                 if (neigh->nud_state & NUD_VALID)
 285                         irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);
 286
 287                 else
 288                         irdma_manage_arp_cache(iwdev->rf, neigh->ha,
 289                                                local_ipaddr, ipv4,
 290                                                IRDMA_ARP_DELETE);
 291                 ib_device_put(ibdev);
 292                 break;
 293         default:
 294                 break;
 295         }
 296
 297         return NOTIFY_DONE;
 298 }
 299
 300 /**
 301  * irdma_netdevice_event - system notifier for netdev events
 302  * @notifier: not used
 303  * @event: event for notifier
 304  * @ptr: netdev
 305  */
 306 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
 307                           void *ptr)
 308 {
 309         struct irdma_device *iwdev;
 310         struct ib_device *ibdev;
 311         struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
 312
 313         ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
 314         if (!ibdev)
 315                 return NOTIFY_DONE;
 316
 317         iwdev = to_iwdev(ibdev);
 318         iwdev->iw_status = 1;
 319         switch (event) {
 320         case NETDEV_DOWN:
 321                 iwdev->iw_status = 0;
 322                 fallthrough;
 323         default:
 324                 break;
 325         }
 326         ib_device_put(ibdev);
 327
 328         return NOTIFY_DONE;
 329 }
 330
 331 /**
 332  * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
 333  * @iwdev: irdma device
 334  */
 335 static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
 336 {
 337         struct net_device *ip_dev;
 338         struct inet6_dev *idev;
 339         struct inet6_ifaddr *ifp, *tmp;
 340         u32 local_ipaddr6[4];
 341
 342         rcu_read_lock();
 343         for_each_netdev_rcu (&init_net, ip_dev) {
 344                 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&
 345                       rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||
 346                       ip_dev == iwdev->netdev) &&
 347                       (READ_ONCE(ip_dev->flags) & IFF_UP)) {
 348                         idev = __in6_dev_get(ip_dev);
 349                         if (!idev) {
 350                                 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");
 351                                 break;
 352                         }
 353                         list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
 354                                                   if_list) {
 355                                 ibdev_dbg(&iwdev->ibdev,
 356                                           "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
 357                                           &ifp->addr,
 358                                           rdma_vlan_dev_vlan_id(ip_dev),
 359                                           ip_dev->dev_addr);
 360
 361                                 irdma_copy_ip_ntohl(local_ipaddr6,
 362                                                     ifp->addr.in6_u.u6_addr32);
 363                                 irdma_manage_arp_cache(iwdev->rf,
 364                                                        ip_dev->dev_addr,
 365                                                        local_ipaddr6, false,
 366                                                        IRDMA_ARP_ADD);
 367                         }
 368                 }
 369         }
 370         rcu_read_unlock();
 371 }
 372
 373 /**
 374  * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
 375  * @iwdev: irdma device
 376  */
 377 static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
 378 {
 379         struct net_device *dev;
 380         struct in_device *idev;
 381         u32 ip_addr;
 382
 383         rcu_read_lock();
 384         for_each_netdev_rcu (&init_net, dev) {
 385                 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
 386                       rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
 387                       dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
 388                         const struct in_ifaddr *ifa;
 389
 390                         idev = __in_dev_get_rcu(dev);
 391                         if (!idev)
 392                                 continue;
 393
 394                         in_dev_for_each_ifa_rcu(ifa, idev) {
 395                                 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
 396                                           &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
 397                                           dev->dev_addr);
 398
 399                                 ip_addr = ntohl(ifa->ifa_address);
 400                                 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,
 401                                                        &ip_addr, true,
 402                                                        IRDMA_ARP_ADD);
 403                         }
 404                 }
 405         }
 406         rcu_read_unlock();
 407 }
 408
 409 /**
 410  * irdma_add_ip - add ip addresses
 411  * @iwdev: irdma device
 412  *
 413  * Add ipv4/ipv6 addresses to the arp cache
 414  */
 415 void irdma_add_ip(struct irdma_device *iwdev)
 416 {
 417         irdma_add_ipv4_addr(iwdev);
 418         irdma_add_ipv6_addr(iwdev);
 419 }
 420
 421 /**
 422  * irdma_alloc_and_get_cqp_request - get cqp struct
 423  * @cqp: device cqp ptr
 424  * @wait: cqp to be used in wait mode
 425  */
 426 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
 427                                                           bool wait)
 428 {
 429         struct irdma_cqp_request *cqp_request = NULL;
 430         unsigned long flags;
 431
 432         spin_lock_irqsave(&cqp->req_lock, flags);
 433         if (!list_empty(&cqp->cqp_avail_reqs)) {
 434                 cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
 435                                                struct irdma_cqp_request, list);
 436                 list_del_init(&cqp_request->list);
 437         }
 438         spin_unlock_irqrestore(&cqp->req_lock, flags);
 439         if (!cqp_request) {
 440                 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
 441                 if (cqp_request) {
 442                         cqp_request->dynamic = true;
 443                         if (wait)
 444                                 init_waitqueue_head(&cqp_request->waitq);
 445                 }
 446         }
 447         if (!cqp_request) {
 448                 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
 449                 return NULL;
 450         }
 451
 452         cqp_request->waiting = wait;
 453         refcount_set(&cqp_request->refcnt, 1);
 454         memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
 455
 456         return cqp_request;
 457 }
 458
 459 /**
 460  * irdma_get_cqp_request - increase refcount for cqp_request
 461  * @cqp_request: pointer to cqp_request instance
 462  */
 463 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
 464 {
 465         refcount_inc(&cqp_request->refcnt);
 466 }
 467
 468 /**
 469  * irdma_free_cqp_request - free cqp request
 470  * @cqp: cqp ptr
 471  * @cqp_request: to be put back in cqp list
 472  */
 473 void irdma_free_cqp_request(struct irdma_cqp *cqp,
 474                             struct irdma_cqp_request *cqp_request)
 475 {
 476         unsigned long flags;
 477
 478         if (cqp_request->dynamic) {
 479                 kfree(cqp_request);
 480         } else {
 481                 WRITE_ONCE(cqp_request->request_done, false);
 482                 cqp_request->callback_fcn = NULL;
 483                 cqp_request->waiting = false;
 484
 485                 spin_lock_irqsave(&cqp->req_lock, flags);
 486                 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
 487                 spin_unlock_irqrestore(&cqp->req_lock, flags);
 488         }
 489         wake_up(&cqp->remove_wq);
 490 }
 491
 492 /**
 493  * irdma_put_cqp_request - dec ref count and free if 0
 494  * @cqp: cqp ptr
 495  * @cqp_request: to be put back in cqp list
 496  */
 497 void irdma_put_cqp_request(struct irdma_cqp *cqp,
 498                            struct irdma_cqp_request *cqp_request)
 499 {
 500         if (refcount_dec_and_test(&cqp_request->refcnt))
 501                 irdma_free_cqp_request(cqp, cqp_request);
 502 }
 503
 504 /**
 505  * irdma_free_pending_cqp_request -free pending cqp request objs
 506  * @cqp: cqp ptr
 507  * @cqp_request: to be put back in cqp list
 508  */
 509 static void
 510 irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
 511                                struct irdma_cqp_request *cqp_request)
 512 {
 513         if (cqp_request->waiting) {
 514                 cqp_request->compl_info.error = true;
 515                 WRITE_ONCE(cqp_request->request_done, true);
 516                 wake_up(&cqp_request->waitq);
 517         }
 518         wait_event_timeout(cqp->remove_wq,
 519                            refcount_read(&cqp_request->refcnt) == 1, 1000);
 520         irdma_put_cqp_request(cqp, cqp_request);
 521 }
 522
 523 /**
 524  * irdma_cleanup_pending_cqp_op - clean-up cqp with no
 525  * completions
 526  * @rf: RDMA PCI function
 527  */
 528 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
 529 {
 530         struct irdma_sc_dev *dev = &rf->sc_dev;
 531         struct irdma_cqp *cqp = &rf->cqp;
 532         struct irdma_cqp_request *cqp_request = NULL;
 533         struct cqp_cmds_info *pcmdinfo = NULL;
 534         u32 i, pending_work, wqe_idx;
 535
 536         pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
 537         wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
 538         for (i = 0; i < pending_work; i++) {
 539                 cqp_request = (struct irdma_cqp_request *)(unsigned long)
 540                                       cqp->scratch_array[wqe_idx];
 541                 if (cqp_request)
 542                         irdma_free_pending_cqp_request(cqp, cqp_request);
 543                 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
 544         }
 545
 546         while (!list_empty(&dev->cqp_cmd_head)) {
 547                 pcmdinfo = irdma_remove_cqp_head(dev);
 548                 cqp_request =
 549                         container_of(pcmdinfo, struct irdma_cqp_request, info);
 550                 if (cqp_request)
 551                         irdma_free_pending_cqp_request(cqp, cqp_request);
 552         }
 553 }
 554
 555 /**
 556  * irdma_wait_event - wait for completion
 557  * @rf: RDMA PCI function
 558  * @cqp_request: cqp request to wait
 559  */
 560 static int irdma_wait_event(struct irdma_pci_f *rf,
 561                             struct irdma_cqp_request *cqp_request)
 562 {
 563         struct irdma_cqp_timeout cqp_timeout = {};
 564         bool cqp_error = false;
 565         int err_code = 0;
 566
 567         cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
 568         do {
 569                 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
 570                 if (wait_event_timeout(cqp_request->waitq,
 571                                        READ_ONCE(cqp_request->request_done),
 572                                        msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
 573                         break;
 574
 575                 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
 576
 577                 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
 578                         continue;
 579
 580                 if (!rf->reset) {
 581                         rf->reset = true;
 582                         rf->gen_ops.request_reset(rf);
 583                 }
 584                 return -ETIMEDOUT;
 585         } while (1);
 586
 587         cqp_error = cqp_request->compl_info.error;
 588         if (cqp_error) {
 589                 err_code = -EIO;
 590                 if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
 591                         if (cqp_request->compl_info.min_err_code == 0x8002)
 592                                 err_code = -EBUSY;
 593                         else if (cqp_request->compl_info.min_err_code == 0x8029) {
 594                                 if (!rf->reset) {
 595                                         rf->reset = true;
 596                                         rf->gen_ops.request_reset(rf);
 597                                 }
 598                         }
 599                 }
 600         }
 601
 602         return err_code;
 603 }
 604
 605 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
 606         [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
 607         [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
 608         [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
 609         [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
 610         [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
 611         [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
 612         [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
 613         [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
 614         [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
 615         [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
 616         [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
 617         [IRDMA_OP_QP_CREATE] = "Create QP Cmd",
 618         [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
 619         [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
 620         [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
 621         [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
 622         [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
 623         [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
 624         [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
 625         [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
 626         [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
 627         [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
 628         [IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
 629         [IRDMA_OP_RESUME] = "Resume QP Cmd",
 630         [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
 631         [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
 632         [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
 633         [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
 634         [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
 635         [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
 636         [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
 637         [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
 638         [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
 639         [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
 640         [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
 641         [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
 642         [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
 643         [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
 644         [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
 645         [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
 646         [IRDMA_OP_GEN_AE] = "Generate AE Cmd",
 647         [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
 648         [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
 649         [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
 650         [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
 651         [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
 652 };
 653
 654 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
 655         {0xffff, 0x8002, "Invalid State"},
 656         {0xffff, 0x8006, "Flush No Wqe Pending"},
 657         {0xffff, 0x8007, "Modify QP Bad Close"},
 658         {0xffff, 0x8009, "LLP Closed"},
 659         {0xffff, 0x800a, "Reset Not Sent"}
 660 };
 661
 662 /**
 663  * irdma_cqp_crit_err - check if CQP error is critical
 664  * @dev: pointer to dev structure
 665  * @cqp_cmd: code for last CQP operation
 666  * @maj_err_code: major error code
 667  * @min_err_code: minot error code
 668  */
 669 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
 670                         u16 maj_err_code, u16 min_err_code)
 671 {
 672         int i;
 673
 674         for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
 675                 if (maj_err_code == irdma_noncrit_err_list[i].maj &&
 676                     min_err_code == irdma_noncrit_err_list[i].min) {
 677                         ibdev_dbg(to_ibdev(dev),
 678                                   "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
 679                                   irdma_noncrit_err_list[i].desc,
 680                                   irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
 681                                   min_err_code);
 682                         return false;
 683                 }
 684         }
 685         return true;
 686 }
 687
 688 /**
 689  * irdma_handle_cqp_op - process cqp command
 690  * @rf: RDMA PCI function
 691  * @cqp_request: cqp request to process
 692  */
 693 int irdma_handle_cqp_op(struct irdma_pci_f *rf,
 694                         struct irdma_cqp_request *cqp_request)
 695 {
 696         struct irdma_sc_dev *dev = &rf->sc_dev;
 697         struct cqp_cmds_info *info = &cqp_request->info;
 698         int status;
 699         bool put_cqp_request = true;
 700
 701         if (rf->reset)
 702                 return -EBUSY;
 703
 704         irdma_get_cqp_request(cqp_request);
 705         status = irdma_process_cqp_cmd(dev, info);
 706         if (status)
 707                 goto err;
 708
 709         if (cqp_request->waiting) {
 710                 put_cqp_request = false;
 711                 status = irdma_wait_event(rf, cqp_request);
 712                 if (status)
 713                         goto err;
 714         }
 715
 716         return 0;
 717
 718 err:
 719         if (irdma_cqp_crit_err(dev, info->cqp_cmd,
 720                                cqp_request->compl_info.maj_err_code,
 721                                cqp_request->compl_info.min_err_code))
 722                 ibdev_err(&rf->iwdev->ibdev,
 723                           "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
 724                           irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
 725                           cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
 726                           cqp_request->compl_info.min_err_code);
 727
 728         if (put_cqp_request)
 729                 irdma_put_cqp_request(&rf->cqp, cqp_request);
 730
 731         return status;
 732 }
 733
 734 void irdma_qp_add_ref(struct ib_qp *ibqp)
 735 {
 736         struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
 737
 738         refcount_inc(&iwqp->refcnt);
 739 }
 740
 741 void irdma_qp_rem_ref(struct ib_qp *ibqp)
 742 {
 743         struct irdma_qp *iwqp = to_iwqp(ibqp);
 744         struct irdma_device *iwdev = iwqp->iwdev;
 745         u32 qp_num;
 746         unsigned long flags;
 747
 748         spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
 749         if (!refcount_dec_and_test(&iwqp->refcnt)) {
 750                 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
 751                 return;
 752         }
 753
 754         qp_num = iwqp->ibqp.qp_num;
 755         iwdev->rf->qp_table[qp_num] = NULL;
 756         spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
 757         complete(&iwqp->free_qp);
 758 }
 759
 760 void irdma_cq_add_ref(struct ib_cq *ibcq)
 761 {
 762         struct irdma_cq *iwcq = to_iwcq(ibcq);
 763
 764         refcount_inc(&iwcq->refcnt);
 765 }
 766
 767 void irdma_cq_rem_ref(struct ib_cq *ibcq)
 768 {
 769         struct ib_device *ibdev = ibcq->device;
 770         struct irdma_device *iwdev = to_iwdev(ibdev);
 771         struct irdma_cq *iwcq = to_iwcq(ibcq);
 772         unsigned long flags;
 773
 774         spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags);
 775         if (!refcount_dec_and_test(&iwcq->refcnt)) {
 776                 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
 777                 return;
 778         }
 779
 780         iwdev->rf->cq_table[iwcq->cq_num] = NULL;
 781         spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
 782         complete(&iwcq->free_cq);
 783 }
 784
 785 struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
 786 {
 787         return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
 788 }
 789
 790 /**
 791  * irdma_get_qp - get qp address
 792  * @device: iwarp device
 793  * @qpn: qp number
 794  */
 795 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
 796 {
 797         struct irdma_device *iwdev = to_iwdev(device);
 798
 799         if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
 800                 return NULL;
 801
 802         return &iwdev->rf->qp_table[qpn]->ibqp;
 803 }
 804
 805 /**
 806  * irdma_remove_cqp_head - return head entry and remove
 807  * @dev: device
 808  */
 809 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
 810 {
 811         struct list_head *entry;
 812         struct list_head *list = &dev->cqp_cmd_head;
 813
 814         if (list_empty(list))
 815                 return NULL;
 816
 817         entry = list->next;
 818         list_del(entry);
 819
 820         return entry;
 821 }
 822
 823 /**
 824  * irdma_cqp_sds_cmd - create cqp command for sd
 825  * @dev: hardware control device structure
 826  * @sdinfo: information for sd cqp
 827  *
 828  */
 829 int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
 830                       struct irdma_update_sds_info *sdinfo)
 831 {
 832         struct irdma_cqp_request *cqp_request;
 833         struct cqp_cmds_info *cqp_info;
 834         struct irdma_pci_f *rf = dev_to_rf(dev);
 835         int status;
 836
 837         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
 838         if (!cqp_request)
 839                 return -ENOMEM;
 840
 841         cqp_info = &cqp_request->info;
 842         memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
 843                sizeof(cqp_info->in.u.update_pe_sds.info));
 844         cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
 845         cqp_info->post_sq = 1;
 846         cqp_info->in.u.update_pe_sds.dev = dev;
 847         cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
 848
 849         status = irdma_handle_cqp_op(rf, cqp_request);
 850         irdma_put_cqp_request(&rf->cqp, cqp_request);
 851
 852         return status;
 853 }
 854
 855 /**
 856  * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
 857  * @qp: hardware control qp
 858  * @op: suspend or resume
 859  */
 860 int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
 861 {
 862         struct irdma_sc_dev *dev = qp->dev;
 863         struct irdma_cqp_request *cqp_request;
 864         struct irdma_sc_cqp *cqp = dev->cqp;
 865         struct cqp_cmds_info *cqp_info;
 866         struct irdma_pci_f *rf = dev_to_rf(dev);
 867         int status;
 868
 869         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
 870         if (!cqp_request)
 871                 return -ENOMEM;
 872
 873         cqp_info = &cqp_request->info;
 874         cqp_info->cqp_cmd = op;
 875         cqp_info->in.u.suspend_resume.cqp = cqp;
 876         cqp_info->in.u.suspend_resume.qp = qp;
 877         cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
 878
 879         status = irdma_handle_cqp_op(rf, cqp_request);
 880         irdma_put_cqp_request(&rf->cqp, cqp_request);
 881
 882         return status;
 883 }
 884
 885 /**
 886  * irdma_term_modify_qp - modify qp for term message
 887  * @qp: hardware control qp
 888  * @next_state: qp's next state
 889  * @term: terminate code
 890  * @term_len: length
 891  */
 892 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
 893                           u8 term_len)
 894 {
 895         struct irdma_qp *iwqp;
 896
 897         iwqp = qp->qp_uk.back_qp;
 898         irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
 899 };
 900
 901 /**
 902  * irdma_terminate_done - after terminate is completed
 903  * @qp: hardware control qp
 904  * @timeout_occurred: indicates if terminate timer expired
 905  */
 906 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
 907 {
 908         struct irdma_qp *iwqp;
 909         u8 hte = 0;
 910         bool first_time;
 911         unsigned long flags;
 912
 913         iwqp = qp->qp_uk.back_qp;
 914         spin_lock_irqsave(&iwqp->lock, flags);
 915         if (iwqp->hte_added) {
 916                 iwqp->hte_added = 0;
 917                 hte = 1;
 918         }
 919         first_time = !(qp->term_flags & IRDMA_TERM_DONE);
 920         qp->term_flags |= IRDMA_TERM_DONE;
 921         spin_unlock_irqrestore(&iwqp->lock, flags);
 922         if (first_time) {
 923                 if (!timeout_occurred)
 924                         irdma_terminate_del_timer(qp);
 925
 926                 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
 927                 irdma_cm_disconn(iwqp);
 928         }
 929 }
 930
 931 static void irdma_terminate_timeout(struct timer_list *t)
 932 {
 933         struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
 934         struct irdma_sc_qp *qp = &iwqp->sc_qp;
 935
 936         irdma_terminate_done(qp, 1);
 937         irdma_qp_rem_ref(&iwqp->ibqp);
 938 }
 939
 940 /**
 941  * irdma_terminate_start_timer - start terminate timeout
 942  * @qp: hardware control qp
 943  */
 944 void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
 945 {
 946         struct irdma_qp *iwqp;
 947
 948         iwqp = qp->qp_uk.back_qp;
 949         irdma_qp_add_ref(&iwqp->ibqp);
 950         timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
 951         iwqp->terminate_timer.expires = jiffies + HZ;
 952
 953         add_timer(&iwqp->terminate_timer);
 954 }
 955
 956 /**
 957  * irdma_terminate_del_timer - delete terminate timeout
 958  * @qp: hardware control qp
 959  */
 960 void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
 961 {
 962         struct irdma_qp *iwqp;
 963         int ret;
 964
 965         iwqp = qp->qp_uk.back_qp;
 966         ret = del_timer(&iwqp->terminate_timer);
 967         if (ret)
 968                 irdma_qp_rem_ref(&iwqp->ibqp);
 969 }
 970
 971 /**
 972  * irdma_cqp_cq_create_cmd - create a cq for the cqp
 973  * @dev: device pointer
 974  * @cq: pointer to created cq
 975  */
 976 int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
 977 {
 978         struct irdma_pci_f *rf = dev_to_rf(dev);
 979         struct irdma_cqp *iwcqp = &rf->cqp;
 980         struct irdma_cqp_request *cqp_request;
 981         struct cqp_cmds_info *cqp_info;
 982         int status;
 983
 984         cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
 985         if (!cqp_request)
 986                 return -ENOMEM;
 987
 988         cqp_info = &cqp_request->info;
 989         cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
 990         cqp_info->post_sq = 1;
 991         cqp_info->in.u.cq_create.cq = cq;
 992         cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
 993
 994         status = irdma_handle_cqp_op(rf, cqp_request);
 995         irdma_put_cqp_request(iwcqp, cqp_request);
 996
 997         return status;
 998 }
 999
1000 /**
1001  * irdma_cqp_qp_create_cmd - create a qp for the cqp
1002  * @dev: device pointer
1003  * @qp: pointer to created qp
1004  */
1005 int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1006 {
1007         struct irdma_pci_f *rf = dev_to_rf(dev);
1008         struct irdma_cqp *iwcqp = &rf->cqp;
1009         struct irdma_cqp_request *cqp_request;
1010         struct cqp_cmds_info *cqp_info;
1011         struct irdma_create_qp_info *qp_info;
1012         int status;
1013
1014         cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1015         if (!cqp_request)
1016                 return -ENOMEM;
1017
1018         cqp_info = &cqp_request->info;
1019         qp_info = &cqp_request->info.in.u.qp_create.info;
1020         memset(qp_info, 0, sizeof(*qp_info));
1021         qp_info->cq_num_valid = true;
1022         qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1023         cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1024         cqp_info->post_sq = 1;
1025         cqp_info->in.u.qp_create.qp = qp;
1026         cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1027
1028         status = irdma_handle_cqp_op(rf, cqp_request);
1029         irdma_put_cqp_request(iwcqp, cqp_request);
1030
1031         return status;
1032 }
1033
1034 /**
1035  * irdma_dealloc_push_page - free a push page for qp
1036  * @rf: RDMA PCI function
1037  * @qp: hardware control qp
1038  */
1039 static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1040                                     struct irdma_sc_qp *qp)
1041 {
1042         struct irdma_cqp_request *cqp_request;
1043         struct cqp_cmds_info *cqp_info;
1044         int status;
1045
1046         if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1047                 return;
1048
1049         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1050         if (!cqp_request)
1051                 return;
1052
1053         cqp_info = &cqp_request->info;
1054         cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1055         cqp_info->post_sq = 1;
1056         cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1057         cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1058         cqp_info->in.u.manage_push_page.info.free_page = 1;
1059         cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1060         cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1061         cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1062         status = irdma_handle_cqp_op(rf, cqp_request);
1063         if (!status)
1064                 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1065         irdma_put_cqp_request(&rf->cqp, cqp_request);
1066 }
1067
1068 /**
1069  * irdma_free_qp_rsrc - free up memory resources for qp
1070  * @iwqp: qp ptr (user or kernel)
1071  */
1072 void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1073 {
1074         struct irdma_device *iwdev = iwqp->iwdev;
1075         struct irdma_pci_f *rf = iwdev->rf;
1076         u32 qp_num = iwqp->ibqp.qp_num;
1077
1078         irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
1079         irdma_dealloc_push_page(rf, &iwqp->sc_qp);
1080         if (iwqp->sc_qp.vsi) {
1081                 irdma_qp_rem_qos(&iwqp->sc_qp);
1082                 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1083                                            iwqp->sc_qp.user_pri);
1084         }
1085
1086         if (qp_num > 2)
1087                 irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1088         dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,
1089                           iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);
1090         iwqp->q2_ctx_mem.va = NULL;
1091         dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,
1092                           iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);
1093         iwqp->kqp.dma_mem.va = NULL;
1094         kfree(iwqp->kqp.sq_wrid_mem);
1095         kfree(iwqp->kqp.rq_wrid_mem);
1096 }
1097
1098 /**
1099  * irdma_cq_wq_destroy - send cq destroy cqp
1100  * @rf: RDMA PCI function
1101  * @cq: hardware control cq
1102  */
1103 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1104 {
1105         struct irdma_cqp_request *cqp_request;
1106         struct cqp_cmds_info *cqp_info;
1107
1108         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1109         if (!cqp_request)
1110                 return;
1111
1112         cqp_info = &cqp_request->info;
1113         cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1114         cqp_info->post_sq = 1;
1115         cqp_info->in.u.cq_destroy.cq = cq;
1116         cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1117
1118         irdma_handle_cqp_op(rf, cqp_request);
1119         irdma_put_cqp_request(&rf->cqp, cqp_request);
1120 }
1121
1122 /**
1123  * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1124  * @cqp_request: modify QP completion
1125  */
1126 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1127 {
1128         struct cqp_cmds_info *cqp_info;
1129         struct irdma_qp *iwqp;
1130
1131         cqp_info = &cqp_request->info;
1132         iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1133         atomic_dec(&iwqp->hw_mod_qp_pend);
1134         wake_up(&iwqp->mod_qp_waitq);
1135 }
1136
1137 /**
1138  * irdma_hw_modify_qp - setup cqp for modify qp
1139  * @iwdev: RDMA device
1140  * @iwqp: qp ptr (user or kernel)
1141  * @info: info for modify qp
1142  * @wait: flag to wait or not for modify qp completion
1143  */
1144 int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1145                        struct irdma_modify_qp_info *info, bool wait)
1146 {
1147         int status;
1148         struct irdma_pci_f *rf = iwdev->rf;
1149         struct irdma_cqp_request *cqp_request;
1150         struct cqp_cmds_info *cqp_info;
1151         struct irdma_modify_qp_info *m_info;
1152
1153         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1154         if (!cqp_request)
1155                 return -ENOMEM;
1156
1157         if (!wait) {
1158                 cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1159                 atomic_inc(&iwqp->hw_mod_qp_pend);
1160         }
1161         cqp_info = &cqp_request->info;
1162         m_info = &cqp_info->in.u.qp_modify.info;
1163         memcpy(m_info, info, sizeof(*m_info));
1164         cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1165         cqp_info->post_sq = 1;
1166         cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1167         cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1168         status = irdma_handle_cqp_op(rf, cqp_request);
1169         irdma_put_cqp_request(&rf->cqp, cqp_request);
1170         if (status) {
1171                 if (rdma_protocol_roce(&iwdev->ibdev, 1))
1172                         return status;
1173
1174                 switch (m_info->next_iwarp_state) {
1175                         struct irdma_gen_ae_info ae_info;
1176
1177                 case IRDMA_QP_STATE_RTS:
1178                 case IRDMA_QP_STATE_IDLE:
1179                 case IRDMA_QP_STATE_TERMINATE:
1180                 case IRDMA_QP_STATE_CLOSING:
1181                         if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1182                                 irdma_send_reset(iwqp->cm_node);
1183                         else
1184                                 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1185                         if (!wait) {
1186                                 ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1187                                 ae_info.ae_src = 0;
1188                                 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1189                         } else {
1190                                 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1191                                                                               wait);
1192                                 if (!cqp_request)
1193                                         return -ENOMEM;
1194
1195                                 cqp_info = &cqp_request->info;
1196                                 m_info = &cqp_info->in.u.qp_modify.info;
1197                                 memcpy(m_info, info, sizeof(*m_info));
1198                                 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1199                                 cqp_info->post_sq = 1;
1200                                 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1201                                 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1202                                 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1203                                 m_info->reset_tcp_conn = true;
1204                                 irdma_handle_cqp_op(rf, cqp_request);
1205                                 irdma_put_cqp_request(&rf->cqp, cqp_request);
1206                         }
1207                         break;
1208                 case IRDMA_QP_STATE_ERROR:
1209                 default:
1210                         break;
1211                 }
1212         }
1213
1214         return status;
1215 }
1216
1217 /**
1218  * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1219  * @dev: device pointer
1220  * @cq: pointer to cq
1221  */
1222 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1223 {
1224         struct irdma_pci_f *rf = dev_to_rf(dev);
1225
1226         irdma_cq_wq_destroy(rf, cq);
1227 }
1228
1229 /**
1230  * irdma_cqp_qp_destroy_cmd - destroy the cqp
1231  * @dev: device pointer
1232  * @qp: pointer to qp
1233  */
1234 int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1235 {
1236         struct irdma_pci_f *rf = dev_to_rf(dev);
1237         struct irdma_cqp *iwcqp = &rf->cqp;
1238         struct irdma_cqp_request *cqp_request;
1239         struct cqp_cmds_info *cqp_info;
1240         int status;
1241
1242         cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1243         if (!cqp_request)
1244                 return -ENOMEM;
1245
1246         cqp_info = &cqp_request->info;
1247         memset(cqp_info, 0, sizeof(*cqp_info));
1248         cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1249         cqp_info->post_sq = 1;
1250         cqp_info->in.u.qp_destroy.qp = qp;
1251         cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1252         cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1253
1254         status = irdma_handle_cqp_op(rf, cqp_request);
1255         irdma_put_cqp_request(&rf->cqp, cqp_request);
1256
1257         return status;
1258 }
1259
1260 /**
1261  * irdma_ieq_mpa_crc_ae - generate AE for crc error
1262  * @dev: hardware control device structure
1263  * @qp: hardware control qp
1264  */
1265 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1266 {
1267         struct irdma_gen_ae_info info = {};
1268         struct irdma_pci_f *rf = dev_to_rf(dev);
1269
1270         ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1271         info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1272         info.ae_src = IRDMA_AE_SOURCE_RQ;
1273         irdma_gen_ae(rf, qp, &info, false);
1274 }
1275
1276 /**
1277  * irdma_init_hash_desc - initialize hash for crc calculation
1278  * @desc: cryption type
1279  */
1280 int irdma_init_hash_desc(struct shash_desc **desc)
1281 {
1282         struct crypto_shash *tfm;
1283         struct shash_desc *tdesc;
1284
1285         tfm = crypto_alloc_shash("crc32c", 0, 0);
1286         if (IS_ERR(tfm))
1287                 return -EINVAL;
1288
1289         tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
1290                         GFP_KERNEL);
1291         if (!tdesc) {
1292                 crypto_free_shash(tfm);
1293                 return -EINVAL;
1294         }
1295
1296         tdesc->tfm = tfm;
1297         *desc = tdesc;
1298
1299         return 0;
1300 }
1301
1302 /**
1303  * irdma_free_hash_desc - free hash desc
1304  * @desc: to be freed
1305  */
1306 void irdma_free_hash_desc(struct shash_desc *desc)
1307 {
1308         if (desc) {
1309                 crypto_free_shash(desc->tfm);
1310                 kfree(desc);
1311         }
1312 }
1313
1314 /**
1315  * irdma_ieq_check_mpacrc - check if mpa crc is OK
1316  * @desc: desc for hash
1317  * @addr: address of buffer for crc
1318  * @len: length of buffer
1319  * @val: value to be compared
1320  */
1321 int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,
1322                            u32 val)
1323 {
1324         u32 crc = 0;
1325
1326         crypto_shash_digest(desc, addr, len, (u8 *)&crc);
1327         if (crc != val)
1328                 return -EINVAL;
1329
1330         return 0;
1331 }
1332
1333 /**
1334  * irdma_ieq_get_qp - get qp based on quad in puda buffer
1335  * @dev: hardware control device structure
1336  * @buf: receive puda buffer on exception q
1337  */
1338 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1339                                      struct irdma_puda_buf *buf)
1340 {
1341         struct irdma_qp *iwqp;
1342         struct irdma_cm_node *cm_node;
1343         struct irdma_device *iwdev = buf->vsi->back_vsi;
1344         u32 loc_addr[4] = {};
1345         u32 rem_addr[4] = {};
1346         u16 loc_port, rem_port;
1347         struct ipv6hdr *ip6h;
1348         struct iphdr *iph = (struct iphdr *)buf->iph;
1349         struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1350
1351         if (iph->version == 4) {
1352                 loc_addr[0] = ntohl(iph->daddr);
1353                 rem_addr[0] = ntohl(iph->saddr);
1354         } else {
1355                 ip6h = (struct ipv6hdr *)buf->iph;
1356                 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
1357                 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
1358         }
1359         loc_port = ntohs(tcph->dest);
1360         rem_port = ntohs(tcph->source);
1361         cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1362                                   loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1363         if (!cm_node)
1364                 return NULL;
1365
1366         iwqp = cm_node->iwqp;
1367         irdma_rem_ref_cm_node(cm_node);
1368
1369         return &iwqp->sc_qp;
1370 }
1371
1372 /**
1373  * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1374  * @qp: qp ptr
1375  */
1376 void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1377 {
1378         struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1379         struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1380         struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1381
1382         cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1383         cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1384
1385         irdma_send_ack(cm_node);
1386 }
1387
1388 /**
1389  * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1390  * @qp: qp pointer
1391  * @ah_info: AH info pointer
1392  */
1393 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1394                                 struct irdma_ah_info *ah_info)
1395 {
1396         struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1397         struct iphdr *iph;
1398         struct ipv6hdr *ip6h;
1399
1400         memset(ah_info, 0, sizeof(*ah_info));
1401         ah_info->do_lpbk = true;
1402         ah_info->vlan_tag = buf->vlan_id;
1403         ah_info->insert_vlan_tag = buf->vlan_valid;
1404         ah_info->ipv4_valid = buf->ipv4;
1405         ah_info->vsi = qp->vsi;
1406
1407         if (buf->smac_valid)
1408                 ether_addr_copy(ah_info->mac_addr, buf->smac);
1409
1410         if (buf->ipv4) {
1411                 ah_info->ipv4_valid = true;
1412                 iph = (struct iphdr *)buf->iph;
1413                 ah_info->hop_ttl = iph->ttl;
1414                 ah_info->tc_tos = iph->tos;
1415                 ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1416                 ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1417         } else {
1418                 ip6h = (struct ipv6hdr *)buf->iph;
1419                 ah_info->hop_ttl = ip6h->hop_limit;
1420                 ah_info->tc_tos = ip6h->priority;
1421                 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1422                                     ip6h->daddr.in6_u.u6_addr32);
1423                 irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1424                                     ip6h->saddr.in6_u.u6_addr32);
1425         }
1426
1427         ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1428                                                 ah_info->dest_ip_addr,
1429                                                 ah_info->ipv4_valid,
1430                                                 NULL, IRDMA_ARP_RESOLVE);
1431 }
1432
1433 /**
1434  * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1435  * @buf: puda to update
1436  * @len: length of buffer
1437  * @seqnum: seq number for tcp
1438  */
1439 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1440                                              u16 len, u32 seqnum)
1441 {
1442         struct tcphdr *tcph;
1443         struct iphdr *iph;
1444         u16 iphlen;
1445         u16 pktsize;
1446         u8 *addr = buf->mem.va;
1447
1448         iphlen = (buf->ipv4) ? 20 : 40;
1449         iph = (struct iphdr *)(addr + buf->maclen);
1450         tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1451         pktsize = len + buf->tcphlen + iphlen;
1452         iph->tot_len = htons(pktsize);
1453         tcph->seq = htonl(seqnum);
1454 }
1455
1456 /**
1457  * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1458  * @buf: puda to update
1459  * @len: length of buffer
1460  * @seqnum: seq number for tcp
1461  */
1462 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1463                                  u32 seqnum)
1464 {
1465         struct tcphdr *tcph;
1466         u8 *addr;
1467
1468         if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1469                 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1470
1471         addr = buf->mem.va;
1472         tcph = (struct tcphdr *)addr;
1473         tcph->seq = htonl(seqnum);
1474 }
1475
1476 /**
1477  * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1478  * buffer
1479  * @info: to get information
1480  * @buf: puda buffer
1481  */
1482 static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1483                                           struct irdma_puda_buf *buf)
1484 {
1485         struct iphdr *iph;
1486         struct ipv6hdr *ip6h;
1487         struct tcphdr *tcph;
1488         u16 iphlen;
1489         u16 pkt_len;
1490         u8 *mem = buf->mem.va;
1491         struct ethhdr *ethh = buf->mem.va;
1492
1493         if (ethh->h_proto == htons(0x8100)) {
1494                 info->vlan_valid = true;
1495                 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1496                                VLAN_VID_MASK;
1497         }
1498
1499         buf->maclen = (info->vlan_valid) ? 18 : 14;
1500         iphlen = (info->l3proto) ? 40 : 20;
1501         buf->ipv4 = (info->l3proto) ? false : true;
1502         buf->iph = mem + buf->maclen;
1503         iph = (struct iphdr *)buf->iph;
1504         buf->tcph = buf->iph + iphlen;
1505         tcph = (struct tcphdr *)buf->tcph;
1506
1507         if (buf->ipv4) {
1508                 pkt_len = ntohs(iph->tot_len);
1509         } else {
1510                 ip6h = (struct ipv6hdr *)buf->iph;
1511                 pkt_len = ntohs(ip6h->payload_len) + iphlen;
1512         }
1513
1514         buf->totallen = pkt_len + buf->maclen;
1515
1516         if (info->payload_len < buf->totallen) {
1517                 ibdev_dbg(to_ibdev(buf->vsi->dev),
1518                           "ERR: payload_len = 0x%x totallen expected0x%x\n",
1519                           info->payload_len, buf->totallen);
1520                 return -EINVAL;
1521         }
1522
1523         buf->tcphlen = tcph->doff << 2;
1524         buf->datalen = pkt_len - iphlen - buf->tcphlen;
1525         buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1526         buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1527         buf->seqnum = ntohl(tcph->seq);
1528
1529         return 0;
1530 }
1531
1532 /**
1533  * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1534  * @info: to get information
1535  * @buf: puda buffer
1536  */
1537 int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1538                               struct irdma_puda_buf *buf)
1539 {
1540         struct tcphdr *tcph;
1541         u32 pkt_len;
1542         u8 *mem;
1543
1544         if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1545                 return irdma_gen1_puda_get_tcpip_info(info, buf);
1546
1547         mem = buf->mem.va;
1548         buf->vlan_valid = info->vlan_valid;
1549         if (info->vlan_valid)
1550                 buf->vlan_id = info->vlan;
1551
1552         buf->ipv4 = info->ipv4;
1553         if (buf->ipv4)
1554                 buf->iph = mem + IRDMA_IPV4_PAD;
1555         else
1556                 buf->iph = mem;
1557
1558         buf->tcph = mem + IRDMA_TCP_OFFSET;
1559         tcph = (struct tcphdr *)buf->tcph;
1560         pkt_len = info->payload_len;
1561         buf->totallen = pkt_len;
1562         buf->tcphlen = tcph->doff << 2;
1563         buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1564         buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1565         buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1566         buf->seqnum = ntohl(tcph->seq);
1567
1568         if (info->smac_valid) {
1569                 ether_addr_copy(buf->smac, info->smac);
1570                 buf->smac_valid = true;
1571         }
1572
1573         return 0;
1574 }
1575
1576 /**
1577  * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1578  * @t: timer_list pointer
1579  */
1580 static void irdma_hw_stats_timeout(struct timer_list *t)
1581 {
1582         struct irdma_vsi_pestat *pf_devstat =
1583                 from_timer(pf_devstat, t, stats_timer);
1584         struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1585
1586         if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1587                 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1588         else
1589                 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
1590
1591         mod_timer(&pf_devstat->stats_timer,
1592                   jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1593 }
1594
1595 /**
1596  * irdma_hw_stats_start_timer - Start periodic stats timer
1597  * @vsi: vsi structure pointer
1598  */
1599 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1600 {
1601         struct irdma_vsi_pestat *devstat = vsi->pestat;
1602
1603         timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1604         mod_timer(&devstat->stats_timer,
1605                   jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1606 }
1607
1608 /**
1609  * irdma_hw_stats_stop_timer - Delete periodic stats timer
1610  * @vsi: pointer to vsi structure
1611  */
1612 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1613 {
1614         struct irdma_vsi_pestat *devstat = vsi->pestat;
1615
1616         del_timer_sync(&devstat->stats_timer);
1617 }
1618
1619 /**
1620  * irdma_process_stats - Checking for wrap and update stats
1621  * @pestat: stats structure pointer
1622  */
1623 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1624 {
1625         sc_vsi_update_stats(pestat->vsi);
1626 }
1627
1628 /**
1629  * irdma_cqp_gather_stats_gen1 - Gather stats
1630  * @dev: pointer to device structure
1631  * @pestat: statistics structure
1632  */
1633 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1634                                  struct irdma_vsi_pestat *pestat)
1635 {
1636         struct irdma_gather_stats *gather_stats =
1637                 pestat->gather_info.gather_stats_va;
1638         const struct irdma_hw_stat_map *map = dev->hw_stats_map;
1639         u16 max_stats_idx = dev->hw_attrs.max_stat_idx;
1640         u32 stats_inst_offset_32;
1641         u32 stats_inst_offset_64;
1642         u64 new_val;
1643         u16 i;
1644
1645         stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1646                                 pestat->gather_info.stats_inst_index :
1647                                 pestat->hw->hmc.hmc_fn_id;
1648         stats_inst_offset_32 *= 4;
1649         stats_inst_offset_64 = stats_inst_offset_32 * 2;
1650
1651         for (i = 0; i < max_stats_idx; i++) {
1652                 if (map[i].bitmask <= IRDMA_MAX_STATS_32)
1653                         new_val = rd32(dev->hw,
1654                                        dev->hw_stats_regs[i] + stats_inst_offset_32);
1655                 else
1656                         new_val = rd64(dev->hw,
1657                                        dev->hw_stats_regs[i] + stats_inst_offset_64);
1658                 gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val;
1659         }
1660
1661         irdma_process_stats(pestat);
1662 }
1663
1664 /**
1665  * irdma_process_cqp_stats - Checking for wrap and update stats
1666  * @cqp_request: cqp_request structure pointer
1667  */
1668 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1669 {
1670         struct irdma_vsi_pestat *pestat = cqp_request->param;
1671
1672         irdma_process_stats(pestat);
1673 }
1674
1675 /**
1676  * irdma_cqp_gather_stats_cmd - Gather stats
1677  * @dev: pointer to device structure
1678  * @pestat: pointer to stats info
1679  * @wait: flag to wait or not wait for stats
1680  */
1681 int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1682                                struct irdma_vsi_pestat *pestat, bool wait)
1683
1684 {
1685         struct irdma_pci_f *rf = dev_to_rf(dev);
1686         struct irdma_cqp *iwcqp = &rf->cqp;
1687         struct irdma_cqp_request *cqp_request;
1688         struct cqp_cmds_info *cqp_info;
1689         int status;
1690
1691         cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1692         if (!cqp_request)
1693                 return -ENOMEM;
1694
1695         cqp_info = &cqp_request->info;
1696         memset(cqp_info, 0, sizeof(*cqp_info));
1697         cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1698         cqp_info->post_sq = 1;
1699         cqp_info->in.u.stats_gather.info = pestat->gather_info;
1700         cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1701         cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1702         cqp_request->param = pestat;
1703         if (!wait)
1704                 cqp_request->callback_fcn = irdma_process_cqp_stats;
1705         status = irdma_handle_cqp_op(rf, cqp_request);
1706         if (wait)
1707                 irdma_process_stats(pestat);
1708         irdma_put_cqp_request(&rf->cqp, cqp_request);
1709
1710         return status;
1711 }
1712
1713 /**
1714  * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1715  * @vsi: pointer to vsi structure
1716  * @cmd: command to allocate or free
1717  * @stats_info: pointer to allocate stats info
1718  */
1719 int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1720                              struct irdma_stats_inst_info *stats_info)
1721 {
1722         struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1723         struct irdma_cqp *iwcqp = &rf->cqp;
1724         struct irdma_cqp_request *cqp_request;
1725         struct cqp_cmds_info *cqp_info;
1726         int status;
1727         bool wait = false;
1728
1729         if (cmd == IRDMA_OP_STATS_ALLOCATE)
1730                 wait = true;
1731         cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1732         if (!cqp_request)
1733                 return -ENOMEM;
1734
1735         cqp_info = &cqp_request->info;
1736         memset(cqp_info, 0, sizeof(*cqp_info));
1737         cqp_info->cqp_cmd = cmd;
1738         cqp_info->post_sq = 1;
1739         cqp_info->in.u.stats_manage.info = *stats_info;
1740         cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1741         cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1742         status = irdma_handle_cqp_op(rf, cqp_request);
1743         if (wait)
1744                 stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1745         irdma_put_cqp_request(iwcqp, cqp_request);
1746
1747         return status;
1748 }
1749
1750 /**
1751  * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1752  * @dev: pointer to device info
1753  * @sc_ceq: pointer to ceq structure
1754  * @op: Create or Destroy
1755  */
1756 int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1757                       u8 op)
1758 {
1759         struct irdma_cqp_request *cqp_request;
1760         struct cqp_cmds_info *cqp_info;
1761         struct irdma_pci_f *rf = dev_to_rf(dev);
1762         int status;
1763
1764         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1765         if (!cqp_request)
1766                 return -ENOMEM;
1767
1768         cqp_info = &cqp_request->info;
1769         cqp_info->post_sq = 1;
1770         cqp_info->cqp_cmd = op;
1771         cqp_info->in.u.ceq_create.ceq = sc_ceq;
1772         cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1773
1774         status = irdma_handle_cqp_op(rf, cqp_request);
1775         irdma_put_cqp_request(&rf->cqp, cqp_request);
1776
1777         return status;
1778 }
1779
1780 /**
1781  * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1782  * @dev: pointer to device info
1783  * @sc_aeq: pointer to aeq structure
1784  * @op: Create or Destroy
1785  */
1786 int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1787                       u8 op)
1788 {
1789         struct irdma_cqp_request *cqp_request;
1790         struct cqp_cmds_info *cqp_info;
1791         struct irdma_pci_f *rf = dev_to_rf(dev);
1792         int status;
1793
1794         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1795         if (!cqp_request)
1796                 return -ENOMEM;
1797
1798         cqp_info = &cqp_request->info;
1799         cqp_info->post_sq = 1;
1800         cqp_info->cqp_cmd = op;
1801         cqp_info->in.u.aeq_create.aeq = sc_aeq;
1802         cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1803
1804         status = irdma_handle_cqp_op(rf, cqp_request);
1805         irdma_put_cqp_request(&rf->cqp, cqp_request);
1806
1807         return status;
1808 }
1809
1810 /**
1811  * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1812  * @dev: pointer to device structure
1813  * @cmd: Add, modify or delete
1814  * @node_info: pointer to ws node info
1815  */
1816 int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
1817                           struct irdma_ws_node_info *node_info)
1818 {
1819         struct irdma_pci_f *rf = dev_to_rf(dev);
1820         struct irdma_cqp *iwcqp = &rf->cqp;
1821         struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
1822         struct irdma_cqp_request *cqp_request;
1823         struct cqp_cmds_info *cqp_info;
1824         int status;
1825         bool poll;
1826
1827         if (!rf->sc_dev.ceq_valid)
1828                 poll = true;
1829         else
1830                 poll = false;
1831
1832         cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
1833         if (!cqp_request)
1834                 return -ENOMEM;
1835
1836         cqp_info = &cqp_request->info;
1837         memset(cqp_info, 0, sizeof(*cqp_info));
1838         cqp_info->cqp_cmd = cmd;
1839         cqp_info->post_sq = 1;
1840         cqp_info->in.u.ws_node.info = *node_info;
1841         cqp_info->in.u.ws_node.cqp = cqp;
1842         cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
1843         status = irdma_handle_cqp_op(rf, cqp_request);
1844         if (status)
1845                 goto exit;
1846
1847         if (poll) {
1848                 struct irdma_ccq_cqe_info compl_info;
1849
1850                 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
1851                                                        &compl_info);
1852                 node_info->qs_handle = compl_info.op_ret_val;
1853                 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
1854                           compl_info.op_code, compl_info.op_ret_val);
1855         } else {
1856                 node_info->qs_handle = cqp_request->compl_info.op_ret_val;
1857         }
1858
1859 exit:
1860         irdma_put_cqp_request(&rf->cqp, cqp_request);
1861
1862         return status;
1863 }
1864
1865 /**
1866  * irdma_ah_cqp_op - perform an AH cqp operation
1867  * @rf: RDMA PCI function
1868  * @sc_ah: address handle
1869  * @cmd: AH operation
1870  * @wait: wait if true
1871  * @callback_fcn: Callback function on CQP op completion
1872  * @cb_param: parameter for callback function
1873  *
1874  * returns errno
1875  */
1876 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
1877                     bool wait,
1878                     void (*callback_fcn)(struct irdma_cqp_request *),
1879                     void *cb_param)
1880 {
1881         struct irdma_cqp_request *cqp_request;
1882         struct cqp_cmds_info *cqp_info;
1883         int status;
1884
1885         if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
1886                 return -EINVAL;
1887
1888         cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1889         if (!cqp_request)
1890                 return -ENOMEM;
1891
1892         cqp_info = &cqp_request->info;
1893         cqp_info->cqp_cmd = cmd;
1894         cqp_info->post_sq = 1;
1895         if (cmd == IRDMA_OP_AH_CREATE) {
1896                 cqp_info->in.u.ah_create.info = sc_ah->ah_info;
1897                 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
1898                 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
1899         } else if (cmd == IRDMA_OP_AH_DESTROY) {
1900                 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
1901                 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
1902                 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
1903         }
1904
1905         if (!wait) {
1906                 cqp_request->callback_fcn = callback_fcn;
1907                 cqp_request->param = cb_param;
1908         }
1909         status = irdma_handle_cqp_op(rf, cqp_request);
1910         irdma_put_cqp_request(&rf->cqp, cqp_request);
1911
1912         if (status)
1913                 return -ENOMEM;
1914
1915         if (wait)
1916                 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
1917
1918         return 0;
1919 }
1920
1921 /**
1922  * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1923  * @cqp_request: pointer to cqp_request of create AH
1924  */
1925 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
1926 {
1927         struct irdma_sc_qp *qp = cqp_request->param;
1928         struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
1929         unsigned long flags;
1930
1931         spin_lock_irqsave(&qp->pfpdu.lock, flags);
1932         if (!cqp_request->compl_info.op_ret_val) {
1933                 sc_ah->ah_info.ah_valid = true;
1934                 irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
1935         } else {
1936                 sc_ah->ah_info.ah_valid = false;
1937                 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
1938         }
1939         spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
1940 }
1941
1942 /**
1943  * irdma_ilq_ah_cb - callback after creation of AH for ILQ
1944  * @cqp_request: pointer to cqp_request of create AH
1945  */
1946 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
1947 {
1948         struct irdma_cm_node *cm_node = cqp_request->param;
1949         struct irdma_sc_ah *sc_ah = cm_node->ah;
1950
1951         sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
1952         irdma_add_conn_est_qh(cm_node);
1953 }
1954
1955 /**
1956  * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
1957  * @dev: device pointer
1958  * @ah_info: Address handle info
1959  * @wait: When true will wait for operation to complete
1960  * @type: ILQ/IEQ
1961  * @cb_param: Callback param when not waiting
1962  * @ah_ret: Returned pointer to address handle if created
1963  *
1964  */
1965 int irdma_puda_create_ah(struct irdma_sc_dev *dev,
1966                          struct irdma_ah_info *ah_info, bool wait,
1967                          enum puda_rsrc_type type, void *cb_param,
1968                          struct irdma_sc_ah **ah_ret)
1969 {
1970         struct irdma_sc_ah *ah;
1971         struct irdma_pci_f *rf = dev_to_rf(dev);
1972         int err;
1973
1974         ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
1975         *ah_ret = ah;
1976         if (!ah)
1977                 return -ENOMEM;
1978
1979         err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
1980                                &ah_info->ah_idx, &rf->next_ah);
1981         if (err)
1982                 goto err_free;
1983
1984         ah->dev = dev;
1985         ah->ah_info = *ah_info;
1986
1987         if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
1988                 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
1989                                       irdma_ilq_ah_cb, cb_param);
1990         else
1991                 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
1992                                       irdma_ieq_ah_cb, cb_param);
1993
1994         if (err)
1995                 goto error;
1996         return 0;
1997
1998 error:
1999         irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2000 err_free:
2001         kfree(ah);
2002         *ah_ret = NULL;
2003         return -ENOMEM;
2004 }
2005
2006 /**
2007  * irdma_puda_free_ah - free a puda address handle
2008  * @dev: device pointer
2009  * @ah: The address handle to free
2010  */
2011 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2012 {
2013         struct irdma_pci_f *rf = dev_to_rf(dev);
2014
2015         if (!ah)
2016                 return;
2017
2018         if (ah->ah_info.ah_valid) {
2019                 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
2020                 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2021         }
2022
2023         kfree(ah);
2024 }
2025
2026 /**
2027  * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2028  * @cqp_request: pointer to cqp_request of create AH
2029  */
2030 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2031 {
2032         struct irdma_sc_ah *sc_ah = cqp_request->param;
2033
2034         if (!cqp_request->compl_info.op_ret_val)
2035                 sc_ah->ah_info.ah_valid = true;
2036         else
2037                 sc_ah->ah_info.ah_valid = false;
2038 }
2039
2040 /**
2041  * irdma_prm_add_pble_mem - add moemory to pble resources
2042  * @pprm: pble resource manager
2043  * @pchunk: chunk of memory to add
2044  */
2045 int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2046                            struct irdma_chunk *pchunk)
2047 {
2048         u64 sizeofbitmap;
2049
2050         if (pchunk->size & 0xfff)
2051                 return -EINVAL;
2052
2053         sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2054
2055         pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
2056         if (!pchunk->bitmapbuf)
2057                 return -ENOMEM;
2058
2059         pchunk->sizeofbitmap = sizeofbitmap;
2060         /* each pble is 8 bytes hence shift by 3 */
2061         pprm->total_pble_alloc += pchunk->size >> 3;
2062         pprm->free_pble_cnt += pchunk->size >> 3;
2063
2064         return 0;
2065 }
2066
2067 /**
2068  * irdma_prm_get_pbles - get pble's from prm
2069  * @pprm: pble resource manager
2070  * @chunkinfo: nformation about chunk where pble's were acquired
2071  * @mem_size: size of pble memory needed
2072  * @vaddr: returns virtual address of pble memory
2073  * @fpm_addr: returns fpm address of pble memory
2074  */
2075 int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
2076                         struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2077                         u64 **vaddr, u64 *fpm_addr)
2078 {
2079         u64 bits_needed;
2080         u64 bit_idx = PBLE_INVALID_IDX;
2081         struct irdma_chunk *pchunk = NULL;
2082         struct list_head *chunk_entry = pprm->clist.next;
2083         u32 offset;
2084         unsigned long flags;
2085         *vaddr = NULL;
2086         *fpm_addr = 0;
2087
2088         bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
2089
2090         spin_lock_irqsave(&pprm->prm_lock, flags);
2091         while (chunk_entry != &pprm->clist) {
2092                 pchunk = (struct irdma_chunk *)chunk_entry;
2093                 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2094                                                      pchunk->sizeofbitmap, 0,
2095                                                      bits_needed, 0);
2096                 if (bit_idx < pchunk->sizeofbitmap)
2097                         break;
2098
2099                 /* list.next used macro */
2100                 chunk_entry = pchunk->list.next;
2101         }
2102
2103         if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2104                 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2105                 return -ENOMEM;
2106         }
2107
2108         bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2109         offset = bit_idx << pprm->pble_shift;
2110         *vaddr = pchunk->vaddr + offset;
2111         *fpm_addr = pchunk->fpm_addr + offset;
2112
2113         chunkinfo->pchunk = pchunk;
2114         chunkinfo->bit_idx = bit_idx;
2115         chunkinfo->bits_used = bits_needed;
2116         /* 3 is sizeof pble divide */
2117         pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2118         spin_unlock_irqrestore(&pprm->prm_lock, flags);
2119
2120         return 0;
2121 }
2122
2123 /**
2124  * irdma_prm_return_pbles - return pbles back to prm
2125  * @pprm: pble resource manager
2126  * @chunkinfo: chunk where pble's were acquired and to be freed
2127  */
2128 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2129                             struct irdma_pble_chunkinfo *chunkinfo)
2130 {
2131         unsigned long flags;
2132
2133         spin_lock_irqsave(&pprm->prm_lock, flags);
2134         pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2135         bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2136                      chunkinfo->bits_used);
2137         spin_unlock_irqrestore(&pprm->prm_lock, flags);
2138 }
2139
2140 int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
2141                            u32 pg_cnt)
2142 {
2143         struct page *vm_page;
2144         int i;
2145         u8 *addr;
2146
2147         addr = (u8 *)(uintptr_t)va;
2148         for (i = 0; i < pg_cnt; i++) {
2149                 vm_page = vmalloc_to_page(addr);
2150                 if (!vm_page)
2151                         goto err;
2152
2153                 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2154                                          DMA_BIDIRECTIONAL);
2155                 if (dma_mapping_error(hw->device, pg_dma[i]))
2156                         goto err;
2157
2158                 addr += PAGE_SIZE;
2159         }
2160
2161         return 0;
2162
2163 err:
2164         irdma_unmap_vm_page_list(hw, pg_dma, i);
2165         return -ENOMEM;
2166 }
2167
2168 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2169 {
2170         int i;
2171
2172         for (i = 0; i < pg_cnt; i++)
2173                 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2174 }
2175
2176 /**
2177  * irdma_pble_free_paged_mem - free virtual paged memory
2178  * @chunk: chunk to free with paged memory
2179  */
2180 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2181 {
2182         if (!chunk->pg_cnt)
2183                 goto done;
2184
2185         irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2186                                  chunk->pg_cnt);
2187
2188 done:
2189         kfree(chunk->dmainfo.dmaaddrs);
2190         chunk->dmainfo.dmaaddrs = NULL;
2191         vfree(chunk->vaddr);
2192         chunk->vaddr = NULL;
2193         chunk->type = 0;
2194 }
2195
2196 /**
2197  * irdma_pble_get_paged_mem -allocate paged memory for pbles
2198  * @chunk: chunk to add for paged memory
2199  * @pg_cnt: number of pages needed
2200  */
2201 int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2202 {
2203         u32 size;
2204         void *va;
2205
2206         chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2207         if (!chunk->dmainfo.dmaaddrs)
2208                 return -ENOMEM;
2209
2210         size = PAGE_SIZE * pg_cnt;
2211         va = vmalloc(size);
2212         if (!va)
2213                 goto err;
2214
2215         if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2216                                    pg_cnt)) {
2217                 vfree(va);
2218                 goto err;
2219         }
2220         chunk->vaddr = va;
2221         chunk->size = size;
2222         chunk->pg_cnt = pg_cnt;
2223         chunk->type = PBLE_SD_PAGED;
2224
2225         return 0;
2226 err:
2227         kfree(chunk->dmainfo.dmaaddrs);
2228         chunk->dmainfo.dmaaddrs = NULL;
2229
2230         return -ENOMEM;
2231 }
2232
2233 /**
2234  * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2235  * @dev: device pointer
2236  */
2237 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2238 {
2239         struct irdma_pci_f *rf = dev_to_rf(dev);
2240         u32 next = 1;
2241         u32 node_id;
2242
2243         if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2244                              &node_id, &next))
2245                 return IRDMA_WS_NODE_INVALID;
2246
2247         return (u16)node_id;
2248 }
2249
2250 /**
2251  * irdma_free_ws_node_id - Free a tx scheduler node ID
2252  * @dev: device pointer
2253  * @node_id: Work scheduler node ID
2254  */
2255 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2256 {
2257         struct irdma_pci_f *rf = dev_to_rf(dev);
2258
2259         irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2260 }
2261
2262 /**
2263  * irdma_modify_qp_to_err - Modify a QP to error
2264  * @sc_qp: qp structure
2265  */
2266 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2267 {
2268         struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2269         struct ib_qp_attr attr;
2270
2271         if (qp->iwdev->rf->reset)
2272                 return;
2273         attr.qp_state = IB_QPS_ERR;
2274
2275         if (rdma_protocol_roce(qp->ibqp.device, 1))
2276                 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2277         else
2278                 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2279 }
2280
2281 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2282 {
2283         struct ib_event ibevent;
2284
2285         if (!iwqp->ibqp.event_handler)
2286                 return;
2287
2288         switch (event) {
2289         case IRDMA_QP_EVENT_CATASTROPHIC:
2290                 ibevent.event = IB_EVENT_QP_FATAL;
2291                 break;
2292         case IRDMA_QP_EVENT_ACCESS_ERR:
2293                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2294                 break;
2295         case IRDMA_QP_EVENT_REQ_ERR:
2296                 ibevent.event = IB_EVENT_QP_REQ_ERR;
2297                 break;
2298         }
2299         ibevent.device = iwqp->ibqp.device;
2300         ibevent.element.qp = &iwqp->ibqp;
2301         iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2302 }
2303
2304 bool irdma_cq_empty(struct irdma_cq *iwcq)
2305 {
2306         struct irdma_cq_uk *ukcq;
2307         u64 qword3;
2308         __le64 *cqe;
2309         u8 polarity;
2310
2311         ukcq  = &iwcq->sc_cq.cq_uk;
2312         cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
2313         get_64bit_val(cqe, 24, &qword3);
2314         polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
2315
2316         return polarity != ukcq->polarity;
2317 }
2318
2319 void irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2320 {
2321         struct irdma_cmpl_gen *cmpl_node;
2322         struct list_head *tmp_node, *list_node;
2323
2324         list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) {
2325                 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2326                 list_del(&cmpl_node->list);
2327                 kfree(cmpl_node);
2328         }
2329 }
2330
2331 int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2332 {
2333         struct irdma_cmpl_gen *cmpl;
2334
2335         if (list_empty(&iwcq->cmpl_generated))
2336                 return -ENOENT;
2337         cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2338         list_del(&cmpl->list);
2339         memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2340         kfree(cmpl);
2341
2342         ibdev_dbg(iwcq->ibcq.device,
2343                   "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",
2344                   __func__, cq_poll_info->qp_id, cq_poll_info->op_type,
2345                   cq_poll_info->wr_id);
2346
2347         return 0;
2348 }
2349
2350 /**
2351  * irdma_set_cpi_common_values - fill in values for polling info struct
2352  * @cpi: resulting structure of cq_poll_info type
2353  * @qp: QPair
2354  * @qp_num: id of the QP
2355  */
2356 static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2357                                         struct irdma_qp_uk *qp, u32 qp_num)
2358 {
2359         cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2360         cpi->error = true;
2361         cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2362         cpi->minor_err = FLUSH_GENERAL_ERR;
2363         cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp;
2364         cpi->qp_id = qp_num;
2365 }
2366
2367 static inline void irdma_comp_handler(struct irdma_cq *cq)
2368 {
2369         if (!cq->ibcq.comp_handler)
2370                 return;
2371         if (atomic_cmpxchg(&cq->armed, 1, 0))
2372                 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2373 }
2374
2375 void irdma_generate_flush_completions(struct irdma_qp *iwqp)
2376 {
2377         struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2378         struct irdma_ring *sq_ring = &qp->sq_ring;
2379         struct irdma_ring *rq_ring = &qp->rq_ring;
2380         struct irdma_cmpl_gen *cmpl;
2381         __le64 *sw_wqe;
2382         u64 wqe_qword;
2383         u32 wqe_idx;
2384         bool compl_generated = false;
2385         unsigned long flags1;
2386
2387         spin_lock_irqsave(&iwqp->iwscq->lock, flags1);
2388         if (irdma_cq_empty(iwqp->iwscq)) {
2389                 unsigned long flags2;
2390
2391                 spin_lock_irqsave(&iwqp->lock, flags2);
2392                 while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2393                         cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2394                         if (!cmpl) {
2395                                 spin_unlock_irqrestore(&iwqp->lock, flags2);
2396                                 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2397                                 return;
2398                         }
2399
2400                         wqe_idx = sq_ring->tail;
2401                         irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2402
2403                         cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2404                         sw_wqe = qp->sq_base[wqe_idx].elem;
2405                         get_64bit_val(sw_wqe, 24, &wqe_qword);
2406                         cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
2407                         cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2408                         /* remove the SQ WR by moving SQ tail*/
2409                         IRDMA_RING_SET_TAIL(*sq_ring,
2410                                 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2411                         if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {
2412                                 kfree(cmpl);
2413                                 continue;
2414                         }
2415                         ibdev_dbg(iwqp->iwscq->ibcq.device,
2416                                   "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",
2417                                   __func__, cmpl->cpi.wr_id, qp->qp_id);
2418                         list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);
2419                         compl_generated = true;
2420                 }
2421                 spin_unlock_irqrestore(&iwqp->lock, flags2);
2422                 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2423                 if (compl_generated)
2424                         irdma_comp_handler(iwqp->iwscq);
2425         } else {
2426                 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2427                 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2428                                  msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2429         }
2430
2431         spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);
2432         if (irdma_cq_empty(iwqp->iwrcq)) {
2433                 unsigned long flags2;
2434
2435                 spin_lock_irqsave(&iwqp->lock, flags2);
2436                 while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2437                         cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2438                         if (!cmpl) {
2439                                 spin_unlock_irqrestore(&iwqp->lock, flags2);
2440                                 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2441                                 return;
2442                         }
2443
2444                         wqe_idx = rq_ring->tail;
2445                         irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2446
2447                         cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2448                         cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2449                         cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2450                         /* remove the RQ WR by moving RQ tail */
2451                         IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2452                         ibdev_dbg(iwqp->iwrcq->ibcq.device,
2453                                   "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
2454                                   __func__, cmpl->cpi.wr_id, qp->qp_id,
2455                                   wqe_idx);
2456                         list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);
2457
2458                         compl_generated = true;
2459                 }
2460                 spin_unlock_irqrestore(&iwqp->lock, flags2);
2461                 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2462                 if (compl_generated)
2463                         irdma_comp_handler(iwqp->iwrcq);
2464         } else {
2465                 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2466                 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2467                                  msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2468         }
2469 }