drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/slab.h>
  40
  41 #include "nes.h"
  42
  43 static int wide_ppm_offset;
  44 module_param(wide_ppm_offset, int, 0644);
  45 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  46
  47 static u32 crit_err_count;
  48 u32 int_mod_timer_init;
  49 u32 int_mod_cq_depth_256;
  50 u32 int_mod_cq_depth_128;
  51 u32 int_mod_cq_depth_32;
  52 u32 int_mod_cq_depth_24;
  53 u32 int_mod_cq_depth_16;
  54 u32 int_mod_cq_depth_4;
  55 u32 int_mod_cq_depth_1;
  56 static const u8 nes_max_critical_error_count = 100;
  57 #include "nes_cm.h"
  58
  59 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  60 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  61 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  62                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  63 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  64 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  65 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  66 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  67                                    struct nes_hw_aeqe *aeqe);
  68 static void process_critical_error(struct nes_device *nesdev);
  69 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  70 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  71 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  72
  73 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  74 static unsigned char *nes_iwarp_state_str[] = {
  75         "Non-Existent",
  76         "Idle",
  77         "RTS",
  78         "Closing",
  79         "RSVD1",
  80         "Terminate",
  81         "Error",
  82         "RSVD2",
  83 };
  84
  85 static unsigned char *nes_tcp_state_str[] = {
  86         "Non-Existent",
  87         "Closed",
  88         "Listen",
  89         "SYN Sent",
  90         "SYN Rcvd",
  91         "Established",
  92         "Close Wait",
  93         "FIN Wait 1",
  94         "Closing",
  95         "Last Ack",
  96         "FIN Wait 2",
  97         "Time Wait",
  98         "RSVD1",
  99         "RSVD2",
 100         "RSVD3",
 101         "RSVD4",
 102 };
 103 #endif
 104
 105 static inline void print_ip(struct nes_cm_node *cm_node)
 106 {
 107         unsigned char *rem_addr;
 108         if (cm_node) {
 109                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 110                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 111         }
 112 }
 113
 114 /**
 115  * nes_nic_init_timer_defaults
 116  */
 117 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 118 {
 119         unsigned long flags;
 120         struct nes_adapter *nesadapter = nesdev->nesadapter;
 121         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 122
 123         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 124
 125         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 126         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 127         if (jumbomode) {
 128                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 129                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 130                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 131         } else {
 132                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 133                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 134                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 135         }
 136
 137         /* todo use netdev->mtu to set thresholds */
 138         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 139 }
 140
 141
 142 /**
 143  * nes_nic_init_timer
 144  */
 145 static void  nes_nic_init_timer(struct nes_device *nesdev)
 146 {
 147         unsigned long flags;
 148         struct nes_adapter *nesadapter = nesdev->nesadapter;
 149         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 150
 151         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 152
 153         if (shared_timer->timer_in_use_old == 0) {
 154                 nesdev->deepcq_count = 0;
 155                 shared_timer->timer_direction_upward = 0;
 156                 shared_timer->timer_direction_downward = 0;
 157                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 158                 shared_timer->timer_in_use_old = 0;
 159
 160         }
 161         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 162                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 163                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 164                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 165         }
 166         /* todo use netdev->mtu to set thresholds */
 167         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 168 }
 169
 170
 171 /**
 172  * nes_nic_tune_timer
 173  */
 174 static void nes_nic_tune_timer(struct nes_device *nesdev)
 175 {
 176         unsigned long flags;
 177         struct nes_adapter *nesadapter = nesdev->nesadapter;
 178         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 179         u16 cq_count = nesdev->currcq_count;
 180
 181         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 182
 183         if (shared_timer->cq_count_old <= cq_count)
 184                 shared_timer->cq_direction_downward = 0;
 185         else
 186                 shared_timer->cq_direction_downward++;
 187         shared_timer->cq_count_old = cq_count;
 188         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 189                 if (cq_count <= shared_timer->threshold_low &&
 190                     shared_timer->threshold_low > 4) {
 191                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 192                         shared_timer->cq_direction_downward=0;
 193                         nesdev->currcq_count = 0;
 194                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 195                         return;
 196                 }
 197         }
 198
 199         if (cq_count > 1) {
 200                 nesdev->deepcq_count += cq_count;
 201                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 202                         shared_timer->timer_direction_upward++;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward = 0;
 207                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 208                         shared_timer->timer_direction_downward++;
 209                         shared_timer->timer_direction_upward = 0;
 210                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 211                         shared_timer->timer_in_use -= 2;
 212                         shared_timer->timer_direction_upward = 0;
 213                         shared_timer->timer_direction_downward++;
 214                 } else {
 215                         shared_timer->timer_in_use -= 4;
 216                         shared_timer->timer_direction_upward = 0;
 217                         shared_timer->timer_direction_downward++;
 218                 }
 219
 220                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 221                         shared_timer->timer_in_use += 3;
 222                         shared_timer->timer_direction_upward = 0;
 223                         shared_timer->timer_direction_downward = 0;
 224                 }
 225                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 226                         shared_timer->timer_in_use -= 4 ;
 227                         shared_timer->timer_direction_downward = 0;
 228                         shared_timer->timer_direction_upward = 0;
 229                 }
 230         }
 231
 232         /* boundary checking */
 233         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 234                 shared_timer->timer_in_use = shared_timer->threshold_high;
 235         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 236                 shared_timer->timer_in_use = shared_timer->threshold_low;
 237
 238         nesdev->currcq_count = 0;
 239
 240         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 241 }
 242
 243
 244 /**
 245  * nes_init_adapter - initialize adapter
 246  */
 247 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 248         struct nes_adapter *nesadapter = NULL;
 249         unsigned long num_pds;
 250         u32 u32temp;
 251         u32 port_count;
 252         u16 max_rq_wrs;
 253         u16 max_sq_wrs;
 254         u32 max_mr;
 255         u32 max_256pbl;
 256         u32 max_4kpbl;
 257         u32 max_qp;
 258         u32 max_irrq;
 259         u32 max_cq;
 260         u32 hte_index_mask;
 261         u32 adapter_size;
 262         u32 arp_table_size;
 263         u16 vendor_id;
 264         u16 device_id;
 265         u8  OneG_Mode;
 266         u8  func_index;
 267
 268         /* search the list of existing adapters */
 269         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 270                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 271                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 272                                 nesdev->pcidev->devfn,
 273                                 PCI_SLOT(nesadapter->devfn),
 274                                 nesadapter->bus_number,
 275                                 PCI_SLOT(nesdev->pcidev->devfn),
 276                                 nesdev->pcidev->bus->number );
 277                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 278                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 279                         nesadapter->ref_count++;
 280                         return nesadapter;
 281                 }
 282         }
 283
 284         /* no adapter found */
 285         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 286         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 287                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 288                                 hw_rev);
 289                 return NULL;
 290         }
 291
 292         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 293                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 296                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 297
 298         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 299
 300
 301         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 302                 return NULL;
 303
 304         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 305         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 306
 307         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 308         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 309                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 310                                 max_qp, u32temp);
 311                 max_qp = (u32)1 << (u32temp & 0x001f);
 312         }
 313
 314         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 315         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 316                         max_qp, hte_index_mask);
 317
 318         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 319
 320         max_irrq = 1 << (u32temp & 0x001f);
 321
 322         if (max_qp > max_irrq) {
 323                 max_qp = max_irrq;
 324                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 325                                 max_qp);
 326         }
 327
 328         /* there should be no reason to allocate more pds than qps */
 329         if (num_pds > max_qp)
 330                 num_pds = max_qp;
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 333         max_mr = (u32)8192 << (u32temp & 0x7);
 334
 335         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 336         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 337         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 338         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 339
 340         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 341         arp_table_size = 1 << u32temp;
 342
 343         adapter_size = (sizeof(struct nes_adapter) +
 344                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 349         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 350         adapter_size += sizeof(struct nes_qp **) * max_qp;
 351
 352         /* allocate a new adapter struct */
 353         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 354         if (!nesadapter)
 355                 return NULL;
 356
 357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 359
 360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 362                 kfree(nesadapter);
 363                 return NULL;
 364         }
 365
 366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 367                                 (nesadapter->mac_addr_low >> 24);
 368
 369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 370                                  PCI_DEVICE_ID, &device_id);
 371         nesadapter->vendor_part_id = device_id;
 372
 373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 374                                                         OneG_Mode)) {
 375                 kfree(nesadapter);
 376                 return NULL;
 377         }
 378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 379
 380         memset(nesadapter->pft_mcast_map, 255,
 381                sizeof nesadapter->pft_mcast_map);
 382
 383         /* populate the new nesadapter */
 384         nesadapter->devfn = nesdev->pcidev->devfn;
 385         nesadapter->bus_number = nesdev->pcidev->bus->number;
 386         nesadapter->ref_count = 1;
 387         nesadapter->timer_int_req = 0xffff0000;
 388         nesadapter->OneG_Mode = OneG_Mode;
 389         nesadapter->doorbell_start = nesdev->doorbell_region;
 390
 391         /* nesadapter->tick_delta = clk_divisor; */
 392         nesadapter->hw_rev = hw_rev;
 393         nesadapter->port_count = port_count;
 394
 395         nesadapter->max_qp = max_qp;
 396         nesadapter->hte_index_mask = hte_index_mask;
 397         nesadapter->max_irrq = max_irrq;
 398         nesadapter->max_mr = max_mr;
 399         nesadapter->max_256pbl = max_256pbl - 1;
 400         nesadapter->max_4kpbl = max_4kpbl - 1;
 401         nesadapter->max_cq = max_cq;
 402         nesadapter->free_256pbl = max_256pbl - 1;
 403         nesadapter->free_4kpbl = max_4kpbl - 1;
 404         nesadapter->max_pd = num_pds;
 405         nesadapter->arp_table_size = arp_table_size;
 406
 407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 412         } else {
 413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 417         }
 418         /* Setup and enable the periodic timer */
 419         if (nesadapter->et_rx_coalesce_usecs_irq)
 420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 422         else
 423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 424
 425         nesadapter->base_pd = 1;
 426
 427         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 428                                        IB_DEVICE_MEM_WINDOW |
 429                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 430
 431         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 432                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 433         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 434         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 435         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 436         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 437         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 438
 439
 440         /* mark the usual suspect QPs, MR and CQs as in use */
 441         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 442                 set_bit(u32temp, nesadapter->allocated_qps);
 443                 set_bit(u32temp, nesadapter->allocated_cqs);
 444         }
 445         set_bit(0, nesadapter->allocated_mrs);
 446
 447         for (u32temp = 0; u32temp < 20; u32temp++)
 448                 set_bit(u32temp, nesadapter->allocated_pds);
 449         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 450
 451         max_rq_wrs = ((u32temp >> 8) & 3);
 452         switch (max_rq_wrs) {
 453                 case 0:
 454                         max_rq_wrs = 4;
 455                         break;
 456                 case 1:
 457                         max_rq_wrs = 16;
 458                         break;
 459                 case 2:
 460                         max_rq_wrs = 32;
 461                         break;
 462                 case 3:
 463                         max_rq_wrs = 512;
 464                         break;
 465         }
 466
 467         max_sq_wrs = (u32temp & 3);
 468         switch (max_sq_wrs) {
 469                 case 0:
 470                         max_sq_wrs = 4;
 471                         break;
 472                 case 1:
 473                         max_sq_wrs = 16;
 474                         break;
 475                 case 2:
 476                         max_sq_wrs = 32;
 477                         break;
 478                 case 3:
 479                         max_sq_wrs = 512;
 480                         break;
 481         }
 482         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 483         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 484
 485         nesadapter->max_sge = 4;
 486         nesadapter->max_cqe = 32766;
 487
 488         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 489                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 490                 kfree(nesadapter);
 491                 return NULL;
 492         }
 493
 494         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 495         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 496                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 497
 498         /* setup port configuration */
 499         if (nesadapter->port_count == 1) {
 500                 nesadapter->log_port = 0x00000000;
 501                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 502                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 503                 else
 504                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 505         } else {
 506                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 507                         nesadapter->log_port = 0x000000D8;
 508                 } else {
 509                         if (nesadapter->port_count == 2)
 510                                 nesadapter->log_port = 0x00000044;
 511                         else
 512                                 nesadapter->log_port = 0x000000e4;
 513                 }
 514                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 515         }
 516
 517         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 518                                                 nesadapter->log_port);
 519         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 520                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 521
 522         spin_lock_init(&nesadapter->resource_lock);
 523         spin_lock_init(&nesadapter->phy_lock);
 524         spin_lock_init(&nesadapter->pbl_lock);
 525         spin_lock_init(&nesadapter->periodic_timer_lock);
 526
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 530         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 531
 532         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 533                 u32 pcs_control_status0, pcs_control_status1;
 534                 u32 reset_value;
 535                 u32 i = 0;
 536                 u32 int_cnt = 0;
 537                 u32 ext_cnt = 0;
 538                 unsigned long flags;
 539                 u32 j = 0;
 540
 541                 pcs_control_status0 = nes_read_indexed(nesdev,
 542                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 543                 pcs_control_status1 = nes_read_indexed(nesdev,
 544                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 545
 546                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 547                         pcs_control_status0 = nes_read_indexed(nesdev,
 548                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 549                         pcs_control_status1 = nes_read_indexed(nesdev,
 550                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 551                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 552                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 553                                 int_cnt++;
 554                         msleep(1);
 555                 }
 556                 if (int_cnt > 1) {
 557                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 558                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 559                         mh_detected++;
 560                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 561                         reset_value |= 0x0000003d;
 562                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 563
 564                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 565                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 566                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 567
 568                         pcs_control_status0 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 570                         pcs_control_status1 = nes_read_indexed(nesdev,
 571                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 572
 573                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 574                                 pcs_control_status0 = nes_read_indexed(nesdev,
 575                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 576                                 pcs_control_status1 = nes_read_indexed(nesdev,
 577                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 578                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 579                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 580                                         if (++ext_cnt > int_cnt) {
 581                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 582                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 583                                                                 0x0000F088);
 584                                                 mh_detected++;
 585                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 586                                                 reset_value |= 0x0000003d;
 587                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 588
 589                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 590                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 591                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 592                                                 break;
 593                                         }
 594                                 }
 595                                 msleep(1);
 596                         }
 597                 }
 598         }
 599
 600         if (nesadapter->hw_rev == NE020_REV) {
 601                 init_timer(&nesadapter->mh_timer);
 602                 nesadapter->mh_timer.function = nes_mh_fix;
 603                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 604                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 605                 add_timer(&nesadapter->mh_timer);
 606         } else {
 607                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 608         }
 609
 610         init_timer(&nesadapter->lc_timer);
 611         nesadapter->lc_timer.function = nes_clc;
 612         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 613         nesadapter->lc_timer.data = (unsigned long)nesdev;
 614         add_timer(&nesadapter->lc_timer);
 615
 616         list_add_tail(&nesadapter->list, &nes_adapter_list);
 617
 618         for (func_index = 0; func_index < 8; func_index++) {
 619                 pci_bus_read_config_word(nesdev->pcidev->bus,
 620                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 621                                         func_index), 0, &vendor_id);
 622                 if (vendor_id == 0xffff)
 623                         break;
 624         }
 625         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 626                 func_index, pci_name(nesdev->pcidev));
 627         nesadapter->adapter_fcn_count = func_index;
 628
 629         return nesadapter;
 630 }
 631
 632
 633 /**
 634  * nes_reset_adapter_ne020
 635  */
 636 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 637 {
 638         u32 port_count;
 639         u32 u32temp;
 640         u32 i;
 641
 642         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 643         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 644         /* TODO: assuming that both SERDES are set the same for now */
 645         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 646         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 647                         u32temp, port_count);
 648         if (*OneG_Mode)
 649                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 650         u32temp &= 0xff00ffc0;
 651         switch (port_count) {
 652                 case 1:
 653                         u32temp |= 0x00ee0000;
 654                         break;
 655                 case 2:
 656                         u32temp |= 0x00cc0000;
 657                         break;
 658                 case 4:
 659                         u32temp |= 0x00000000;
 660                         break;
 661                 default:
 662                         return 0;
 663                         break;
 664         }
 665
 666         /* check and do full reset if needed */
 667         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 668                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 669                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 670
 671                 i = 0;
 672                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 673                         mdelay(1);
 674                 if (i > 10000) {
 675                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 676                         return 0;
 677                 }
 678
 679                 i = 0;
 680                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 681                         mdelay(1);
 682                 if (i > 10000) {
 683                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 684                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 685                         return 0;
 686                 }
 687         }
 688
 689         /* port reset */
 690         switch (port_count) {
 691                 case 1:
 692                         u32temp |= 0x00ee0010;
 693                         break;
 694                 case 2:
 695                         u32temp |= 0x00cc0030;
 696                         break;
 697                 case 4:
 698                         u32temp |= 0x00000030;
 699                         break;
 700         }
 701
 702         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 703         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 704
 705         i = 0;
 706         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 707                 mdelay(1);
 708         if (i > 10000) {
 709                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 710                 return 0;
 711         }
 712
 713         /* serdes 0 */
 714         i = 0;
 715         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 716                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 717                 mdelay(1);
 718         if (i > 5000) {
 719                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 720                 return 0;
 721         }
 722
 723         /* serdes 1 */
 724         if (port_count > 1) {
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i > 5000) {
 730                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 731                         return 0;
 732                 }
 733         }
 734
 735         return port_count;
 736 }
 737
 738
 739 /**
 740  * nes_init_serdes
 741  */
 742 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 743                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 744 {
 745         int i;
 746         u32 u32temp;
 747         u32 sds;
 748
 749         if (hw_rev != NE020_REV) {
 750                 /* init serdes 0 */
 751                 switch (nesadapter->phy_type[0]) {
 752                 case NES_PHY_TYPE_CX4:
 753                         if (wide_ppm_offset)
 754                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 755                         else
 756                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 757                         break;
 758                 case NES_PHY_TYPE_KR:
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 761                         break;
 762                 case NES_PHY_TYPE_PUMA_1G:
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 764                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 765                         sds |= 0x00000100;
 766                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 767                         break;
 768                 default:
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 770                         break;
 771                 }
 772
 773                 if (!OneG_Mode)
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 775
 776                 if (port_count < 2)
 777                         return 0;
 778
 779                 /* init serdes 1 */
 780                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 781                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 782
 783                 switch (nesadapter->phy_type[1]) {
 784                 case NES_PHY_TYPE_ARGUS:
 785                 case NES_PHY_TYPE_SFP_D:
 786                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 788                         break;
 789                 case NES_PHY_TYPE_CX4:
 790                         if (wide_ppm_offset)
 791                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 792                         break;
 793                 case NES_PHY_TYPE_KR:
 794                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 795                         break;
 796                 case NES_PHY_TYPE_PUMA_1G:
 797                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 798                         sds |= 0x000000100;
 799                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 800                 }
 801                 if (!OneG_Mode) {
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 803                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 804                         sds &= 0xFFFFFFBF;
 805                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 806                 }
 807         } else {
 808                 /* init serdes 0 */
 809                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 810                 i = 0;
 811                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 812                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 813                         mdelay(1);
 814                 if (i > 5000) {
 815                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 816                         return 1;
 817                 }
 818                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 819                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 820                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 821                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 822                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 823                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 824                 if (OneG_Mode)
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 826                 else
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 828
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 830                 if (port_count > 1) {
 831                         /* init serdes 1 */
 832                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 833                         i = 0;
 834                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 835                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 836                                 mdelay(1);
 837                         if (i > 5000) {
 838                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 839                                 /* return 1; */
 840                         }
 841                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 842                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 843                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 844                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 845                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 846                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 847                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 848                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 849                 }
 850         }
 851         return 0;
 852 }
 853
 854
 855 /**
 856  * nes_init_csr_ne020
 857  * Initialize registers for ne020 hardware
 858  */
 859 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 860 {
 861         u32 u32temp;
 862
 863         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 864
 865         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 866         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 867         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 868         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 869         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 870         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 871         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 872         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 873
 874         /* TODO: move these MAC register settings to NIC bringup */
 875         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 876         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 877         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 878         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 879         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 880         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 881         if (port_count > 1) {
 882                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 883                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 885                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 886                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 887                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 888                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 889         }
 890         if (port_count > 2) {
 891                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 894                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 895                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 896                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 897                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 898
 899                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 900                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 902                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 903                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 904                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 905                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 906         }
 907
 908         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 909         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 910         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 911                                                          0x00000001);
 912         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 913         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 914         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 915         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 916         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 917
 918         /* TODO: move this to code, get from EEPROM */
 919         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 920         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 921         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 922
 923         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 924         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 925
 926         if (hw_rev != NE020_REV) {
 927                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 928                 u32temp |= 0x80000000;
 929                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 930                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 931                 u32temp &= 0x7fffffff;
 932                 u32temp |= 0x7fff0010;
 933                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 934                 if (port_count > 1) {
 935                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 936                         u32temp &= 0x7fffffff;
 937                         u32temp |= 0x7fff0010;
 938                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 939                 }
 940         }
 941 }
 942
 943
 944 /**
 945  * nes_destroy_adapter - destroy the adapter structure
 946  */
 947 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 948 {
 949         struct nes_adapter *tmp_adapter;
 950
 951         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 952                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 953                                 tmp_adapter);
 954         }
 955
 956         nesadapter->ref_count--;
 957         if (!nesadapter->ref_count) {
 958                 if (nesadapter->hw_rev == NE020_REV) {
 959                         del_timer(&nesadapter->mh_timer);
 960                 }
 961                 del_timer(&nesadapter->lc_timer);
 962
 963                 list_del(&nesadapter->list);
 964                 kfree(nesadapter);
 965         }
 966 }
 967
 968
 969 /**
 970  * nes_init_cqp
 971  */
 972 int nes_init_cqp(struct nes_device *nesdev)
 973 {
 974         struct nes_adapter *nesadapter = nesdev->nesadapter;
 975         struct nes_hw_cqp_qp_context *cqp_qp_context;
 976         struct nes_hw_cqp_wqe *cqp_wqe;
 977         struct nes_hw_ceq *ceq;
 978         struct nes_hw_ceq *nic_ceq;
 979         struct nes_hw_aeq *aeq;
 980         void *vmem;
 981         dma_addr_t pmem;
 982         u32 count=0;
 983         u32 cqp_head;
 984         u64 u64temp;
 985         u32 u32temp;
 986
 987         /* allocate CQP memory */
 988         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 989         /* SQ is 512 byte aligned, others are 256 byte aligned */
 990         nesdev->cqp_mem_size = 512 +
 991                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 992                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 993                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 994                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 995                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 996                         sizeof(struct nes_hw_cqp_qp_context);
 997
 998         nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
 999                                                   nesdev->cqp_mem_size,
1000                                                   &nesdev->cqp_pbase);
1001         if (!nesdev->cqp_vbase) {
1002                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1003                 return -ENOMEM;
1004         }
1005
1006         /* Allocate a twice the number of CQP requests as the SQ size */
1007         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1008                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1009         if (!nesdev->nes_cqp_requests) {
1010                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1011                                 nesdev->cqp.sq_pbase);
1012                 return -ENOMEM;
1013         }
1014
1015         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1016                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1017
1018         spin_lock_init(&nesdev->cqp.lock);
1019         init_waitqueue_head(&nesdev->cqp.waitq);
1020
1021         /* Setup Various Structures */
1022         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1023                         ~(unsigned long)(512 - 1));
1024         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1025                         ~(unsigned long long)(512 - 1));
1026
1027         nesdev->cqp.sq_vbase = vmem;
1028         nesdev->cqp.sq_pbase = pmem;
1029         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1030         nesdev->cqp.sq_head = 0;
1031         nesdev->cqp.sq_tail = 0;
1032         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1033
1034         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1035         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1036
1037         nesdev->ccq.cq_vbase = vmem;
1038         nesdev->ccq.cq_pbase = pmem;
1039         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1040         nesdev->ccq.cq_head = 0;
1041         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1042         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1043
1044         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1045         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1046
1047         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1048         ceq = &nesadapter->ceq[nesdev->ceq_index];
1049         ceq->ceq_vbase = vmem;
1050         ceq->ceq_pbase = pmem;
1051         ceq->ceq_size = NES_CCEQ_SIZE;
1052         ceq->ceq_head = 0;
1053
1054         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1055         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1056
1057         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1058         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1059         nic_ceq->ceq_vbase = vmem;
1060         nic_ceq->ceq_pbase = pmem;
1061         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1062         nic_ceq->ceq_head = 0;
1063
1064         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1065         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1066
1067         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1068         aeq->aeq_vbase = vmem;
1069         aeq->aeq_pbase = pmem;
1070         aeq->aeq_size = nesadapter->max_qp;
1071         aeq->aeq_head = 0;
1072
1073         /* Setup QP Context */
1074         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1075         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1076
1077         cqp_qp_context = vmem;
1078         cqp_qp_context->context_words[0] =
1079                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1080         cqp_qp_context->context_words[1] = 0;
1081         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1082         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1083
1084
1085         /* Write the address to Create CQP */
1086         if ((sizeof(dma_addr_t) > 4)) {
1087                 nes_write_indexed(nesdev,
1088                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1089                                 ((u64)pmem) >> 32);
1090         } else {
1091                 nes_write_indexed(nesdev,
1092                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1093         }
1094         nes_write_indexed(nesdev,
1095                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1096                         (u32)pmem);
1097
1098         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1099         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1100
1101         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1102                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1103                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1104         }
1105
1106         /* Write Create CCQ WQE */
1107         cqp_head = nesdev->cqp.sq_head++;
1108         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1109         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1110         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1111                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1112                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1113         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1114                             (nesdev->ccq.cq_number |
1115                              ((u32)nesdev->ceq_index << 16)));
1116         u64temp = (u64)nesdev->ccq.cq_pbase;
1117         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1118         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1119         u64temp = (unsigned long)&nesdev->ccq;
1120         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1121                         cpu_to_le32((u32)(u64temp >> 1));
1122         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1123                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1124         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1125
1126         /* Write Create CEQ WQE */
1127         cqp_head = nesdev->cqp.sq_head++;
1128         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1129         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1130         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1131                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1132         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1133         u64temp = (u64)ceq->ceq_pbase;
1134         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1135
1136         /* Write Create AEQ WQE */
1137         cqp_head = nesdev->cqp.sq_head++;
1138         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1139         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1140         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1141                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1142         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1143         u64temp = (u64)aeq->aeq_pbase;
1144         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1145
1146         /* Write Create NIC CEQ WQE */
1147         cqp_head = nesdev->cqp.sq_head++;
1148         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1149         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1150         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1151                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1152         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1153         u64temp = (u64)nic_ceq->ceq_pbase;
1154         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1155
1156         /* Poll until CCQP done */
1157         count = 0;
1158         do {
1159                 if (count++ > 1000) {
1160                         printk(KERN_ERR PFX "Error creating CQP\n");
1161                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1162                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1163                         return -1;
1164                 }
1165                 udelay(10);
1166         } while (!(nes_read_indexed(nesdev,
1167                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1168
1169         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1170                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1171
1172         u32temp = 0x04800000;
1173         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1174
1175         /* wait for the CCQ, CEQ, and AEQ to get created */
1176         count = 0;
1177         do {
1178                 if (count++ > 1000) {
1179                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1180                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1181                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1182                         return -1;
1183                 }
1184                 udelay(10);
1185         } while (((nes_read_indexed(nesdev,
1186                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1187
1188         /* dump the QP status value */
1189         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1190                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1191
1192         nesdev->cqp.sq_tail++;
1193
1194         return 0;
1195 }
1196
1197
1198 /**
1199  * nes_destroy_cqp
1200  */
1201 int nes_destroy_cqp(struct nes_device *nesdev)
1202 {
1203         struct nes_hw_cqp_wqe *cqp_wqe;
1204         u32 count = 0;
1205         u32 cqp_head;
1206         unsigned long flags;
1207
1208         do {
1209                 if (count++ > 1000)
1210                         break;
1211                 udelay(10);
1212         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1213
1214         /* Reset CCQ */
1215         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1216                         nesdev->ccq.cq_number);
1217
1218         /* Disable device interrupts */
1219         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1220
1221         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1222
1223         /* Destroy the AEQ */
1224         cqp_head = nesdev->cqp.sq_head++;
1225         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1226         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1227         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1228                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1229         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1230
1231         /* Destroy the NIC CEQ */
1232         cqp_head = nesdev->cqp.sq_head++;
1233         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1234         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1235         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1236                         ((u32)nesdev->nic_ceq_index << 8));
1237
1238         /* Destroy the CEQ */
1239         cqp_head = nesdev->cqp.sq_head++;
1240         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1241         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1242         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1243                         (nesdev->ceq_index << 8));
1244
1245         /* Destroy the CCQ */
1246         cqp_head = nesdev->cqp.sq_head++;
1247         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1248         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1249         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1250         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1251                         ((u32)nesdev->ceq_index << 16));
1252
1253         /* Destroy CQP */
1254         cqp_head = nesdev->cqp.sq_head++;
1255         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1256         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1257         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1258                         NES_CQP_QP_TYPE_CQP);
1259         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1260
1261         barrier();
1262         /* Ring doorbell (5 WQEs) */
1263         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1264
1265         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1266
1267         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1268         count = 0;
1269         do {
1270                 if (count++ > 1000) {
1271                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1272                                         PCI_FUNC(nesdev->pcidev->devfn));
1273                         break;
1274                 }
1275                 udelay(10);
1276         } while (((nes_read_indexed(nesdev,
1277                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1278
1279         /* dump the QP status value */
1280         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1281                         PCI_FUNC(nesdev->pcidev->devfn),
1282                         nes_read_indexed(nesdev,
1283                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1284
1285         kfree(nesdev->nes_cqp_requests);
1286
1287         /* Free the control structures */
1288         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1289                         nesdev->cqp.sq_pbase);
1290
1291         return 0;
1292 }
1293
1294
1295 /**
1296  * nes_init_1g_phy
1297  */
1298 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1299 {
1300         u32 counter = 0;
1301         u16 phy_data;
1302         int ret = 0;
1303
1304         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1305         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1306
1307         /* Reset the PHY */
1308         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1309         udelay(100);
1310         counter = 0;
1311         do {
1312                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1313                 if (counter++ > 100) {
1314                         ret = -1;
1315                         break;
1316                 }
1317         } while (phy_data & 0x8000);
1318
1319         /* Setting no phy loopback */
1320         phy_data &= 0xbfff;
1321         phy_data |= 0x1140;
1322         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1323         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1324         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1325         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1326
1327         /* Setting the interrupt mask */
1328         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1329         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1330         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1331
1332         /* turning on flow control */
1333         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1334         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1335         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1336
1337         /* Clear Half duplex */
1338         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1339         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1340         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1341
1342         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1343         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1344
1345         return ret;
1346 }
1347
1348
1349 /**
1350  * nes_init_2025_phy
1351  */
1352 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1353 {
1354         u32 temp_phy_data = 0;
1355         u32 temp_phy_data2 = 0;
1356         u32 counter = 0;
1357         u32 sds;
1358         u32 mac_index = nesdev->mac_index;
1359         int ret = 0;
1360         unsigned int first_attempt = 1;
1361
1362         /* Check firmware heartbeat */
1363         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1364         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1365         udelay(1500);
1366         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1367         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1368
1369         if (temp_phy_data != temp_phy_data2) {
1370                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1371                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1372                 if ((temp_phy_data & 0xff) > 0x20)
1373                         return 0;
1374                 printk(PFX "Reinitialize external PHY\n");
1375         }
1376
1377         /* no heartbeat, configure the PHY */
1378         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1379         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1380         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1381         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1382
1383         switch (phy_type) {
1384         case NES_PHY_TYPE_ARGUS:
1385                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1389                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1392
1393                 /* setup LEDs */
1394                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1397                 break;
1398
1399         case NES_PHY_TYPE_SFP_D:
1400                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1402                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1407
1408                 /* setup LEDs */
1409                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1412                 break;
1413
1414         case NES_PHY_TYPE_KR:
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1422
1423                 /* setup LEDs */
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1427
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1430                 break;
1431         }
1432
1433         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1434
1435         /* Bring PHY out of reset */
1436         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1437
1438         /* Check for heartbeat */
1439         counter = 0;
1440         mdelay(690);
1441         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1442         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1443         do {
1444                 if (counter++ > 150) {
1445                         printk(PFX "No PHY heartbeat\n");
1446                         break;
1447                 }
1448                 mdelay(1);
1449                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1450                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1451         } while ((temp_phy_data2 == temp_phy_data));
1452
1453         /* wait for tracking */
1454         counter = 0;
1455         do {
1456                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1457                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1458                 if (counter++ > 300) {
1459                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1460                                 first_attempt = 0;
1461                                 counter = 0;
1462                                 /* reset AMCC PHY and try again */
1463                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1464                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1465                                 continue;
1466                         } else {
1467                                 ret = 1;
1468                                 break;
1469                         }
1470                 }
1471                 mdelay(10);
1472         } while ((temp_phy_data & 0xff) < 0x30);
1473
1474         /* setup signal integrity */
1475         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1476         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1477         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1478         if (phy_type == NES_PHY_TYPE_KR) {
1479                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1480         } else {
1481                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1482                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1483         }
1484
1485         /* reset serdes */
1486         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1487         sds |= 0x1;
1488         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1489         sds &= 0xfffffffe;
1490         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1491
1492         counter = 0;
1493         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1494                         && (counter++ < 5000))
1495                 ;
1496
1497         return ret;
1498 }
1499
1500
1501 /**
1502  * nes_init_phy
1503  */
1504 int nes_init_phy(struct nes_device *nesdev)
1505 {
1506         struct nes_adapter *nesadapter = nesdev->nesadapter;
1507         u32 mac_index = nesdev->mac_index;
1508         u32 tx_config = 0;
1509         unsigned long flags;
1510         u8  phy_type = nesadapter->phy_type[mac_index];
1511         u8  phy_index = nesadapter->phy_index[mac_index];
1512         int ret = 0;
1513
1514         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1515         if (phy_type == NES_PHY_TYPE_1G) {
1516                 /* setup 1G MDIO operation */
1517                 tx_config &= 0xFFFFFFE3;
1518                 tx_config |= 0x04;
1519         } else {
1520                 /* setup 10G MDIO operation */
1521                 tx_config &= 0xFFFFFFE3;
1522                 tx_config |= 0x1D;
1523         }
1524         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1525
1526         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1527
1528         switch (phy_type) {
1529         case NES_PHY_TYPE_1G:
1530                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1531                 break;
1532         case NES_PHY_TYPE_ARGUS:
1533         case NES_PHY_TYPE_SFP_D:
1534         case NES_PHY_TYPE_KR:
1535                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1536                 break;
1537         }
1538
1539         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1540
1541         return ret;
1542 }
1543
1544
1545 /**
1546  * nes_replenish_nic_rq
1547  */
1548 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1549 {
1550         unsigned long flags;
1551         dma_addr_t bus_address;
1552         struct sk_buff *skb;
1553         struct nes_hw_nic_rq_wqe *nic_rqe;
1554         struct nes_hw_nic *nesnic;
1555         struct nes_device *nesdev;
1556         struct nes_rskb_cb *cb;
1557         u32 rx_wqes_posted = 0;
1558
1559         nesnic = &nesvnic->nic;
1560         nesdev = nesvnic->nesdev;
1561         spin_lock_irqsave(&nesnic->rq_lock, flags);
1562         if (nesnic->replenishing_rq !=0) {
1563                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1564                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1565                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1566                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1567                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1568                         add_timer(&nesvnic->rq_wqes_timer);
1569                 } else
1570                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1571                 return;
1572         }
1573         nesnic->replenishing_rq = 1;
1574         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1575         do {
1576                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1577                 if (skb) {
1578                         skb->dev = nesvnic->netdev;
1579
1580                         bus_address = pci_map_single(nesdev->pcidev,
1581                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1582                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1583                         cb->busaddr = bus_address;
1584                         cb->maplen = nesvnic->max_frame_size;
1585
1586                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1587                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1588                                         cpu_to_le32(nesvnic->max_frame_size);
1589                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1590                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1591                                         cpu_to_le32((u32)bus_address);
1592                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1593                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1594                         nesnic->rx_skb[nesnic->rq_head] = skb;
1595                         nesnic->rq_head++;
1596                         nesnic->rq_head &= nesnic->rq_size - 1;
1597                         atomic_dec(&nesvnic->rx_skbs_needed);
1598                         barrier();
1599                         if (++rx_wqes_posted == 255) {
1600                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1601                                 rx_wqes_posted = 0;
1602                         }
1603                 } else {
1604                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1605                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1606                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1607                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1608                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1609                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1610                                 add_timer(&nesvnic->rq_wqes_timer);
1611                         } else
1612                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1613                         break;
1614                 }
1615         } while (atomic_read(&nesvnic->rx_skbs_needed));
1616         barrier();
1617         if (rx_wqes_posted)
1618                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1619         nesnic->replenishing_rq = 0;
1620 }
1621
1622
1623 /**
1624  * nes_rq_wqes_timeout
1625  */
1626 static void nes_rq_wqes_timeout(unsigned long parm)
1627 {
1628         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1629         printk("%s: Timer fired.\n", __func__);
1630         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1631         if (atomic_read(&nesvnic->rx_skbs_needed))
1632                 nes_replenish_nic_rq(nesvnic);
1633 }
1634
1635
1636 /**
1637  * nes_init_nic_qp
1638  */
1639 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1640 {
1641         struct nes_hw_cqp_wqe *cqp_wqe;
1642         struct nes_hw_nic_sq_wqe *nic_sqe;
1643         struct nes_hw_nic_qp_context *nic_context;
1644         struct sk_buff *skb;
1645         struct nes_hw_nic_rq_wqe *nic_rqe;
1646         struct nes_vnic *nesvnic = netdev_priv(netdev);
1647         unsigned long flags;
1648         void *vmem;
1649         dma_addr_t pmem;
1650         u64 u64temp;
1651         int ret;
1652         u32 cqp_head;
1653         u32 counter;
1654         u32 wqe_count;
1655         struct nes_rskb_cb *cb;
1656         u8 jumbomode=0;
1657
1658         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1659         nesvnic->nic_mem_size = 256 +
1660                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1661                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1662                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1663                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1664                         sizeof(struct nes_hw_nic_qp_context);
1665
1666         nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
1667                                                    nesvnic->nic_mem_size,
1668                                                    &nesvnic->nic_pbase);
1669         if (!nesvnic->nic_vbase) {
1670                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1671                 return -ENOMEM;
1672         }
1673         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1674                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1675
1676         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1677                         ~(unsigned long)(256 - 1));
1678         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1679                         ~(unsigned long long)(256 - 1));
1680
1681         /* Setup the first Fragment buffers */
1682         nesvnic->nic.first_frag_vbase = vmem;
1683
1684         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1685                 nesvnic->nic.frag_paddr[counter] = pmem;
1686                 pmem += sizeof(struct nes_first_frag);
1687         }
1688
1689         /* setup the SQ */
1690         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1691
1692         nesvnic->nic.sq_vbase = (void *)vmem;
1693         nesvnic->nic.sq_pbase = pmem;
1694         nesvnic->nic.sq_head = 0;
1695         nesvnic->nic.sq_tail = 0;
1696         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1697         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1698                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1699                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1700                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1701                                 NES_NIC_SQ_WQE_COMPLETION);
1702                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1703                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1704                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1705                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1706                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1707                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1708         }
1709
1710         nesvnic->get_cqp_request = nes_get_cqp_request;
1711         nesvnic->post_cqp_request = nes_post_cqp_request;
1712         nesvnic->mcrq_mcast_filter = NULL;
1713
1714         spin_lock_init(&nesvnic->nic.rq_lock);
1715
1716         /* setup the RQ */
1717         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1718         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1719
1720
1721         nesvnic->nic.rq_vbase = vmem;
1722         nesvnic->nic.rq_pbase = pmem;
1723         nesvnic->nic.rq_head = 0;
1724         nesvnic->nic.rq_tail = 0;
1725         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1726
1727         /* setup the CQ */
1728         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1729         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1730
1731         if (nesdev->nesadapter->netdev_count > 2)
1732                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1733         else
1734                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1735
1736         nesvnic->nic_cq.cq_vbase = vmem;
1737         nesvnic->nic_cq.cq_pbase = pmem;
1738         nesvnic->nic_cq.cq_head = 0;
1739         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1740
1741         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1742
1743         /* Send CreateCQ request to CQP */
1744         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1745         cqp_head = nesdev->cqp.sq_head;
1746
1747         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1748         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1749
1750         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1751                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1752                         ((u32)nesvnic->nic_cq.cq_size << 16));
1753         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1754                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1755         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1756         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1757         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1758         u64temp = (unsigned long)&nesvnic->nic_cq;
1759         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1760         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1761                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1762         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1763         if (++cqp_head >= nesdev->cqp.sq_size)
1764                 cqp_head = 0;
1765         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1766         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1767
1768         /* Send CreateQP request to CQP */
1769         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1770         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1771                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1772                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1773         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1774                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1775                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1776         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1777                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1778         }
1779
1780         u64temp = (u64)nesvnic->nic.sq_pbase;
1781         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1782         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1783         u64temp = (u64)nesvnic->nic.rq_pbase;
1784         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1785         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1786
1787         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1788                         NES_CQP_QP_TYPE_NIC);
1789         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1790         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1791                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1792         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1793
1794         if (++cqp_head >= nesdev->cqp.sq_size)
1795                 cqp_head = 0;
1796         nesdev->cqp.sq_head = cqp_head;
1797
1798         barrier();
1799
1800         /* Ring doorbell (2 WQEs) */
1801         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1802
1803         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1804         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1805                         nesvnic->nic.qp_id);
1806
1807         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1808                         NES_EVENT_TIMEOUT);
1809         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1810                         nesvnic->nic.qp_id, ret);
1811         if (!ret) {
1812                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1813                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1814                                 nesvnic->nic_pbase);
1815                 return -EIO;
1816         }
1817
1818         /* Populate the RQ */
1819         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1820                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1821                 if (!skb) {
1822                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1823
1824                         nes_destroy_nic_qp(nesvnic);
1825                         return -ENOMEM;
1826                 }
1827
1828                 skb->dev = netdev;
1829
1830                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1831                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1832                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1833                 cb->busaddr = pmem;
1834                 cb->maplen = nesvnic->max_frame_size;
1835
1836                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1837                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1838                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1839                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1840                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1841                 nesvnic->nic.rx_skb[counter] = skb;
1842         }
1843
1844         wqe_count = NES_NIC_WQ_SIZE - 1;
1845         nesvnic->nic.rq_head = wqe_count;
1846         barrier();
1847         do {
1848                 counter = min(wqe_count, ((u32)255));
1849                 wqe_count -= counter;
1850                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1851         } while (wqe_count);
1852         init_timer(&nesvnic->rq_wqes_timer);
1853         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1854         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1855         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1856         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1857         {
1858                 nes_nic_init_timer(nesdev);
1859                 if (netdev->mtu > 1500)
1860                         jumbomode = 1;
1861                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1862         }
1863         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1864                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1865                         nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1866                         nes_destroy_nic_qp(nesvnic);
1867                 return -ENOMEM;
1868         }
1869
1870         return 0;
1871 }
1872
1873
1874 /**
1875  * nes_destroy_nic_qp
1876  */
1877 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1878 {
1879         u64 u64temp;
1880         dma_addr_t bus_address;
1881         struct nes_device *nesdev = nesvnic->nesdev;
1882         struct nes_hw_cqp_wqe *cqp_wqe;
1883         struct nes_hw_nic_sq_wqe *nic_sqe;
1884         __le16 *wqe_fragment_length;
1885         u16  wqe_fragment_index;
1886         u32 cqp_head;
1887         u32 wqm_cfg0;
1888         unsigned long flags;
1889         struct sk_buff *rx_skb;
1890         struct nes_rskb_cb *cb;
1891         int ret;
1892
1893         if (nesdev->nesadapter->allow_unaligned_fpdus)
1894                 nes_destroy_mgt(nesvnic);
1895
1896         /* clear wqe stall before destroying NIC QP */
1897         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1898         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1899
1900         /* Free remaining NIC receive buffers */
1901         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1902                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1903                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1904                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1905                         PCI_DMA_FROMDEVICE);
1906
1907                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1908                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1909         }
1910
1911         /* Free remaining NIC transmit buffers */
1912         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1913                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1914                 wqe_fragment_index = 1;
1915                 wqe_fragment_length = (__le16 *)
1916                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1917                 /* bump past the vlan tag */
1918                 wqe_fragment_length++;
1919                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1920                         u64temp = (u64)le32_to_cpu(
1921                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1922                                 wqe_fragment_index*2]);
1923                         u64temp += ((u64)le32_to_cpu(
1924                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1925                                 + wqe_fragment_index*2]))<<32;
1926                         bus_address = (dma_addr_t)u64temp;
1927                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1928                                         nesvnic->nic.first_frag_overflow)) {
1929                                 pci_unmap_single(nesdev->pcidev,
1930                                                 bus_address,
1931                                                 le16_to_cpu(wqe_fragment_length[
1932                                                         wqe_fragment_index++]),
1933                                                 PCI_DMA_TODEVICE);
1934                         }
1935                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1936                                 if (wqe_fragment_length[wqe_fragment_index]) {
1937                                         u64temp = le32_to_cpu(
1938                                                 nic_sqe->wqe_words[
1939                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1940                                                 wqe_fragment_index*2]);
1941                                         u64temp += ((u64)le32_to_cpu(
1942                                                 nic_sqe->wqe_words[
1943                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1944                                                 wqe_fragment_index*2]))<<32;
1945                                         bus_address = (dma_addr_t)u64temp;
1946                                         pci_unmap_page(nesdev->pcidev,
1947                                                         bus_address,
1948                                                         le16_to_cpu(
1949                                                         wqe_fragment_length[
1950                                                         wqe_fragment_index]),
1951                                                         PCI_DMA_TODEVICE);
1952                                 } else
1953                                         break;
1954                         }
1955                 }
1956                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1957                         dev_kfree_skb(
1958                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1959
1960                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1961                                         & (nesvnic->nic.sq_size - 1);
1962         }
1963
1964         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1965
1966         /* Destroy NIC QP */
1967         cqp_head = nesdev->cqp.sq_head;
1968         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1969         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1970
1971         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1972                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1973         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1974                 nesvnic->nic.qp_id);
1975
1976         if (++cqp_head >= nesdev->cqp.sq_size)
1977                 cqp_head = 0;
1978
1979         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1980
1981         /* Destroy NIC CQ */
1982         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1983         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1984                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1985         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1986                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1987
1988         if (++cqp_head >= nesdev->cqp.sq_size)
1989                 cqp_head = 0;
1990
1991         nesdev->cqp.sq_head = cqp_head;
1992         barrier();
1993
1994         /* Ring doorbell (2 WQEs) */
1995         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1996
1997         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1998         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1999                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2000                         cqp_head, nesdev->cqp.sq_head,
2001                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2002
2003         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2004                         NES_EVENT_TIMEOUT);
2005
2006         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2007                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2008                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2009         if (!ret) {
2010                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2011                                 nesvnic->nic.qp_id);
2012         }
2013
2014         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2015                         nesvnic->nic_pbase);
2016
2017         /* restore old wqm_cfg0 value */
2018         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2019 }
2020
2021 /**
2022  * nes_napi_isr
2023  */
2024 int nes_napi_isr(struct nes_device *nesdev)
2025 {
2026         struct nes_adapter *nesadapter = nesdev->nesadapter;
2027         u32 int_stat;
2028
2029         if (nesdev->napi_isr_ran) {
2030                 /* interrupt status has already been read in ISR */
2031                 int_stat = nesdev->int_stat;
2032         } else {
2033                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2034                 nesdev->int_stat = int_stat;
2035                 nesdev->napi_isr_ran = 1;
2036         }
2037
2038         int_stat &= nesdev->int_req;
2039         /* iff NIC, process here, else wait for DPC */
2040         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2041                 nesdev->napi_isr_ran = 0;
2042                 nes_write32(nesdev->regs + NES_INT_STAT,
2043                         (int_stat &
2044                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2045
2046                 /* Process the CEQs */
2047                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2048
2049                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2050                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2051                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2052                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2053                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2054                                 /* Enable Periodic timer interrupts */
2055                                 nesdev->int_req |= NES_INT_TIMER;
2056                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2057                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2058                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2059                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2060                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2061                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2062                         }
2063
2064                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2065                         {
2066                                 nes_nic_init_timer(nesdev);
2067                         }
2068                         /* Enable interrupts, except CEQs */
2069                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2070                 } else {
2071                         /* Enable interrupts, make sure timer is off */
2072                         nesdev->int_req &= ~NES_INT_TIMER;
2073                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2074                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2075                 }
2076                 nesdev->deepcq_count = 0;
2077                 return 1;
2078         } else {
2079                 return 0;
2080         }
2081 }
2082
2083 static void process_critical_error(struct nes_device *nesdev)
2084 {
2085         u32 debug_error;
2086         u32 nes_idx_debug_error_masks0 = 0;
2087         u16 error_module = 0;
2088
2089         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2090         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2091                         (u16)debug_error);
2092         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2093                         0x01010000 | (debug_error & 0x0000ffff));
2094         if (crit_err_count++ > 10)
2095                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2096         error_module = (u16) (debug_error & 0x1F00) >> 8;
2097         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2098                         nes_max_critical_error_count) {
2099                 printk(KERN_ERR PFX "Masking off critical error for module "
2100                         "0x%02X\n", (u16)error_module);
2101                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2102                         NES_IDX_DEBUG_ERROR_MASKS0);
2103                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2104                         nes_idx_debug_error_masks0 | (1 << error_module));
2105         }
2106 }
2107 /**
2108  * nes_dpc
2109  */
2110 void nes_dpc(unsigned long param)
2111 {
2112         struct nes_device *nesdev = (struct nes_device *)param;
2113         struct nes_adapter *nesadapter = nesdev->nesadapter;
2114         u32 counter;
2115         u32 loop_counter = 0;
2116         u32 int_status_bit;
2117         u32 int_stat;
2118         u32 timer_stat;
2119         u32 temp_int_stat;
2120         u32 intf_int_stat;
2121         u32 processed_intf_int = 0;
2122         u16 processed_timer_int = 0;
2123         u16 completion_ints = 0;
2124         u16 timer_ints = 0;
2125
2126         /* nes_debug(NES_DBG_ISR, "\n"); */
2127
2128         do {
2129                 timer_stat = 0;
2130                 if (nesdev->napi_isr_ran) {
2131                         nesdev->napi_isr_ran = 0;
2132                         int_stat = nesdev->int_stat;
2133                 } else
2134                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2135                 if (processed_intf_int != 0)
2136                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2137                 else
2138                         int_stat &= nesdev->int_req;
2139                 if (processed_timer_int == 0) {
2140                         processed_timer_int = 1;
2141                         if (int_stat & NES_INT_TIMER) {
2142                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2143                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2144                                         int_stat &= ~NES_INT_TIMER;
2145                                 }
2146                         }
2147                 } else {
2148                         int_stat &= ~NES_INT_TIMER;
2149                 }
2150
2151                 if (int_stat) {
2152                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2153                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2154                                 /* Ack the interrupts */
2155                                 nes_write32(nesdev->regs+NES_INT_STAT,
2156                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2157                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2158                         }
2159
2160                         temp_int_stat = int_stat;
2161                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2162                                 if (int_stat & int_status_bit) {
2163                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2164                                         temp_int_stat &= ~int_status_bit;
2165                                         completion_ints = 1;
2166                                 }
2167                                 if (!(temp_int_stat & 0x0000ffff))
2168                                         break;
2169                                 int_status_bit <<= 1;
2170                         }
2171
2172                         /* Process the AEQ for this pci function */
2173                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2174                         if (int_stat & int_status_bit) {
2175                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2176                         }
2177
2178                         /* Process the MAC interrupt for this pci function */
2179                         int_status_bit = 1 << (24 + nesdev->mac_index);
2180                         if (int_stat & int_status_bit) {
2181                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2182                         }
2183
2184                         if (int_stat & NES_INT_TIMER) {
2185                                 if (timer_stat & nesdev->timer_int_req) {
2186                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2187                                                         (timer_stat & nesdev->timer_int_req) |
2188                                                         ~(nesdev->nesadapter->timer_int_req));
2189                                         timer_ints = 1;
2190                                 }
2191                         }
2192
2193                         if (int_stat & NES_INT_INTF) {
2194                                 processed_intf_int = 1;
2195                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2196                                 intf_int_stat &= nesdev->intf_int_req;
2197                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2198                                         process_critical_error(nesdev);
2199                                 }
2200                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2201                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2202                                         BUG();
2203                                 }
2204                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2205                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2206                                         BUG();
2207                                 }
2208                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2209                         }
2210
2211                         if (int_stat & NES_INT_TSW) {
2212                         }
2213                 }
2214                 /* Don't use the interface interrupt bit stay in loop */
2215                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2216                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2217         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2218
2219         if (timer_ints == 1) {
2220                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2221                         if (completion_ints == 0) {
2222                                 nesdev->timer_only_int_count++;
2223                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2224                                         nesdev->timer_only_int_count = 0;
2225                                         nesdev->int_req &= ~NES_INT_TIMER;
2226                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2227                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2228                                 } else {
2229                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2230                                 }
2231                         } else {
2232                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2233                                 {
2234                                         nes_nic_init_timer(nesdev);
2235                                 }
2236                                 nesdev->timer_only_int_count = 0;
2237                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2238                         }
2239                 } else {
2240                         nesdev->timer_only_int_count = 0;
2241                         nesdev->int_req &= ~NES_INT_TIMER;
2242                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2243                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2244                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2245                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2246                 }
2247         } else {
2248                 if ( (completion_ints == 1) &&
2249                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2250                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2251                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2252                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2253                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2254                         nesdev->timer_only_int_count = 0;
2255                         nesdev->int_req |= NES_INT_TIMER;
2256                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2257                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2258                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2259                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2260                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2261                 } else {
2262                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2263                 }
2264         }
2265         nesdev->deepcq_count = 0;
2266 }
2267
2268
2269 /**
2270  * nes_process_ceq
2271  */
2272 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2273 {
2274         u64 u64temp;
2275         struct nes_hw_cq *cq;
2276         u32 head;
2277         u32 ceq_size;
2278
2279         /* nes_debug(NES_DBG_CQ, "\n"); */
2280         head = ceq->ceq_head;
2281         ceq_size = ceq->ceq_size;
2282
2283         do {
2284                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2285                                 NES_CEQE_VALID) {
2286                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2287                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2288                         u64temp <<= 1;
2289                         cq = *((struct nes_hw_cq **)&u64temp);
2290                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2291                         barrier();
2292                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2293
2294                         /* call the event handler */
2295                         cq->ce_handler(nesdev, cq);
2296
2297                         if (++head >= ceq_size)
2298                                 head = 0;
2299                 } else {
2300                         break;
2301                 }
2302
2303         } while (1);
2304
2305         ceq->ceq_head = head;
2306 }
2307
2308
2309 /**
2310  * nes_process_aeq
2311  */
2312 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2313 {
2314         /* u64 u64temp; */
2315         u32 head;
2316         u32 aeq_size;
2317         u32 aeqe_misc;
2318         u32 aeqe_cq_id;
2319         struct nes_hw_aeqe volatile *aeqe;
2320
2321         head = aeq->aeq_head;
2322         aeq_size = aeq->aeq_size;
2323
2324         do {
2325                 aeqe = &aeq->aeq_vbase[head];
2326                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2327                         break;
2328                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2329                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2330                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2331                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2332                                 /* dealing with an accelerated QP related AE */
2333                                 /*
2334                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2335                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2336                                  */
2337                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2338                         } else {
2339                                 /* TODO: dealing with a CQP related AE */
2340                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2341                                                 (u16)(aeqe_misc >> 16));
2342                         }
2343                 }
2344
2345                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2346
2347                 if (++head >= aeq_size)
2348                         head = 0;
2349
2350                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2351         }
2352         while (1);
2353         aeq->aeq_head = head;
2354 }
2355
2356 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2357 {
2358         struct nes_adapter *nesadapter = nesdev->nesadapter;
2359         u32 reset_value;
2360         u32 i=0;
2361         u32 u32temp;
2362
2363         if (nesadapter->hw_rev == NE020_REV) {
2364                 return;
2365         }
2366         mh_detected++;
2367
2368         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2369
2370         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2371                 reset_value |= 0x0000001d;
2372         else
2373                 reset_value |= 0x0000002d;
2374
2375         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2376                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2377                         nesadapter->link_interrupt_count[0] = 0;
2378                         nesadapter->link_interrupt_count[1] = 0;
2379                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2380                         if (0x00000040 & u32temp)
2381                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2382                         else
2383                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2384
2385                         reset_value |= 0x0000003d;
2386                 }
2387                 nesadapter->link_interrupt_count[mac_index] = 0;
2388         }
2389
2390         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2391
2392         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2393                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2394
2395         if (0x0000003d == (reset_value & 0x0000003d)) {
2396                 u32 pcs_control_status0, pcs_control_status1;
2397
2398                 for (i = 0; i < 10; i++) {
2399                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2400                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2401                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2402                              && (pcs_control_status0 & 0x00100000))
2403                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2404                                 && (pcs_control_status1 & 0x00100000)))
2405                                 continue;
2406                         else
2407                                 break;
2408                 }
2409                 if (10 == i) {
2410                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2411                         if (0x00000040 & u32temp)
2412                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2413                         else
2414                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2415
2416                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2417
2418                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2419                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2420                 }
2421         }
2422 }
2423
2424 /**
2425  * nes_process_mac_intr
2426  */
2427 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2428 {
2429         unsigned long flags;
2430         u32 pcs_control_status;
2431         struct nes_adapter *nesadapter = nesdev->nesadapter;
2432         struct nes_vnic *nesvnic;
2433         u32 mac_status;
2434         u32 mac_index = nesdev->mac_index;
2435         u32 u32temp;
2436         u16 phy_data;
2437         u16 temp_phy_data;
2438         u32 pcs_val  = 0x0f0f0000;
2439         u32 pcs_mask = 0x0f1f0000;
2440         u32 cdr_ctrl;
2441
2442         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2443         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2444                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2445                 return;
2446         }
2447         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2448
2449         /* ack the MAC interrupt */
2450         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2451         /* Clear the interrupt */
2452         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2453
2454         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2455
2456         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2457                 nesdev->link_status_interrupts++;
2458                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2459                         nes_reset_link(nesdev, mac_index);
2460
2461                 /* read the PHY interrupt status register */
2462                 if ((nesadapter->OneG_Mode) &&
2463                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2464                         do {
2465                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2466                                                 nesadapter->phy_index[mac_index], &phy_data);
2467                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2468                                                 nesadapter->phy_index[mac_index], phy_data);
2469                         } while (phy_data&0x8000);
2470
2471                         temp_phy_data = 0;
2472                         do {
2473                                 nes_read_1G_phy_reg(nesdev, 0x11,
2474                                                 nesadapter->phy_index[mac_index], &phy_data);
2475                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2476                                                 nesadapter->phy_index[mac_index], phy_data);
2477                                 if (temp_phy_data == phy_data)
2478                                         break;
2479                                 temp_phy_data = phy_data;
2480                         } while (1);
2481
2482                         nes_read_1G_phy_reg(nesdev, 0x1e,
2483                                         nesadapter->phy_index[mac_index], &phy_data);
2484                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2485                                         nesadapter->phy_index[mac_index], phy_data);
2486
2487                         nes_read_1G_phy_reg(nesdev, 1,
2488                                         nesadapter->phy_index[mac_index], &phy_data);
2489                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2490                                         nesadapter->phy_index[mac_index], phy_data);
2491
2492                         if (temp_phy_data & 0x1000) {
2493                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2494                                 phy_data = 4;
2495                         } else {
2496                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2497                         }
2498                 }
2499                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2500                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2501                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2502
2503                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2504                         switch (mac_index) {
2505                         case 1:
2506                         case 3:
2507                                 pcs_control_status = nes_read_indexed(nesdev,
2508                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2509                                 break;
2510                         default:
2511                                 pcs_control_status = nes_read_indexed(nesdev,
2512                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2513                                 break;
2514                         }
2515                 } else {
2516                         pcs_control_status = nes_read_indexed(nesdev,
2517                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2518                         pcs_control_status = nes_read_indexed(nesdev,
2519                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2520                 }
2521
2522                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2523                                 mac_index, pcs_control_status);
2524                 if ((nesadapter->OneG_Mode) &&
2525                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2526                         u32temp = 0x01010000;
2527                         if (nesadapter->port_count > 2) {
2528                                 u32temp |= 0x02020000;
2529                         }
2530                         if ((pcs_control_status & u32temp)!= u32temp) {
2531                                 phy_data = 0;
2532                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2533                         }
2534                 } else {
2535                         switch (nesadapter->phy_type[mac_index]) {
2536                         case NES_PHY_TYPE_ARGUS:
2537                         case NES_PHY_TYPE_SFP_D:
2538                         case NES_PHY_TYPE_KR:
2539                                 /* clear the alarms */
2540                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2541                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2542                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2543                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2544                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2545                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2546                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2547                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2548                                 /* check link status */
2549                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2550                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2551
2552                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2553                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2554                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2555                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2556
2557                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2558
2559                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2560                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2561                                 break;
2562
2563                         case NES_PHY_TYPE_PUMA_1G:
2564                                 if (mac_index < 2)
2565                                         pcs_val = pcs_mask = 0x01010000;
2566                                 else
2567                                         pcs_val = pcs_mask = 0x02020000;
2568                                 /* fall through */
2569                         default:
2570                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2571                                 break;
2572                         }
2573                 }
2574
2575                 if (phy_data & 0x0004) {
2576                         if (wide_ppm_offset &&
2577                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2578                             (nesadapter->hw_rev != NE020_REV)) {
2579                                 cdr_ctrl = nes_read_indexed(nesdev,
2580                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2581                                                             mac_index * 0x200);
2582                                 nes_write_indexed(nesdev,
2583                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2584                                                   mac_index * 0x200,
2585                                                   cdr_ctrl | 0x000F0000);
2586                         }
2587                         nesadapter->mac_link_down[mac_index] = 0;
2588                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2589                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2590                                                 nesvnic->linkup);
2591                                 if (nesvnic->linkup == 0) {
2592                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2593                                                         nesvnic->netdev->name, nesvnic->netdev);
2594                                         if (netif_queue_stopped(nesvnic->netdev))
2595                                                 netif_start_queue(nesvnic->netdev);
2596                                         nesvnic->linkup = 1;
2597                                         netif_carrier_on(nesvnic->netdev);
2598
2599                                         spin_lock(&nesvnic->port_ibevent_lock);
2600                                         if (nesvnic->of_device_registered) {
2601                                                 if (nesdev->iw_status == 0) {
2602                                                         nesdev->iw_status = 1;
2603                                                         nes_port_ibevent(nesvnic);
2604                                                 }
2605                                         }
2606                                         spin_unlock(&nesvnic->port_ibevent_lock);
2607                                 }
2608                         }
2609                 } else {
2610                         if (wide_ppm_offset &&
2611                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2612                             (nesadapter->hw_rev != NE020_REV)) {
2613                                 cdr_ctrl = nes_read_indexed(nesdev,
2614                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2615                                                             mac_index * 0x200);
2616                                 nes_write_indexed(nesdev,
2617                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2618                                                   mac_index * 0x200,
2619                                                   cdr_ctrl & 0xFFF0FFFF);
2620                         }
2621                         nesadapter->mac_link_down[mac_index] = 1;
2622                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2623                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2624                                                 nesvnic->linkup);
2625                                 if (nesvnic->linkup == 1) {
2626                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2627                                                         nesvnic->netdev->name, nesvnic->netdev);
2628                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2629                                                 netif_stop_queue(nesvnic->netdev);
2630                                         nesvnic->linkup = 0;
2631                                         netif_carrier_off(nesvnic->netdev);
2632
2633                                         spin_lock(&nesvnic->port_ibevent_lock);
2634                                         if (nesvnic->of_device_registered) {
2635                                                 if (nesdev->iw_status == 1) {
2636                                                         nesdev->iw_status = 0;
2637                                                         nes_port_ibevent(nesvnic);
2638                                                 }
2639                                         }
2640                                         spin_unlock(&nesvnic->port_ibevent_lock);
2641                                 }
2642                         }
2643                 }
2644                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2645                         nesdev->link_recheck = 1;
2646                         mod_delayed_work(system_wq, &nesdev->work,
2647                                          NES_LINK_RECHECK_DELAY);
2648                 }
2649         }
2650
2651         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2652
2653         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2654 }
2655
2656 void nes_recheck_link_status(struct work_struct *work)
2657 {
2658         unsigned long flags;
2659         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2660         struct nes_adapter *nesadapter = nesdev->nesadapter;
2661         struct nes_vnic *nesvnic;
2662         u32 mac_index = nesdev->mac_index;
2663         u16 phy_data;
2664         u16 temp_phy_data;
2665
2666         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2667
2668         /* check link status */
2669         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2670         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2671
2672         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2673         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2674         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2675         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2676
2677         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2678
2679         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2680                 __func__, phy_data,
2681                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2682
2683         if (phy_data & 0x0004) {
2684                 nesadapter->mac_link_down[mac_index] = 0;
2685                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2686                         if (nesvnic->linkup == 0) {
2687                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2688                                                 nesvnic->netdev->name, nesvnic->netdev);
2689                                 if (netif_queue_stopped(nesvnic->netdev))
2690                                         netif_start_queue(nesvnic->netdev);
2691                                 nesvnic->linkup = 1;
2692                                 netif_carrier_on(nesvnic->netdev);
2693
2694                                 spin_lock(&nesvnic->port_ibevent_lock);
2695                                 if (nesvnic->of_device_registered) {
2696                                         if (nesdev->iw_status == 0) {
2697                                                 nesdev->iw_status = 1;
2698                                                 nes_port_ibevent(nesvnic);
2699                                         }
2700                                 }
2701                                 spin_unlock(&nesvnic->port_ibevent_lock);
2702                         }
2703                 }
2704
2705         } else {
2706                 nesadapter->mac_link_down[mac_index] = 1;
2707                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2708                         if (nesvnic->linkup == 1) {
2709                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2710                                                 nesvnic->netdev->name, nesvnic->netdev);
2711                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2712                                         netif_stop_queue(nesvnic->netdev);
2713                                 nesvnic->linkup = 0;
2714                                 netif_carrier_off(nesvnic->netdev);
2715
2716                                 spin_lock(&nesvnic->port_ibevent_lock);
2717                                 if (nesvnic->of_device_registered) {
2718                                         if (nesdev->iw_status == 1) {
2719                                                 nesdev->iw_status = 0;
2720                                                 nes_port_ibevent(nesvnic);
2721                                         }
2722                                 }
2723                                 spin_unlock(&nesvnic->port_ibevent_lock);
2724                         }
2725                 }
2726         }
2727         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2728                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2729         else
2730                 nesdev->link_recheck = 0;
2731
2732         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2733 }
2734
2735
2736 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2737 {
2738         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2739
2740         napi_schedule(&nesvnic->napi);
2741 }
2742
2743
2744 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2745 * getting out of nic_ce_handler
2746 */
2747 #define MAX_RQES_TO_PROCESS     384
2748
2749 /**
2750  * nes_nic_ce_handler
2751  */
2752 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2753 {
2754         u64 u64temp;
2755         dma_addr_t bus_address;
2756         struct nes_hw_nic *nesnic;
2757         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2758         struct nes_adapter *nesadapter = nesdev->nesadapter;
2759         struct nes_hw_nic_rq_wqe *nic_rqe;
2760         struct nes_hw_nic_sq_wqe *nic_sqe;
2761         struct sk_buff *skb;
2762         struct sk_buff *rx_skb;
2763         struct nes_rskb_cb *cb;
2764         __le16 *wqe_fragment_length;
2765         u32 head;
2766         u32 cq_size;
2767         u32 rx_pkt_size;
2768         u32 cqe_count=0;
2769         u32 cqe_errv;
2770         u32 cqe_misc;
2771         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2772         u16 vlan_tag;
2773         u16 pkt_type;
2774         u16 rqes_processed = 0;
2775         u8 sq_cqes = 0;
2776
2777         head = cq->cq_head;
2778         cq_size = cq->cq_size;
2779         cq->cqes_pending = 1;
2780         do {
2781                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2782                                 NES_NIC_CQE_VALID) {
2783                         nesnic = &nesvnic->nic;
2784                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2785                         if (cqe_misc & NES_NIC_CQE_SQ) {
2786                                 sq_cqes++;
2787                                 wqe_fragment_index = 1;
2788                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2789                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2790                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2791                                 /* bump past the vlan tag */
2792                                 wqe_fragment_length++;
2793                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2794                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2795                                                         wqe_fragment_index * 2]);
2796                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2797                                                         wqe_fragment_index * 2])) << 32;
2798                                         bus_address = (dma_addr_t)u64temp;
2799                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2800                                                 pci_unmap_single(nesdev->pcidev,
2801                                                                 bus_address,
2802                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2803                                                                 PCI_DMA_TODEVICE);
2804                                         }
2805                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2806                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2807                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2808                                                                                 wqe_fragment_index * 2]);
2809                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2810                                                                                 + wqe_fragment_index * 2])) <<32;
2811                                                         bus_address = (dma_addr_t)u64temp;
2812                                                         pci_unmap_page(nesdev->pcidev,
2813                                                                         bus_address,
2814                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2815                                                                         PCI_DMA_TODEVICE);
2816                                                 } else
2817                                                         break;
2818                                         }
2819                                 }
2820                                 if (skb)
2821                                         dev_kfree_skb_any(skb);
2822                                 nesnic->sq_tail++;
2823                                 nesnic->sq_tail &= nesnic->sq_size-1;
2824                                 if (sq_cqes > 128) {
2825                                         barrier();
2826                                         /* restart the queue if it had been stopped */
2827                                         if (netif_queue_stopped(nesvnic->netdev))
2828                                                 netif_wake_queue(nesvnic->netdev);
2829                                         sq_cqes = 0;
2830                                 }
2831                         } else {
2832                                 rqes_processed ++;
2833
2834                                 cq->rx_cqes_completed++;
2835                                 cq->rx_pkts_indicated++;
2836                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2837                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2838                                 /* Get the skb */
2839                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2840                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2841                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2842                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2843                                 pci_unmap_single(nesdev->pcidev, bus_address,
2844                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2845                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2846                                 cb->busaddr = 0;
2847                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2848                                 /* rx_skb->len = rx_pkt_size; */
2849                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2850                                 skb_put(rx_skb, rx_pkt_size);
2851                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2852                                 nesnic->rq_tail++;
2853                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2854
2855                                 atomic_inc(&nesvnic->rx_skbs_needed);
2856                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2857                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2858                                                         cq->cq_number | (cqe_count << 16));
2859                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2860                                         nesdev->currcq_count += cqe_count;
2861                                         cqe_count = 0;
2862                                         nes_replenish_nic_rq(nesvnic);
2863                                 }
2864                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2865                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2866                                 rx_skb->ip_summed = CHECKSUM_NONE;
2867
2868                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2869                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2870                                         if ((cqe_errv &
2871                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2872                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2873                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2874                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2875                                         } else
2876                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2877                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2878                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2879
2880                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2881                                         if ((cqe_errv &
2882                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2883                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2884                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2885                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2886                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2887                                                                   nesvnic->netdev->name); */
2888                                                 }
2889                                         } else
2890                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2891                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2892                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2893                                         }
2894                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2895                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2896
2897                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2898                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2899                                                 rx_skb = NULL;
2900                                 }
2901                                 if (rx_skb == NULL)
2902                                         goto skip_rx_indicate0;
2903
2904
2905                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2906                                         vlan_tag = (u16)(le32_to_cpu(
2907                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2908                                                         >> 16);
2909                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2910                                                         nesvnic->netdev->name, vlan_tag);
2911
2912                                         __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
2913                                 }
2914                                 napi_gro_receive(&nesvnic->napi, rx_skb);
2915
2916 skip_rx_indicate0:
2917                                 ;
2918                                 /* nesvnic->netstats.rx_packets++; */
2919                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2920                         }
2921
2922                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2923                         /* Accounting... */
2924                         cqe_count++;
2925                         if (++head >= cq_size)
2926                                 head = 0;
2927                         if (cqe_count == 255) {
2928                                 /* Replenish Nic CQ */
2929                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2930                                                 cq->cq_number | (cqe_count << 16));
2931                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2932                                 nesdev->currcq_count += cqe_count;
2933                                 cqe_count = 0;
2934                         }
2935
2936                         if (cq->rx_cqes_completed >= nesvnic->budget)
2937                                 break;
2938                 } else {
2939                         cq->cqes_pending = 0;
2940                         break;
2941                 }
2942
2943         } while (1);
2944
2945         if (sq_cqes) {
2946                 barrier();
2947                 /* restart the queue if it had been stopped */
2948                 if (netif_queue_stopped(nesvnic->netdev))
2949                         netif_wake_queue(nesvnic->netdev);
2950         }
2951         cq->cq_head = head;
2952         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2953                         cq->cq_number, cqe_count, cq->cq_head); */
2954         cq->cqe_allocs_pending = cqe_count;
2955         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2956         {
2957                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2958                 nesdev->currcq_count += cqe_count;
2959                 nes_nic_tune_timer(nesdev);
2960         }
2961         if (atomic_read(&nesvnic->rx_skbs_needed))
2962                 nes_replenish_nic_rq(nesvnic);
2963 }
2964
2965
2966
2967 /**
2968  * nes_cqp_ce_handler
2969  */
2970 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2971 {
2972         u64 u64temp;
2973         unsigned long flags;
2974         struct nes_hw_cqp *cqp = NULL;
2975         struct nes_cqp_request *cqp_request;
2976         struct nes_hw_cqp_wqe *cqp_wqe;
2977         u32 head;
2978         u32 cq_size;
2979         u32 cqe_count=0;
2980         u32 error_code;
2981         u32 opcode;
2982         u32 ctx_index;
2983         /* u32 counter; */
2984
2985         head = cq->cq_head;
2986         cq_size = cq->cq_size;
2987
2988         do {
2989                 /* process the CQE */
2990                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2991                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2992
2993                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
2994                 if (opcode & NES_CQE_VALID) {
2995                         cqp = &nesdev->cqp;
2996
2997                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2998                         if (error_code) {
2999                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3000                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3001                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3002                                                 (u16)(error_code >> 16),
3003                                                 (u16)error_code);
3004                         }
3005
3006                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3007                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3008                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3009                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3010
3011                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3012                         if (cqp_request) {
3013                                 if (cqp_request->waiting) {
3014                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3015                                         cqp_request->major_code = (u16)(error_code >> 16);
3016                                         cqp_request->minor_code = (u16)error_code;
3017                                         barrier();
3018                                         cqp_request->request_done = 1;
3019                                         wake_up(&cqp_request->waitq);
3020                                         nes_put_cqp_request(nesdev, cqp_request);
3021                                 } else {
3022                                         if (cqp_request->callback)
3023                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3024                                         nes_free_cqp_request(nesdev, cqp_request);
3025                                 }
3026                         } else {
3027                                 wake_up(&nesdev->cqp.waitq);
3028                         }
3029
3030                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3031                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3032                         if (++cqp->sq_tail >= cqp->sq_size)
3033                                 cqp->sq_tail = 0;
3034
3035                         /* Accounting... */
3036                         cqe_count++;
3037                         if (++head >= cq_size)
3038                                 head = 0;
3039                 } else {
3040                         break;
3041                 }
3042         } while (1);
3043         cq->cq_head = head;
3044
3045         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3046         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3047                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3048                         (nesdev->cqp.sq_size - 1)) != 1)) {
3049                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3050                                 struct nes_cqp_request, list);
3051                 list_del_init(&cqp_request->list);
3052                 head = nesdev->cqp.sq_head++;
3053                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3054                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3055                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3056                 barrier();
3057
3058                 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3059                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3060                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3061                 else
3062                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3063                 cqp_wqe->wqe_words[ctx_index] =
3064                         cpu_to_le32((u32)((unsigned long)cqp_request));
3065                 cqp_wqe->wqe_words[ctx_index + 1] =
3066                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3067                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3068                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3069                 /* Ring doorbell (1 WQEs) */
3070                 barrier();
3071                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3072         }
3073         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3074
3075         /* Arm the CCQ */
3076         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3077                         cq->cq_number);
3078         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3079 }
3080
3081 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3082 {
3083         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3084                 /* skip over ethernet header */
3085                 pkt += ETH_HLEN;
3086
3087                 /* Skip over IP and TCP headers */
3088                 pkt += 4 * (pkt[0] & 0x0f);
3089                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3090         }
3091         return pkt;
3092 }
3093
3094 /* Determine if incoming error pkt is rdma layer */
3095 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3096 {
3097         u8 *pkt;
3098         u16 *mpa;
3099         u32 opcode = 0xffffffff;
3100
3101         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3102                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3103                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3104                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3105         }
3106
3107         return opcode;
3108 }
3109
3110 /* Build iWARP terminate header */
3111 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3112 {
3113         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3114         u16 ddp_seg_len;
3115         int copy_len = 0;
3116         u8 is_tagged = 0;
3117         u8 flush_code = 0;
3118         struct nes_terminate_hdr *termhdr;
3119
3120         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3121         memset(termhdr, 0, 64);
3122
3123         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3124
3125                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3126                 pkt = locate_mpa(pkt, aeq_info);
3127                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3128                 if (ddp_seg_len) {
3129                         copy_len = 2;
3130                         termhdr->hdrct = DDP_LEN_FLAG;
3131                         if (pkt[2] & 0x80) {
3132                                 is_tagged = 1;
3133                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3134                                         copy_len += TERM_DDP_LEN_TAGGED;
3135                                         termhdr->hdrct |= DDP_HDR_FLAG;
3136                                 }
3137                         } else {
3138                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3139                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3140                                         termhdr->hdrct |= DDP_HDR_FLAG;
3141                                 }
3142
3143                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3144                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3145                                                 copy_len += TERM_RDMA_LEN;
3146                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3147                                         }
3148                                 }
3149                         }
3150                 }
3151         }
3152
3153         switch (async_event_id) {
3154         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3155                 switch (iwarp_opcode(nesqp, aeq_info)) {
3156                 case IWARP_OPCODE_WRITE:
3157                         flush_code = IB_WC_LOC_PROT_ERR;
3158                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3159                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3160                         break;
3161                 default:
3162                         flush_code = IB_WC_REM_ACCESS_ERR;
3163                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3164                         termhdr->error_code = RDMAP_INV_STAG;
3165                 }
3166                 break;
3167         case NES_AEQE_AEID_AMP_INVALID_STAG:
3168                 flush_code = IB_WC_REM_ACCESS_ERR;
3169                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3170                 termhdr->error_code = RDMAP_INV_STAG;
3171                 break;
3172         case NES_AEQE_AEID_AMP_BAD_QP:
3173                 flush_code = IB_WC_LOC_QP_OP_ERR;
3174                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3175                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3176                 break;
3177         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3178         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3179                 switch (iwarp_opcode(nesqp, aeq_info)) {
3180                 case IWARP_OPCODE_SEND_INV:
3181                 case IWARP_OPCODE_SEND_SE_INV:
3182                         flush_code = IB_WC_REM_OP_ERR;
3183                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3184                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3185                         break;
3186                 default:
3187                         flush_code = IB_WC_REM_ACCESS_ERR;
3188                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3189                         termhdr->error_code = RDMAP_INV_STAG;
3190                 }
3191                 break;
3192         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3193                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3194                         flush_code = IB_WC_LOC_PROT_ERR;
3195                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3196                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3197                 } else {
3198                         flush_code = IB_WC_REM_ACCESS_ERR;
3199                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3200                         termhdr->error_code = RDMAP_INV_BOUNDS;
3201                 }
3202                 break;
3203         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3204         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3205         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3206                 flush_code = IB_WC_REM_ACCESS_ERR;
3207                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3208                 termhdr->error_code = RDMAP_ACCESS;
3209                 break;
3210         case NES_AEQE_AEID_AMP_TO_WRAP:
3211                 flush_code = IB_WC_REM_ACCESS_ERR;
3212                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3213                 termhdr->error_code = RDMAP_TO_WRAP;
3214                 break;
3215         case NES_AEQE_AEID_AMP_BAD_PD:
3216                 switch (iwarp_opcode(nesqp, aeq_info)) {
3217                 case IWARP_OPCODE_WRITE:
3218                         flush_code = IB_WC_LOC_PROT_ERR;
3219                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3220                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3221                         break;
3222                 case IWARP_OPCODE_SEND_INV:
3223                 case IWARP_OPCODE_SEND_SE_INV:
3224                         flush_code = IB_WC_REM_ACCESS_ERR;
3225                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3226                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3227                         break;
3228                 default:
3229                         flush_code = IB_WC_REM_ACCESS_ERR;
3230                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3231                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3232                 }
3233                 break;
3234         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3235                 flush_code = IB_WC_LOC_LEN_ERR;
3236                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3237                 termhdr->error_code = MPA_MARKER;
3238                 break;
3239         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3240                 flush_code = IB_WC_GENERAL_ERR;
3241                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3242                 termhdr->error_code = MPA_CRC;
3243                 break;
3244         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3245         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3246                 flush_code = IB_WC_LOC_LEN_ERR;
3247                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3248                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3249                 break;
3250         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3251         case NES_AEQE_AEID_DDP_NO_L_BIT:
3252                 flush_code = IB_WC_FATAL_ERR;
3253                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3254                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3255                 break;
3256         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3257         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3258                 flush_code = IB_WC_GENERAL_ERR;
3259                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3260                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3261                 break;
3262         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3263                 flush_code = IB_WC_LOC_LEN_ERR;
3264                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3265                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3266                 break;
3267         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3268                 flush_code = IB_WC_GENERAL_ERR;
3269                 if (is_tagged) {
3270                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3271                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3272                 } else {
3273                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3274                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3275                 }
3276                 break;
3277         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3278                 flush_code = IB_WC_GENERAL_ERR;
3279                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3280                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3281                 break;
3282         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3283                 flush_code = IB_WC_REM_OP_ERR;
3284                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3285                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3286                 break;
3287         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3288                 flush_code = IB_WC_GENERAL_ERR;
3289                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3290                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3291                 break;
3292         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3293                 flush_code = IB_WC_GENERAL_ERR;
3294                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3295                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3296                 break;
3297         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3298                 flush_code = IB_WC_LOC_QP_OP_ERR;
3299                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3300                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3301                 break;
3302         default:
3303                 flush_code = IB_WC_FATAL_ERR;
3304                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3305                 termhdr->error_code = RDMAP_UNSPECIFIED;
3306                 break;
3307         }
3308
3309         if (copy_len)
3310                 memcpy(termhdr + 1, pkt, copy_len);
3311
3312         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3313                 if (aeq_info & NES_AEQE_SQ)
3314                         nesqp->term_sq_flush_code = flush_code;
3315                 else
3316                         nesqp->term_rq_flush_code = flush_code;
3317         }
3318
3319         return sizeof(struct nes_terminate_hdr) + copy_len;
3320 }
3321
3322 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3323                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3324 {
3325         u64 context;
3326         unsigned long flags;
3327         u32 aeq_info;
3328         u16 async_event_id;
3329         u8 tcp_state;
3330         u8 iwarp_state;
3331         u32 termlen = 0;
3332         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3333                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3334         struct nes_adapter *nesadapter = nesdev->nesadapter;
3335
3336         if (nesqp->term_flags & NES_TERM_SENT)
3337                 return; /* Sanity check */
3338
3339         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3340         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3341         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3342         async_event_id = (u16)aeq_info;
3343
3344         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3345                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3346         if (!context) {
3347                 WARN_ON(!context);
3348                 return;
3349         }
3350
3351         nesqp = (struct nes_qp *)(unsigned long)context;
3352         spin_lock_irqsave(&nesqp->lock, flags);
3353         nesqp->hw_iwarp_state = iwarp_state;
3354         nesqp->hw_tcp_state = tcp_state;
3355         nesqp->last_aeq = async_event_id;
3356         nesqp->terminate_eventtype = eventtype;
3357         spin_unlock_irqrestore(&nesqp->lock, flags);
3358
3359         if (nesadapter->send_term_ok)
3360                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3361         else
3362                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3363
3364         if (!nesdev->iw_status)  {
3365                 nesqp->term_flags = NES_TERM_DONE;
3366                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3367                 nes_cm_disconn(nesqp);
3368         } else {
3369                 nes_terminate_start_timer(nesqp);
3370                 nesqp->term_flags |= NES_TERM_SENT;
3371                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3372         }
3373 }
3374
3375 static void nes_terminate_send_fin(struct nes_device *nesdev,
3376                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3377 {
3378         u32 aeq_info;
3379         u16 async_event_id;
3380         u8 tcp_state;
3381         u8 iwarp_state;
3382         unsigned long flags;
3383
3384         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3385         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3386         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3387         async_event_id = (u16)aeq_info;
3388
3389         spin_lock_irqsave(&nesqp->lock, flags);
3390         nesqp->hw_iwarp_state = iwarp_state;
3391         nesqp->hw_tcp_state = tcp_state;
3392         nesqp->last_aeq = async_event_id;
3393         spin_unlock_irqrestore(&nesqp->lock, flags);
3394
3395         /* Send the fin only */
3396         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3397                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3398 }
3399
3400 /* Cleanup after a terminate sent or received */
3401 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3402 {
3403         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3404         unsigned long flags;
3405         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3406         struct nes_device *nesdev = nesvnic->nesdev;
3407         u8 first_time = 0;
3408
3409         spin_lock_irqsave(&nesqp->lock, flags);
3410         if (nesqp->hte_added) {
3411                 nesqp->hte_added = 0;
3412                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3413         }
3414
3415         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3416         nesqp->term_flags |= NES_TERM_DONE;
3417         spin_unlock_irqrestore(&nesqp->lock, flags);
3418
3419         /* Make sure we go through this only once */
3420         if (first_time) {
3421                 if (timeout_occurred == 0)
3422                         del_timer(&nesqp->terminate_timer);
3423                 else
3424                         next_iwarp_state |= NES_CQP_QP_RESET;
3425
3426                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3427                 nes_cm_disconn(nesqp);
3428         }
3429 }
3430
3431 static void nes_terminate_received(struct nes_device *nesdev,
3432                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3433 {
3434         u32 aeq_info;
3435         u8 *pkt;
3436         u32 *mpa;
3437         u8 ddp_ctl;
3438         u8 rdma_ctl;
3439         u16 aeq_id = 0;
3440
3441         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3442         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3443                 /* Terminate is not a performance path so the silicon */
3444                 /* did not validate the frame - do it now */
3445                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3446                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3447                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3448                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3449                 if ((ddp_ctl & 0xc0) != 0x40)
3450                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3451                 else if ((ddp_ctl & 0x03) != 1)
3452                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3453                 else if (be32_to_cpu(mpa[2]) != 2)
3454                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3455                 else if (be32_to_cpu(mpa[3]) != 1)
3456                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3457                 else if (be32_to_cpu(mpa[4]) != 0)
3458                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3459                 else if ((rdma_ctl & 0xc0) != 0x40)
3460                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3461
3462                 if (aeq_id) {
3463                         /* Bad terminate recvd - send back a terminate */
3464                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3465                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3466                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3467                         return;
3468                 }
3469         }
3470
3471         nesqp->term_flags |= NES_TERM_RCVD;
3472         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3473         nes_terminate_start_timer(nesqp);
3474         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3475 }
3476
3477 /* Timeout routine in case terminate fails to complete */
3478 void nes_terminate_timeout(unsigned long context)
3479 {
3480         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3481
3482         nes_terminate_done(nesqp, 1);
3483 }
3484
3485 /* Set a timer in case hw cannot complete the terminate sequence */
3486 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3487 {
3488         mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3489 }
3490
3491 /**
3492  * nes_process_iwarp_aeqe
3493  */
3494 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3495                                    struct nes_hw_aeqe *aeqe)
3496 {
3497         u64 context;
3498         unsigned long flags;
3499         struct nes_qp *nesqp;
3500         struct nes_hw_cq *hw_cq;
3501         struct nes_cq *nescq;
3502         int resource_allocated;
3503         struct nes_adapter *nesadapter = nesdev->nesadapter;
3504         u32 aeq_info;
3505         u32 next_iwarp_state = 0;
3506         u32 aeqe_cq_id;
3507         u16 async_event_id;
3508         u8 tcp_state;
3509         u8 iwarp_state;
3510         struct ib_event ibevent;
3511
3512         nes_debug(NES_DBG_AEQ, "\n");
3513         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3514         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3515                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3516                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3517         } else {
3518                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3519                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3520                 BUG_ON(!context);
3521         }
3522
3523         /* context is nesqp unless async_event_id == CQ ERROR */
3524         nesqp = (struct nes_qp *)(unsigned long)context;
3525         async_event_id = (u16)aeq_info;
3526         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3527         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3528         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3529                         " Tcp state = %s, iWARP state = %s\n",
3530                         async_event_id,
3531                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3532                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3533
3534         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3535         if (aeq_info & NES_AEQE_QP) {
3536                 if (!nes_is_resource_allocated(nesadapter,
3537                                 nesadapter->allocated_qps,
3538                                 aeqe_cq_id))
3539                         return;
3540         }
3541
3542         switch (async_event_id) {
3543                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3544                         if (nesqp->term_flags)
3545                                 return; /* Ignore it, wait for close complete */
3546
3547                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3548                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3549                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3550                                         spin_lock_irqsave(&nesqp->lock, flags);
3551                                         nesqp->hw_iwarp_state = iwarp_state;
3552                                         nesqp->hw_tcp_state = tcp_state;
3553                                         nesqp->last_aeq = async_event_id;
3554                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3555                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3556                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3557                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3558                                         nes_cm_disconn(nesqp);
3559                                 }
3560                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3561                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3562                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3563                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3564                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3565                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3566                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3567                                                 async_event_id, nesqp->last_aeq, tcp_state);
3568                         }
3569                         break;
3570                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3571                         spin_lock_irqsave(&nesqp->lock, flags);
3572                         nesqp->hw_iwarp_state = iwarp_state;
3573                         nesqp->hw_tcp_state = tcp_state;
3574                         nesqp->last_aeq = async_event_id;
3575                         spin_unlock_irqrestore(&nesqp->lock, flags);
3576                         nes_cm_disconn(nesqp);
3577                         break;
3578
3579                 case NES_AEQE_AEID_RESET_SENT:
3580                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3581                         spin_lock_irqsave(&nesqp->lock, flags);
3582                         nesqp->hw_iwarp_state = iwarp_state;
3583                         nesqp->hw_tcp_state = tcp_state;
3584                         nesqp->last_aeq = async_event_id;
3585                         nesqp->hte_added = 0;
3586                         spin_unlock_irqrestore(&nesqp->lock, flags);
3587                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3588                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3589                         nes_cm_disconn(nesqp);
3590                         break;
3591
3592                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3593                         if (atomic_read(&nesqp->close_timer_started))
3594                                 return;
3595                         spin_lock_irqsave(&nesqp->lock, flags);
3596                         nesqp->hw_iwarp_state = iwarp_state;
3597                         nesqp->hw_tcp_state = tcp_state;
3598                         nesqp->last_aeq = async_event_id;
3599                         spin_unlock_irqrestore(&nesqp->lock, flags);
3600                         nes_cm_disconn(nesqp);
3601                         break;
3602
3603                 case NES_AEQE_AEID_TERMINATE_SENT:
3604                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3605                         break;
3606
3607                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3608                         nes_terminate_received(nesdev, nesqp, aeqe);
3609                         break;
3610
3611                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3612                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3613                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3614                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3615                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3616                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3617                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3618                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3619                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3620                 case NES_AEQE_AEID_AMP_TO_WRAP:
3621                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3622                                         nesqp->hwqp.qp_id, async_event_id);
3623                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3624                         break;
3625
3626                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3627                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3628                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3629                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3630                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3631                                 aeq_info &= 0xffff0000;
3632                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3633                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3634                         }
3635
3636                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3637                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3638                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3639                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3640                 case NES_AEQE_AEID_AMP_BAD_QP:
3641                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3642                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3643                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3644                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3645                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3646                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3647                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3648                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3649                 case NES_AEQE_AEID_AMP_BAD_PD:
3650                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3651                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3652                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3653                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3654                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3655                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3656                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3657                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3658                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3659                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3660                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3661                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3662                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3663                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3664                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3665                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3666                 case NES_AEQE_AEID_BAD_CLOSE:
3667                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3668                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3669                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3670                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3671                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3672                                         nesqp->hwqp.qp_id, async_event_id);
3673                         print_ip(nesqp->cm_node);
3674                         if (!atomic_read(&nesqp->close_timer_started))
3675                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3676                         break;
3677
3678                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3679                         context <<= 1;
3680                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3681                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3682                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3683                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3684                         if (resource_allocated) {
3685                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3686                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3687                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3688                                 if (hw_cq) {
3689                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3690                                         if (nescq->ibcq.event_handler) {
3691                                                 ibevent.device = nescq->ibcq.device;
3692                                                 ibevent.event = IB_EVENT_CQ_ERR;
3693                                                 ibevent.element.cq = &nescq->ibcq;
3694                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3695                                         }
3696                                 }
3697                         }
3698                         break;
3699
3700                 default:
3701                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3702                                         async_event_id);
3703                         break;
3704         }
3705
3706 }
3707
3708 /**
3709  * nes_iwarp_ce_handler
3710  */
3711 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3712 {
3713         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3714
3715         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3716                         nescq->hw_cq.cq_number); */
3717         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3718
3719         if (nescq->ibcq.comp_handler)
3720                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3721
3722         return;
3723 }
3724
3725
3726 /**
3727  * nes_manage_apbvt()
3728  */
3729 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3730                 u32 nic_index, u32 add_port)
3731 {
3732         struct nes_device *nesdev = nesvnic->nesdev;
3733         struct nes_hw_cqp_wqe *cqp_wqe;
3734         struct nes_cqp_request *cqp_request;
3735         int ret = 0;
3736         u16 major_code;
3737
3738         /* Send manage APBVT request to CQP */
3739         cqp_request = nes_get_cqp_request(nesdev);
3740         if (cqp_request == NULL) {
3741                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3742                 return -ENOMEM;
3743         }
3744         cqp_request->waiting = 1;
3745         cqp_wqe = &cqp_request->cqp_wqe;
3746
3747         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3748                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3749                         accel_local_port, accel_local_port, nic_index);
3750
3751         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3752         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3753                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3754         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3755                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3756
3757         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3758
3759         atomic_set(&cqp_request->refcount, 2);
3760         nes_post_cqp_request(nesdev, cqp_request);
3761
3762         if (add_port == NES_MANAGE_APBVT_ADD)
3763                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3764                                 NES_EVENT_TIMEOUT);
3765         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3766                         ret, cqp_request->major_code, cqp_request->minor_code);
3767         major_code = cqp_request->major_code;
3768
3769         nes_put_cqp_request(nesdev, cqp_request);
3770
3771         if (!ret)
3772                 return -ETIME;
3773         else if (major_code)
3774                 return -EIO;
3775         else
3776                 return 0;
3777 }
3778
3779
3780 /**
3781  * nes_manage_arp_cache
3782  */
3783 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3784                 u32 ip_addr, u32 action)
3785 {
3786         struct nes_hw_cqp_wqe *cqp_wqe;
3787         struct nes_vnic *nesvnic = netdev_priv(netdev);
3788         struct nes_device *nesdev;
3789         struct nes_cqp_request *cqp_request;
3790         int arp_index;
3791
3792         nesdev = nesvnic->nesdev;
3793         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3794         if (arp_index == -1) {
3795                 return;
3796         }
3797
3798         /* update the ARP entry */
3799         cqp_request = nes_get_cqp_request(nesdev);
3800         if (cqp_request == NULL) {
3801                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3802                 return;
3803         }
3804         cqp_request->waiting = 0;
3805         cqp_wqe = &cqp_request->cqp_wqe;
3806         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3807
3808         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3809                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3810         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3811                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3812         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3813
3814         if (action == NES_ARP_ADD) {
3815                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3816                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3817                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3818                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3819                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3820                                 (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]);
3821         } else {
3822                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3823                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3824         }
3825
3826         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3827                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3828
3829         atomic_set(&cqp_request->refcount, 1);
3830         nes_post_cqp_request(nesdev, cqp_request);
3831 }
3832
3833
3834 /**
3835  * flush_wqes
3836  */
3837 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3838                 u32 which_wq, u32 wait_completion)
3839 {
3840         struct nes_cqp_request *cqp_request;
3841         struct nes_hw_cqp_wqe *cqp_wqe;
3842         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3843         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3844         int ret;
3845
3846         cqp_request = nes_get_cqp_request(nesdev);
3847         if (cqp_request == NULL) {
3848                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3849                 return;
3850         }
3851         if (wait_completion) {
3852                 cqp_request->waiting = 1;
3853                 atomic_set(&cqp_request->refcount, 2);
3854         } else {
3855                 cqp_request->waiting = 0;
3856         }
3857         cqp_wqe = &cqp_request->cqp_wqe;
3858         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3859
3860         /* If wqe in error was identified, set code to be put into cqe */
3861         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3862                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3863                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3864                 nesqp->term_sq_flush_code = 0;
3865         }
3866
3867         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3868                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3869                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3870                 nesqp->term_rq_flush_code = 0;
3871         }
3872
3873         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3874                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3875                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3876         }
3877
3878         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3879                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3880         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3881
3882         nes_post_cqp_request(nesdev, cqp_request);
3883
3884         if (wait_completion) {
3885                 /* Wait for CQP */
3886                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3887                                 NES_EVENT_TIMEOUT);
3888                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3889                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3890                                 ret, cqp_request->major_code, cqp_request->minor_code);
3891                 nes_put_cqp_request(nesdev, cqp_request);
3892         }
3893 }