drivers/staging/octeon/ethernet.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file is based on code from OCTEON SDK by Cavium Networks.
   4  *
   5  * Copyright (c) 2003-2007 Cavium Networks
   6  */
   7
   8 #include <linux/platform_device.h>
   9 #include <linux/kernel.h>
  10 #include <linux/module.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/etherdevice.h>
  13 #include <linux/phy.h>
  14 #include <linux/slab.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/of_net.h>
  17 #include <linux/if_ether.h>
  18 #include <linux/if_vlan.h>
  19
  20 #include <net/dst.h>
  21
  22 #include <asm/octeon/octeon.h>
  23
  24 #include "ethernet-defines.h"
  25 #include "octeon-ethernet.h"
  26 #include "ethernet-mem.h"
  27 #include "ethernet-rx.h"
  28 #include "ethernet-tx.h"
  29 #include "ethernet-mdio.h"
  30 #include "ethernet-util.h"
  31
  32 #include <asm/octeon/cvmx-pip.h>
  33 #include <asm/octeon/cvmx-pko.h>
  34 #include <asm/octeon/cvmx-fau.h>
  35 #include <asm/octeon/cvmx-ipd.h>
  36 #include <asm/octeon/cvmx-helper.h>
  37 #include <asm/octeon/cvmx-asxx-defs.h>
  38 #include <asm/octeon/cvmx-gmxx-defs.h>
  39 #include <asm/octeon/cvmx-smix-defs.h>
  40
  41 #define OCTEON_MAX_MTU 65392
  42
  43 static int num_packet_buffers = 1024;
  44 module_param(num_packet_buffers, int, 0444);
  45 MODULE_PARM_DESC(num_packet_buffers, "\n"
  46         "\tNumber of packet buffers to allocate and store in the\n"
  47         "\tFPA. By default, 1024 packet buffers are used.\n");
  48
  49 static int pow_receive_group = 15;
  50 module_param(pow_receive_group, int, 0444);
  51 MODULE_PARM_DESC(pow_receive_group, "\n"
  52         "\tPOW group to receive packets from. All ethernet hardware\n"
  53         "\twill be configured to send incoming packets to this POW\n"
  54         "\tgroup. Also any other software can submit packets to this\n"
  55         "\tgroup for the kernel to process.");
  56
  57 static int receive_group_order;
  58 module_param(receive_group_order, int, 0444);
  59 MODULE_PARM_DESC(receive_group_order, "\n"
  60         "\tOrder (0..4) of receive groups to take into use. Ethernet hardware\n"
  61         "\twill be configured to send incoming packets to multiple POW\n"
  62         "\tgroups. pow_receive_group parameter is ignored when multiple\n"
  63         "\tgroups are taken into use and groups are allocated starting\n"
  64         "\tfrom 0. By default, a single group is used.\n");
  65
  66 int pow_send_group = -1;
  67 module_param(pow_send_group, int, 0644);
  68 MODULE_PARM_DESC(pow_send_group, "\n"
  69         "\tPOW group to send packets to other software on. This\n"
  70         "\tcontrols the creation of the virtual device pow0.\n"
  71         "\talways_use_pow also depends on this value.");
  72
  73 int always_use_pow;
  74 module_param(always_use_pow, int, 0444);
  75 MODULE_PARM_DESC(always_use_pow, "\n"
  76         "\tWhen set, always send to the pow group. This will cause\n"
  77         "\tpackets sent to real ethernet devices to be sent to the\n"
  78         "\tPOW group instead of the hardware. Unless some other\n"
  79         "\tapplication changes the config, packets will still be\n"
  80         "\treceived from the low level hardware. Use this option\n"
  81         "\tto allow a CVMX app to intercept all packets from the\n"
  82         "\tlinux kernel. You must specify pow_send_group along with\n"
  83         "\tthis option.");
  84
  85 char pow_send_list[128] = "";
  86 module_param_string(pow_send_list, pow_send_list, sizeof(pow_send_list), 0444);
  87 MODULE_PARM_DESC(pow_send_list, "\n"
  88         "\tComma separated list of ethernet devices that should use the\n"
  89         "\tPOW for transmit instead of the actual ethernet hardware. This\n"
  90         "\tis a per port version of always_use_pow. always_use_pow takes\n"
  91         "\tprecedence over this list. For example, setting this to\n"
  92         "\t\"eth2,spi3,spi7\" would cause these three devices to transmit\n"
  93         "\tusing the pow_send_group.");
  94
  95 int rx_napi_weight = 32;
  96 module_param(rx_napi_weight, int, 0444);
  97 MODULE_PARM_DESC(rx_napi_weight, "The NAPI WEIGHT parameter.");
  98
  99 /* Mask indicating which receive groups are in use. */
 100 int pow_receive_groups;
 101
 102 /*
 103  * cvm_oct_poll_queue_stopping - flag to indicate polling should stop.
 104  *
 105  * Set to one right before cvm_oct_poll_queue is destroyed.
 106  */
 107 atomic_t cvm_oct_poll_queue_stopping = ATOMIC_INIT(0);
 108
 109 /*
 110  * Array of every ethernet device owned by this driver indexed by
 111  * the ipd input port number.
 112  */
 113 struct net_device *cvm_oct_device[TOTAL_NUMBER_OF_PORTS];
 114
 115 u64 cvm_oct_tx_poll_interval;
 116
 117 static void cvm_oct_rx_refill_worker(struct work_struct *work);
 118 static DECLARE_DELAYED_WORK(cvm_oct_rx_refill_work, cvm_oct_rx_refill_worker);
 119
 120 static void cvm_oct_rx_refill_worker(struct work_struct *work)
 121 {
 122         /*
 123          * FPA 0 may have been drained, try to refill it if we need
 124          * more than num_packet_buffers / 2, otherwise normal receive
 125          * processing will refill it.  If it were drained, no packets
 126          * could be received so cvm_oct_napi_poll would never be
 127          * invoked to do the refill.
 128          */
 129         cvm_oct_rx_refill_pool(num_packet_buffers / 2);
 130
 131         if (!atomic_read(&cvm_oct_poll_queue_stopping))
 132                 schedule_delayed_work(&cvm_oct_rx_refill_work, HZ);
 133 }
 134
 135 static void cvm_oct_periodic_worker(struct work_struct *work)
 136 {
 137         struct octeon_ethernet *priv = container_of(work,
 138                                                     struct octeon_ethernet,
 139                                                     port_periodic_work.work);
 140
 141         if (priv->poll)
 142                 priv->poll(cvm_oct_device[priv->port]);
 143
 144         cvm_oct_device[priv->port]->netdev_ops->ndo_get_stats(
 145                                                 cvm_oct_device[priv->port]);
 146
 147         if (!atomic_read(&cvm_oct_poll_queue_stopping))
 148                 schedule_delayed_work(&priv->port_periodic_work, HZ);
 149 }
 150
 151 static void cvm_oct_configure_common_hw(void)
 152 {
 153         /* Setup the FPA */
 154         cvmx_fpa_enable();
 155         cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE,
 156                              num_packet_buffers);
 157         cvm_oct_mem_fill_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE,
 158                              num_packet_buffers);
 159         if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL)
 160                 cvm_oct_mem_fill_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL,
 161                                      CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 1024);
 162
 163 #ifdef __LITTLE_ENDIAN
 164         {
 165                 union cvmx_ipd_ctl_status ipd_ctl_status;
 166
 167                 ipd_ctl_status.u64 = cvmx_read_csr(CVMX_IPD_CTL_STATUS);
 168                 ipd_ctl_status.s.pkt_lend = 1;
 169                 ipd_ctl_status.s.wqe_lend = 1;
 170                 cvmx_write_csr(CVMX_IPD_CTL_STATUS, ipd_ctl_status.u64);
 171         }
 172 #endif
 173
 174         cvmx_helper_setup_red(num_packet_buffers / 4, num_packet_buffers / 8);
 175 }
 176
 177 /**
 178  * cvm_oct_free_work- Free a work queue entry
 179  *
 180  * @work_queue_entry: Work queue entry to free
 181  *
 182  * Returns Zero on success, Negative on failure.
 183  */
 184 int cvm_oct_free_work(void *work_queue_entry)
 185 {
 186         cvmx_wqe_t *work = work_queue_entry;
 187
 188         int segments = work->word2.s.bufs;
 189         union cvmx_buf_ptr segment_ptr = work->packet_ptr;
 190
 191         while (segments--) {
 192                 union cvmx_buf_ptr next_ptr = *(union cvmx_buf_ptr *)
 193                         cvmx_phys_to_ptr(segment_ptr.s.addr - 8);
 194                 if (unlikely(!segment_ptr.s.i))
 195                         cvmx_fpa_free(cvm_oct_get_buffer_ptr(segment_ptr),
 196                                       segment_ptr.s.pool,
 197                                       CVMX_FPA_PACKET_POOL_SIZE / 128);
 198                 segment_ptr = next_ptr;
 199         }
 200         cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 1);
 201
 202         return 0;
 203 }
 204 EXPORT_SYMBOL(cvm_oct_free_work);
 205
 206 /**
 207  * cvm_oct_common_get_stats - get the low level ethernet statistics
 208  * @dev:    Device to get the statistics from
 209  *
 210  * Returns Pointer to the statistics
 211  */
 212 static struct net_device_stats *cvm_oct_common_get_stats(struct net_device *dev)
 213 {
 214         cvmx_pip_port_status_t rx_status;
 215         cvmx_pko_port_status_t tx_status;
 216         struct octeon_ethernet *priv = netdev_priv(dev);
 217
 218         if (priv->port < CVMX_PIP_NUM_INPUT_PORTS) {
 219                 if (octeon_is_simulation()) {
 220                         /* The simulator doesn't support statistics */
 221                         memset(&rx_status, 0, sizeof(rx_status));
 222                         memset(&tx_status, 0, sizeof(tx_status));
 223                 } else {
 224                         cvmx_pip_get_port_status(priv->port, 1, &rx_status);
 225                         cvmx_pko_get_port_status(priv->port, 1, &tx_status);
 226                 }
 227
 228                 dev->stats.rx_packets += rx_status.inb_packets;
 229                 dev->stats.tx_packets += tx_status.packets;
 230                 dev->stats.rx_bytes += rx_status.inb_octets;
 231                 dev->stats.tx_bytes += tx_status.octets;
 232                 dev->stats.multicast += rx_status.multicast_packets;
 233                 dev->stats.rx_crc_errors += rx_status.inb_errors;
 234                 dev->stats.rx_frame_errors += rx_status.fcs_align_err_packets;
 235                 dev->stats.rx_dropped += rx_status.dropped_packets;
 236         }
 237
 238         return &dev->stats;
 239 }
 240
 241 /**
 242  * cvm_oct_common_change_mtu - change the link MTU
 243  * @dev:     Device to change
 244  * @new_mtu: The new MTU
 245  *
 246  * Returns Zero on success
 247  */
 248 static int cvm_oct_common_change_mtu(struct net_device *dev, int new_mtu)
 249 {
 250         struct octeon_ethernet *priv = netdev_priv(dev);
 251         int interface = INTERFACE(priv->port);
 252 #if IS_ENABLED(CONFIG_VLAN_8021Q)
 253         int vlan_bytes = VLAN_HLEN;
 254 #else
 255         int vlan_bytes = 0;
 256 #endif
 257         int mtu_overhead = ETH_HLEN + ETH_FCS_LEN + vlan_bytes;
 258
 259         dev->mtu = new_mtu;
 260
 261         if ((interface < 2) &&
 262             (cvmx_helper_interface_get_mode(interface) !=
 263                 CVMX_HELPER_INTERFACE_MODE_SPI)) {
 264                 int index = INDEX(priv->port);
 265                 /* Add ethernet header and FCS, and VLAN if configured. */
 266                 int max_packet = new_mtu + mtu_overhead;
 267
 268                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 269                     OCTEON_IS_MODEL(OCTEON_CN58XX)) {
 270                         /* Signal errors on packets larger than the MTU */
 271                         cvmx_write_csr(CVMX_GMXX_RXX_FRM_MAX(index, interface),
 272                                        max_packet);
 273                 } else {
 274                         /*
 275                          * Set the hardware to truncate packets larger
 276                          * than the MTU and smaller the 64 bytes.
 277                          */
 278                         union cvmx_pip_frm_len_chkx frm_len_chk;
 279
 280                         frm_len_chk.u64 = 0;
 281                         frm_len_chk.s.minlen = VLAN_ETH_ZLEN;
 282                         frm_len_chk.s.maxlen = max_packet;
 283                         cvmx_write_csr(CVMX_PIP_FRM_LEN_CHKX(interface),
 284                                        frm_len_chk.u64);
 285                 }
 286                 /*
 287                  * Set the hardware to truncate packets larger than
 288                  * the MTU. The jabber register must be set to a
 289                  * multiple of 8 bytes, so round up.
 290                  */
 291                 cvmx_write_csr(CVMX_GMXX_RXX_JABBER(index, interface),
 292                                (max_packet + 7) & ~7u);
 293         }
 294         return 0;
 295 }
 296
 297 /**
 298  * cvm_oct_common_set_multicast_list - set the multicast list
 299  * @dev:    Device to work on
 300  */
 301 static void cvm_oct_common_set_multicast_list(struct net_device *dev)
 302 {
 303         union cvmx_gmxx_prtx_cfg gmx_cfg;
 304         struct octeon_ethernet *priv = netdev_priv(dev);
 305         int interface = INTERFACE(priv->port);
 306
 307         if ((interface < 2) &&
 308             (cvmx_helper_interface_get_mode(interface) !=
 309                 CVMX_HELPER_INTERFACE_MODE_SPI)) {
 310                 union cvmx_gmxx_rxx_adr_ctl control;
 311                 int index = INDEX(priv->port);
 312
 313                 control.u64 = 0;
 314                 control.s.bcst = 1;     /* Allow broadcast MAC addresses */
 315
 316                 if (!netdev_mc_empty(dev) || (dev->flags & IFF_ALLMULTI) ||
 317                     (dev->flags & IFF_PROMISC))
 318                         /* Force accept multicast packets */
 319                         control.s.mcst = 2;
 320                 else
 321                         /* Force reject multicast packets */
 322                         control.s.mcst = 1;
 323
 324                 if (dev->flags & IFF_PROMISC)
 325                         /*
 326                          * Reject matches if promisc. Since CAM is
 327                          * shut off, should accept everything.
 328                          */
 329                         control.s.cam_mode = 0;
 330                 else
 331                         /* Filter packets based on the CAM */
 332                         control.s.cam_mode = 1;
 333
 334                 gmx_cfg.u64 =
 335                     cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface));
 336                 cvmx_write_csr(CVMX_GMXX_PRTX_CFG(index, interface),
 337                                gmx_cfg.u64 & ~1ull);
 338
 339                 cvmx_write_csr(CVMX_GMXX_RXX_ADR_CTL(index, interface),
 340                                control.u64);
 341                 if (dev->flags & IFF_PROMISC)
 342                         cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM_EN
 343                                        (index, interface), 0);
 344                 else
 345                         cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM_EN
 346                                        (index, interface), 1);
 347
 348                 cvmx_write_csr(CVMX_GMXX_PRTX_CFG(index, interface),
 349                                gmx_cfg.u64);
 350         }
 351 }
 352
 353 static int cvm_oct_set_mac_filter(struct net_device *dev)
 354 {
 355         struct octeon_ethernet *priv = netdev_priv(dev);
 356         union cvmx_gmxx_prtx_cfg gmx_cfg;
 357         int interface = INTERFACE(priv->port);
 358
 359         if ((interface < 2) &&
 360             (cvmx_helper_interface_get_mode(interface) !=
 361                 CVMX_HELPER_INTERFACE_MODE_SPI)) {
 362                 int i;
 363                 u8 *ptr = dev->dev_addr;
 364                 u64 mac = 0;
 365                 int index = INDEX(priv->port);
 366
 367                 for (i = 0; i < 6; i++)
 368                         mac = (mac << 8) | (u64)ptr[i];
 369
 370                 gmx_cfg.u64 =
 371                     cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface));
 372                 cvmx_write_csr(CVMX_GMXX_PRTX_CFG(index, interface),
 373                                gmx_cfg.u64 & ~1ull);
 374
 375                 cvmx_write_csr(CVMX_GMXX_SMACX(index, interface), mac);
 376                 cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM0(index, interface),
 377                                ptr[0]);
 378                 cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM1(index, interface),
 379                                ptr[1]);
 380                 cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM2(index, interface),
 381                                ptr[2]);
 382                 cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM3(index, interface),
 383                                ptr[3]);
 384                 cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM4(index, interface),
 385                                ptr[4]);
 386                 cvmx_write_csr(CVMX_GMXX_RXX_ADR_CAM5(index, interface),
 387                                ptr[5]);
 388                 cvm_oct_common_set_multicast_list(dev);
 389                 cvmx_write_csr(CVMX_GMXX_PRTX_CFG(index, interface),
 390                                gmx_cfg.u64);
 391         }
 392         return 0;
 393 }
 394
 395 /**
 396  * cvm_oct_common_set_mac_address - set the hardware MAC address for a device
 397  * @dev:    The device in question.
 398  * @addr:   Socket address.
 399  *
 400  * Returns Zero on success
 401  */
 402 static int cvm_oct_common_set_mac_address(struct net_device *dev, void *addr)
 403 {
 404         int r = eth_mac_addr(dev, addr);
 405
 406         if (r)
 407                 return r;
 408         return cvm_oct_set_mac_filter(dev);
 409 }
 410
 411 /**
 412  * cvm_oct_common_init - per network device initialization
 413  * @dev:    Device to initialize
 414  *
 415  * Returns Zero on success
 416  */
 417 int cvm_oct_common_init(struct net_device *dev)
 418 {
 419         struct octeon_ethernet *priv = netdev_priv(dev);
 420         const u8 *mac = NULL;
 421
 422         if (priv->of_node)
 423                 mac = of_get_mac_address(priv->of_node);
 424
 425         if (mac)
 426                 ether_addr_copy(dev->dev_addr, mac);
 427         else
 428                 eth_hw_addr_random(dev);
 429
 430         /*
 431          * Force the interface to use the POW send if always_use_pow
 432          * was specified or it is in the pow send list.
 433          */
 434         if ((pow_send_group != -1) &&
 435             (always_use_pow || strstr(pow_send_list, dev->name)))
 436                 priv->queue = -1;
 437
 438         if (priv->queue != -1)
 439                 dev->features |= NETIF_F_SG | NETIF_F_IP_CSUM;
 440
 441         /* We do our own locking, Linux doesn't need to */
 442         dev->features |= NETIF_F_LLTX;
 443         dev->ethtool_ops = &cvm_oct_ethtool_ops;
 444
 445         cvm_oct_set_mac_filter(dev);
 446         dev_set_mtu(dev, dev->mtu);
 447
 448         /*
 449          * Zero out stats for port so we won't mistakenly show
 450          * counters from the bootloader.
 451          */
 452         memset(dev->netdev_ops->ndo_get_stats(dev), 0,
 453                sizeof(struct net_device_stats));
 454
 455         if (dev->netdev_ops->ndo_stop)
 456                 dev->netdev_ops->ndo_stop(dev);
 457
 458         return 0;
 459 }
 460
 461 void cvm_oct_common_uninit(struct net_device *dev)
 462 {
 463         if (dev->phydev)
 464                 phy_disconnect(dev->phydev);
 465 }
 466
 467 int cvm_oct_common_open(struct net_device *dev,
 468                         void (*link_poll)(struct net_device *))
 469 {
 470         union cvmx_gmxx_prtx_cfg gmx_cfg;
 471         struct octeon_ethernet *priv = netdev_priv(dev);
 472         int interface = INTERFACE(priv->port);
 473         int index = INDEX(priv->port);
 474         cvmx_helper_link_info_t link_info;
 475         int rv;
 476
 477         rv = cvm_oct_phy_setup_device(dev);
 478         if (rv)
 479                 return rv;
 480
 481         gmx_cfg.u64 = cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface));
 482         gmx_cfg.s.en = 1;
 483         if (octeon_has_feature(OCTEON_FEATURE_PKND))
 484                 gmx_cfg.s.pknd = priv->port;
 485         cvmx_write_csr(CVMX_GMXX_PRTX_CFG(index, interface), gmx_cfg.u64);
 486
 487         if (octeon_is_simulation())
 488                 return 0;
 489
 490         if (dev->phydev) {
 491                 int r = phy_read_status(dev->phydev);
 492
 493                 if (r == 0 && dev->phydev->link == 0)
 494                         netif_carrier_off(dev);
 495                 cvm_oct_adjust_link(dev);
 496         } else {
 497                 link_info = cvmx_helper_link_get(priv->port);
 498                 if (!link_info.s.link_up)
 499                         netif_carrier_off(dev);
 500                 priv->poll = link_poll;
 501                 link_poll(dev);
 502         }
 503
 504         return 0;
 505 }
 506
 507 void cvm_oct_link_poll(struct net_device *dev)
 508 {
 509         struct octeon_ethernet *priv = netdev_priv(dev);
 510         cvmx_helper_link_info_t link_info;
 511
 512         link_info = cvmx_helper_link_get(priv->port);
 513         if (link_info.u64 == priv->link_info)
 514                 return;
 515
 516         if (cvmx_helper_link_set(priv->port, link_info))
 517                 link_info.u64 = priv->link_info;
 518         else
 519                 priv->link_info = link_info.u64;
 520
 521         if (link_info.s.link_up) {
 522                 if (!netif_carrier_ok(dev))
 523                         netif_carrier_on(dev);
 524         } else if (netif_carrier_ok(dev)) {
 525                 netif_carrier_off(dev);
 526         }
 527         cvm_oct_note_carrier(priv, link_info);
 528 }
 529
 530 static int cvm_oct_xaui_open(struct net_device *dev)
 531 {
 532         return cvm_oct_common_open(dev, cvm_oct_link_poll);
 533 }
 534
 535 static const struct net_device_ops cvm_oct_npi_netdev_ops = {
 536         .ndo_init               = cvm_oct_common_init,
 537         .ndo_uninit             = cvm_oct_common_uninit,
 538         .ndo_start_xmit         = cvm_oct_xmit,
 539         .ndo_set_rx_mode        = cvm_oct_common_set_multicast_list,
 540         .ndo_set_mac_address    = cvm_oct_common_set_mac_address,
 541         .ndo_do_ioctl           = cvm_oct_ioctl,
 542         .ndo_change_mtu         = cvm_oct_common_change_mtu,
 543         .ndo_get_stats          = cvm_oct_common_get_stats,
 544 #ifdef CONFIG_NET_POLL_CONTROLLER
 545         .ndo_poll_controller    = cvm_oct_poll_controller,
 546 #endif
 547 };
 548
 549 static const struct net_device_ops cvm_oct_xaui_netdev_ops = {
 550         .ndo_init               = cvm_oct_common_init,
 551         .ndo_uninit             = cvm_oct_common_uninit,
 552         .ndo_open               = cvm_oct_xaui_open,
 553         .ndo_stop               = cvm_oct_common_stop,
 554         .ndo_start_xmit         = cvm_oct_xmit,
 555         .ndo_set_rx_mode        = cvm_oct_common_set_multicast_list,
 556         .ndo_set_mac_address    = cvm_oct_common_set_mac_address,
 557         .ndo_do_ioctl           = cvm_oct_ioctl,
 558         .ndo_change_mtu         = cvm_oct_common_change_mtu,
 559         .ndo_get_stats          = cvm_oct_common_get_stats,
 560 #ifdef CONFIG_NET_POLL_CONTROLLER
 561         .ndo_poll_controller    = cvm_oct_poll_controller,
 562 #endif
 563 };
 564
 565 static const struct net_device_ops cvm_oct_sgmii_netdev_ops = {
 566         .ndo_init               = cvm_oct_sgmii_init,
 567         .ndo_uninit             = cvm_oct_common_uninit,
 568         .ndo_open               = cvm_oct_sgmii_open,
 569         .ndo_stop               = cvm_oct_common_stop,
 570         .ndo_start_xmit         = cvm_oct_xmit,
 571         .ndo_set_rx_mode        = cvm_oct_common_set_multicast_list,
 572         .ndo_set_mac_address    = cvm_oct_common_set_mac_address,
 573         .ndo_do_ioctl           = cvm_oct_ioctl,
 574         .ndo_change_mtu         = cvm_oct_common_change_mtu,
 575         .ndo_get_stats          = cvm_oct_common_get_stats,
 576 #ifdef CONFIG_NET_POLL_CONTROLLER
 577         .ndo_poll_controller    = cvm_oct_poll_controller,
 578 #endif
 579 };
 580
 581 static const struct net_device_ops cvm_oct_spi_netdev_ops = {
 582         .ndo_init               = cvm_oct_spi_init,
 583         .ndo_uninit             = cvm_oct_spi_uninit,
 584         .ndo_start_xmit         = cvm_oct_xmit,
 585         .ndo_set_rx_mode        = cvm_oct_common_set_multicast_list,
 586         .ndo_set_mac_address    = cvm_oct_common_set_mac_address,
 587         .ndo_do_ioctl           = cvm_oct_ioctl,
 588         .ndo_change_mtu         = cvm_oct_common_change_mtu,
 589         .ndo_get_stats          = cvm_oct_common_get_stats,
 590 #ifdef CONFIG_NET_POLL_CONTROLLER
 591         .ndo_poll_controller    = cvm_oct_poll_controller,
 592 #endif
 593 };
 594
 595 static const struct net_device_ops cvm_oct_rgmii_netdev_ops = {
 596         .ndo_init               = cvm_oct_common_init,
 597         .ndo_uninit             = cvm_oct_common_uninit,
 598         .ndo_open               = cvm_oct_rgmii_open,
 599         .ndo_stop               = cvm_oct_common_stop,
 600         .ndo_start_xmit         = cvm_oct_xmit,
 601         .ndo_set_rx_mode        = cvm_oct_common_set_multicast_list,
 602         .ndo_set_mac_address    = cvm_oct_common_set_mac_address,
 603         .ndo_do_ioctl           = cvm_oct_ioctl,
 604         .ndo_change_mtu         = cvm_oct_common_change_mtu,
 605         .ndo_get_stats          = cvm_oct_common_get_stats,
 606 #ifdef CONFIG_NET_POLL_CONTROLLER
 607         .ndo_poll_controller    = cvm_oct_poll_controller,
 608 #endif
 609 };
 610
 611 static const struct net_device_ops cvm_oct_pow_netdev_ops = {
 612         .ndo_init               = cvm_oct_common_init,
 613         .ndo_start_xmit         = cvm_oct_xmit_pow,
 614         .ndo_set_rx_mode        = cvm_oct_common_set_multicast_list,
 615         .ndo_set_mac_address    = cvm_oct_common_set_mac_address,
 616         .ndo_do_ioctl           = cvm_oct_ioctl,
 617         .ndo_change_mtu         = cvm_oct_common_change_mtu,
 618         .ndo_get_stats          = cvm_oct_common_get_stats,
 619 #ifdef CONFIG_NET_POLL_CONTROLLER
 620         .ndo_poll_controller    = cvm_oct_poll_controller,
 621 #endif
 622 };
 623
 624 static struct device_node *cvm_oct_of_get_child(
 625                                 const struct device_node *parent, int reg_val)
 626 {
 627         struct device_node *node = NULL;
 628         int size;
 629         const __be32 *addr;
 630
 631         for (;;) {
 632                 node = of_get_next_child(parent, node);
 633                 if (!node)
 634                         break;
 635                 addr = of_get_property(node, "reg", &size);
 636                 if (addr && (be32_to_cpu(*addr) == reg_val))
 637                         break;
 638         }
 639         return node;
 640 }
 641
 642 static struct device_node *cvm_oct_node_for_port(struct device_node *pip,
 643                                                  int interface, int port)
 644 {
 645         struct device_node *ni, *np;
 646
 647         ni = cvm_oct_of_get_child(pip, interface);
 648         if (!ni)
 649                 return NULL;
 650
 651         np = cvm_oct_of_get_child(ni, port);
 652         of_node_put(ni);
 653
 654         return np;
 655 }
 656
 657 static void cvm_set_rgmii_delay(struct device_node *np, int iface, int port)
 658 {
 659         u32 delay_value;
 660
 661         if (!of_property_read_u32(np, "rx-delay", &delay_value))
 662                 cvmx_write_csr(CVMX_ASXX_RX_CLK_SETX(port, iface), delay_value);
 663         if (!of_property_read_u32(np, "tx-delay", &delay_value))
 664                 cvmx_write_csr(CVMX_ASXX_TX_CLK_SETX(port, iface), delay_value);
 665 }
 666
 667 static int cvm_oct_probe(struct platform_device *pdev)
 668 {
 669         int num_interfaces;
 670         int interface;
 671         int fau = FAU_NUM_PACKET_BUFFERS_TO_FREE;
 672         int qos;
 673         struct device_node *pip;
 674         int mtu_overhead = ETH_HLEN + ETH_FCS_LEN;
 675
 676 #if IS_ENABLED(CONFIG_VLAN_8021Q)
 677         mtu_overhead += VLAN_HLEN;
 678 #endif
 679
 680         octeon_mdiobus_force_mod_depencency();
 681
 682         pip = pdev->dev.of_node;
 683         if (!pip) {
 684                 pr_err("Error: No 'pip' in /aliases\n");
 685                 return -EINVAL;
 686         }
 687
 688         cvm_oct_configure_common_hw();
 689
 690         cvmx_helper_initialize_packet_io_global();
 691
 692         if (receive_group_order) {
 693                 if (receive_group_order > 4)
 694                         receive_group_order = 4;
 695                 pow_receive_groups = (1 << (1 << receive_group_order)) - 1;
 696         } else {
 697                 pow_receive_groups = BIT(pow_receive_group);
 698         }
 699
 700         /* Change the input group for all ports before input is enabled */
 701         num_interfaces = cvmx_helper_get_number_of_interfaces();
 702         for (interface = 0; interface < num_interfaces; interface++) {
 703                 int num_ports = cvmx_helper_ports_on_interface(interface);
 704                 int port;
 705
 706                 for (port = cvmx_helper_get_ipd_port(interface, 0);
 707                      port < cvmx_helper_get_ipd_port(interface, num_ports);
 708                      port++) {
 709                         union cvmx_pip_prt_tagx pip_prt_tagx;
 710
 711                         pip_prt_tagx.u64 =
 712                             cvmx_read_csr(CVMX_PIP_PRT_TAGX(port));
 713
 714                         if (receive_group_order) {
 715                                 int tag_mask;
 716
 717                                 /* We support only 16 groups at the moment, so
 718                                  * always disable the two additional "hidden"
 719                                  * tag_mask bits on CN68XX.
 720                                  */
 721                                 if (OCTEON_IS_MODEL(OCTEON_CN68XX))
 722                                         pip_prt_tagx.u64 |= 0x3ull << 44;
 723
 724                                 tag_mask = ~((1 << receive_group_order) - 1);
 725                                 pip_prt_tagx.s.grptagbase       = 0;
 726                                 pip_prt_tagx.s.grptagmask       = tag_mask;
 727                                 pip_prt_tagx.s.grptag           = 1;
 728                                 pip_prt_tagx.s.tag_mode         = 0;
 729                                 pip_prt_tagx.s.inc_prt_flag     = 1;
 730                                 pip_prt_tagx.s.ip6_dprt_flag    = 1;
 731                                 pip_prt_tagx.s.ip4_dprt_flag    = 1;
 732                                 pip_prt_tagx.s.ip6_sprt_flag    = 1;
 733                                 pip_prt_tagx.s.ip4_sprt_flag    = 1;
 734                                 pip_prt_tagx.s.ip6_dst_flag     = 1;
 735                                 pip_prt_tagx.s.ip4_dst_flag     = 1;
 736                                 pip_prt_tagx.s.ip6_src_flag     = 1;
 737                                 pip_prt_tagx.s.ip4_src_flag     = 1;
 738                                 pip_prt_tagx.s.grp              = 0;
 739                         } else {
 740                                 pip_prt_tagx.s.grptag   = 0;
 741                                 pip_prt_tagx.s.grp      = pow_receive_group;
 742                         }
 743
 744                         cvmx_write_csr(CVMX_PIP_PRT_TAGX(port),
 745                                        pip_prt_tagx.u64);
 746                 }
 747         }
 748
 749         cvmx_helper_ipd_and_packet_input_enable();
 750
 751         memset(cvm_oct_device, 0, sizeof(cvm_oct_device));
 752
 753         /*
 754          * Initialize the FAU used for counting packet buffers that
 755          * need to be freed.
 756          */
 757         cvmx_fau_atomic_write32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0);
 758
 759         /* Initialize the FAU used for counting tx SKBs that need to be freed */
 760         cvmx_fau_atomic_write32(FAU_TOTAL_TX_TO_CLEAN, 0);
 761
 762         if ((pow_send_group != -1)) {
 763                 struct net_device *dev;
 764
 765                 dev = alloc_etherdev(sizeof(struct octeon_ethernet));
 766                 if (dev) {
 767                         /* Initialize the device private structure. */
 768                         struct octeon_ethernet *priv = netdev_priv(dev);
 769
 770                         SET_NETDEV_DEV(dev, &pdev->dev);
 771                         dev->netdev_ops = &cvm_oct_pow_netdev_ops;
 772                         priv->imode = CVMX_HELPER_INTERFACE_MODE_DISABLED;
 773                         priv->port = CVMX_PIP_NUM_INPUT_PORTS;
 774                         priv->queue = -1;
 775                         strcpy(dev->name, "pow%d");
 776                         for (qos = 0; qos < 16; qos++)
 777                                 skb_queue_head_init(&priv->tx_free_list[qos]);
 778                         dev->min_mtu = VLAN_ETH_ZLEN - mtu_overhead;
 779                         dev->max_mtu = OCTEON_MAX_MTU - mtu_overhead;
 780
 781                         if (register_netdev(dev) < 0) {
 782                                 pr_err("Failed to register ethernet device for POW\n");
 783                                 free_netdev(dev);
 784                         } else {
 785                                 cvm_oct_device[CVMX_PIP_NUM_INPUT_PORTS] = dev;
 786                                 pr_info("%s: POW send group %d, receive group %d\n",
 787                                         dev->name, pow_send_group,
 788                                         pow_receive_group);
 789                         }
 790                 } else {
 791                         pr_err("Failed to allocate ethernet device for POW\n");
 792                 }
 793         }
 794
 795         num_interfaces = cvmx_helper_get_number_of_interfaces();
 796         for (interface = 0; interface < num_interfaces; interface++) {
 797                 cvmx_helper_interface_mode_t imode =
 798                     cvmx_helper_interface_get_mode(interface);
 799                 int num_ports = cvmx_helper_ports_on_interface(interface);
 800                 int port;
 801                 int port_index;
 802
 803                 for (port_index = 0,
 804                      port = cvmx_helper_get_ipd_port(interface, 0);
 805                      port < cvmx_helper_get_ipd_port(interface, num_ports);
 806                      port_index++, port++) {
 807                         struct octeon_ethernet *priv;
 808                         struct net_device *dev =
 809                             alloc_etherdev(sizeof(struct octeon_ethernet));
 810                         if (!dev) {
 811                                 pr_err("Failed to allocate ethernet device for port %d\n",
 812                                        port);
 813                                 continue;
 814                         }
 815
 816                         /* Initialize the device private structure. */
 817                         SET_NETDEV_DEV(dev, &pdev->dev);
 818                         priv = netdev_priv(dev);
 819                         priv->netdev = dev;
 820                         priv->of_node = cvm_oct_node_for_port(pip, interface,
 821                                                                 port_index);
 822
 823                         INIT_DELAYED_WORK(&priv->port_periodic_work,
 824                                           cvm_oct_periodic_worker);
 825                         priv->imode = imode;
 826                         priv->port = port;
 827                         priv->queue = cvmx_pko_get_base_queue(priv->port);
 828                         priv->fau = fau - cvmx_pko_get_num_queues(port) * 4;
 829                         for (qos = 0; qos < 16; qos++)
 830                                 skb_queue_head_init(&priv->tx_free_list[qos]);
 831                         for (qos = 0; qos < cvmx_pko_get_num_queues(port);
 832                              qos++)
 833                                 cvmx_fau_atomic_write32(priv->fau + qos * 4, 0);
 834                         dev->min_mtu = VLAN_ETH_ZLEN - mtu_overhead;
 835                         dev->max_mtu = OCTEON_MAX_MTU - mtu_overhead;
 836
 837                         switch (priv->imode) {
 838                         /* These types don't support ports to IPD/PKO */
 839                         case CVMX_HELPER_INTERFACE_MODE_DISABLED:
 840                         case CVMX_HELPER_INTERFACE_MODE_PCIE:
 841                         case CVMX_HELPER_INTERFACE_MODE_PICMG:
 842                                 break;
 843
 844                         case CVMX_HELPER_INTERFACE_MODE_NPI:
 845                                 dev->netdev_ops = &cvm_oct_npi_netdev_ops;
 846                                 strcpy(dev->name, "npi%d");
 847                                 break;
 848
 849                         case CVMX_HELPER_INTERFACE_MODE_XAUI:
 850                                 dev->netdev_ops = &cvm_oct_xaui_netdev_ops;
 851                                 strcpy(dev->name, "xaui%d");
 852                                 break;
 853
 854                         case CVMX_HELPER_INTERFACE_MODE_LOOP:
 855                                 dev->netdev_ops = &cvm_oct_npi_netdev_ops;
 856                                 strcpy(dev->name, "loop%d");
 857                                 break;
 858
 859                         case CVMX_HELPER_INTERFACE_MODE_SGMII:
 860                                 dev->netdev_ops = &cvm_oct_sgmii_netdev_ops;
 861                                 strcpy(dev->name, "eth%d");
 862                                 break;
 863
 864                         case CVMX_HELPER_INTERFACE_MODE_SPI:
 865                                 dev->netdev_ops = &cvm_oct_spi_netdev_ops;
 866                                 strcpy(dev->name, "spi%d");
 867                                 break;
 868
 869                         case CVMX_HELPER_INTERFACE_MODE_RGMII:
 870                         case CVMX_HELPER_INTERFACE_MODE_GMII:
 871                                 dev->netdev_ops = &cvm_oct_rgmii_netdev_ops;
 872                                 strcpy(dev->name, "eth%d");
 873                                 cvm_set_rgmii_delay(priv->of_node, interface,
 874                                                     port_index);
 875                                 break;
 876                         }
 877
 878                         if (!dev->netdev_ops) {
 879                                 free_netdev(dev);
 880                         } else if (register_netdev(dev) < 0) {
 881                                 pr_err("Failed to register ethernet device for interface %d, port %d\n",
 882                                        interface, priv->port);
 883                                 free_netdev(dev);
 884                         } else {
 885                                 cvm_oct_device[priv->port] = dev;
 886                                 fau -=
 887                                     cvmx_pko_get_num_queues(priv->port) *
 888                                     sizeof(u32);
 889                                 schedule_delayed_work(&priv->port_periodic_work,
 890                                                       HZ);
 891                         }
 892                 }
 893         }
 894
 895         cvm_oct_tx_initialize();
 896         cvm_oct_rx_initialize();
 897
 898         /*
 899          * 150 uS: about 10 1500-byte packets at 1GE.
 900          */
 901         cvm_oct_tx_poll_interval = 150 * (octeon_get_clock_rate() / 1000000);
 902
 903         schedule_delayed_work(&cvm_oct_rx_refill_work, HZ);
 904
 905         return 0;
 906 }
 907
 908 static int cvm_oct_remove(struct platform_device *pdev)
 909 {
 910         int port;
 911
 912         cvmx_ipd_disable();
 913
 914         atomic_inc_return(&cvm_oct_poll_queue_stopping);
 915         cancel_delayed_work_sync(&cvm_oct_rx_refill_work);
 916
 917         cvm_oct_rx_shutdown();
 918         cvm_oct_tx_shutdown();
 919
 920         cvmx_pko_disable();
 921
 922         /* Free the ethernet devices */
 923         for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++) {
 924                 if (cvm_oct_device[port]) {
 925                         struct net_device *dev = cvm_oct_device[port];
 926                         struct octeon_ethernet *priv = netdev_priv(dev);
 927
 928                         cancel_delayed_work_sync(&priv->port_periodic_work);
 929
 930                         cvm_oct_tx_shutdown_dev(dev);
 931                         unregister_netdev(dev);
 932                         free_netdev(dev);
 933                         cvm_oct_device[port] = NULL;
 934                 }
 935         }
 936
 937         cvmx_pko_shutdown();
 938
 939         cvmx_ipd_free_ptr();
 940
 941         /* Free the HW pools */
 942         cvm_oct_mem_empty_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE,
 943                               num_packet_buffers);
 944         cvm_oct_mem_empty_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE,
 945                               num_packet_buffers);
 946         if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL)
 947                 cvm_oct_mem_empty_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL,
 948                                       CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 128);
 949         return 0;
 950 }
 951
 952 static const struct of_device_id cvm_oct_match[] = {
 953         {
 954                 .compatible = "cavium,octeon-3860-pip",
 955         },
 956         {},
 957 };
 958 MODULE_DEVICE_TABLE(of, cvm_oct_match);
 959
 960 static struct platform_driver cvm_oct_driver = {
 961         .probe          = cvm_oct_probe,
 962         .remove         = cvm_oct_remove,
 963         .driver         = {
 964                 .name   = KBUILD_MODNAME,
 965                 .of_match_table = cvm_oct_match,
 966         },
 967 };
 968
 969 module_platform_driver(cvm_oct_driver);
 970
 971 MODULE_LICENSE("GPL");
 972 MODULE_AUTHOR("Cavium Networks <[email protected]>");
 973 MODULE_DESCRIPTION("Cavium Networks Octeon ethernet driver.");