drivers/net/ethernet/intel/i40e/i40e_txrx.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <[email protected]>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/prefetch.h>
  28 #include "i40e.h"
  29 #include "i40e_prototype.h"
  30
  31 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
  32                                 u32 td_tag)
  33 {
  34         return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
  35                            ((u64)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
  36                            ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
  37                            ((u64)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
  38                            ((u64)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
  39 }
  40
  41 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
  42 /**
  43  * i40e_program_fdir_filter - Program a Flow Director filter
  44  * @fdir_data: Packet data that will be filter parameters
  45  * @raw_packet: the pre-allocated packet buffer for FDir
  46  * @pf: The pf pointer
  47  * @add: True for add/update, False for remove
  48  **/
  49 int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
  50                              struct i40e_pf *pf, bool add)
  51 {
  52         struct i40e_filter_program_desc *fdir_desc;
  53         struct i40e_tx_buffer *tx_buf;
  54         struct i40e_tx_desc *tx_desc;
  55         struct i40e_ring *tx_ring;
  56         unsigned int fpt, dcc;
  57         struct i40e_vsi *vsi;
  58         struct device *dev;
  59         dma_addr_t dma;
  60         u32 td_cmd = 0;
  61         u16 i;
  62
  63         /* find existing FDIR VSI */
  64         vsi = NULL;
  65         for (i = 0; i < pf->num_alloc_vsi; i++)
  66                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
  67                         vsi = pf->vsi[i];
  68         if (!vsi)
  69                 return -ENOENT;
  70
  71         tx_ring = vsi->tx_rings[0];
  72         dev = tx_ring->dev;
  73
  74         dma = dma_map_single(dev, raw_packet,
  75                              I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
  76         if (dma_mapping_error(dev, dma))
  77                 goto dma_fail;
  78
  79         /* grab the next descriptor */
  80         i = tx_ring->next_to_use;
  81         fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
  82
  83         tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
  84
  85         fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
  86               I40E_TXD_FLTR_QW0_QINDEX_MASK;
  87
  88         fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
  89                I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
  90
  91         fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
  92                I40E_TXD_FLTR_QW0_PCTYPE_MASK;
  93
  94         /* Use LAN VSI Id if not programmed by user */
  95         if (fdir_data->dest_vsi == 0)
  96                 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
  97                        I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
  98         else
  99                 fpt |= ((u32)fdir_data->dest_vsi <<
 100                         I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
 101                        I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
 102
 103         fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
 104
 105         dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
 106
 107         if (add)
 108                 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
 109                        I40E_TXD_FLTR_QW1_PCMD_SHIFT;
 110         else
 111                 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
 112                        I40E_TXD_FLTR_QW1_PCMD_SHIFT;
 113
 114         dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
 115                I40E_TXD_FLTR_QW1_DEST_MASK;
 116
 117         dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
 118                I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
 119
 120         if (fdir_data->cnt_index != 0) {
 121                 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
 122                 dcc |= ((u32)fdir_data->cnt_index <<
 123                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
 124                         I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
 125         }
 126
 127         fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
 128         fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
 129
 130         /* Now program a dummy descriptor */
 131         i = tx_ring->next_to_use;
 132         tx_desc = I40E_TX_DESC(tx_ring, i);
 133         tx_buf = &tx_ring->tx_bi[i];
 134
 135         tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
 136
 137         /* record length, and DMA address */
 138         dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
 139         dma_unmap_addr_set(tx_buf, dma, dma);
 140
 141         tx_desc->buffer_addr = cpu_to_le64(dma);
 142         td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
 143
 144         tx_desc->cmd_type_offset_bsz =
 145                 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
 146
 147         /* set the timestamp */
 148         tx_buf->time_stamp = jiffies;
 149
 150         /* Force memory writes to complete before letting h/w
 151          * know there are new descriptors to fetch.  (Only
 152          * applicable for weak-ordered memory model archs,
 153          * such as IA-64).
 154          */
 155         wmb();
 156
 157         /* Mark the data descriptor to be watched */
 158         tx_buf->next_to_watch = tx_desc;
 159
 160         writel(tx_ring->next_to_use, tx_ring->tail);
 161         return 0;
 162
 163 dma_fail:
 164         return -1;
 165 }
 166
 167 #define IP_HEADER_OFFSET 14
 168 #define I40E_UDPIP_DUMMY_PACKET_LEN 42
 169 /**
 170  * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
 171  * @vsi: pointer to the targeted VSI
 172  * @fd_data: the flow director data required for the FDir descriptor
 173  * @raw_packet: the pre-allocated packet buffer for FDir
 174  * @add: true adds a filter, false removes it
 175  *
 176  * Returns 0 if the filters were successfully added or removed
 177  **/
 178 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
 179                                    struct i40e_fdir_filter *fd_data,
 180                                    u8 *raw_packet, bool add)
 181 {
 182         struct i40e_pf *pf = vsi->back;
 183         struct udphdr *udp;
 184         struct iphdr *ip;
 185         bool err = false;
 186         int ret;
 187         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 188                 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
 189                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 190
 191         memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
 192
 193         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 194         udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
 195               + sizeof(struct iphdr));
 196
 197         ip->daddr = fd_data->dst_ip[0];
 198         udp->dest = fd_data->dst_port;
 199         ip->saddr = fd_data->src_ip[0];
 200         udp->source = fd_data->src_port;
 201
 202         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
 203         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 204         if (ret) {
 205                 dev_info(&pf->pdev->dev,
 206                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 207                          fd_data->pctype, ret);
 208                 err = true;
 209         } else {
 210                 dev_info(&pf->pdev->dev,
 211                          "Filter OK for PCTYPE %d (ret = %d)\n",
 212                          fd_data->pctype, ret);
 213         }
 214
 215         return err ? -EOPNOTSUPP : 0;
 216 }
 217
 218 #define I40E_TCPIP_DUMMY_PACKET_LEN 54
 219 /**
 220  * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
 221  * @vsi: pointer to the targeted VSI
 222  * @fd_data: the flow director data required for the FDir descriptor
 223  * @raw_packet: the pre-allocated packet buffer for FDir
 224  * @add: true adds a filter, false removes it
 225  *
 226  * Returns 0 if the filters were successfully added or removed
 227  **/
 228 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
 229                                    struct i40e_fdir_filter *fd_data,
 230                                    u8 *raw_packet, bool add)
 231 {
 232         struct i40e_pf *pf = vsi->back;
 233         struct tcphdr *tcp;
 234         struct iphdr *ip;
 235         bool err = false;
 236         int ret;
 237         /* Dummy packet */
 238         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 239                 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
 240                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
 241                 0x0, 0x72, 0, 0, 0, 0};
 242
 243         memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
 244
 245         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 246         tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
 247               + sizeof(struct iphdr));
 248
 249         ip->daddr = fd_data->dst_ip[0];
 250         tcp->dest = fd_data->dst_port;
 251         ip->saddr = fd_data->src_ip[0];
 252         tcp->source = fd_data->src_port;
 253
 254         if (add) {
 255                 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
 256                         dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
 257                         pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 258                 }
 259         }
 260
 261         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
 262         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 263
 264         if (ret) {
 265                 dev_info(&pf->pdev->dev,
 266                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 267                          fd_data->pctype, ret);
 268                 err = true;
 269         } else {
 270                 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
 271                          fd_data->pctype, ret);
 272         }
 273
 274         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
 275
 276         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 277         if (ret) {
 278                 dev_info(&pf->pdev->dev,
 279                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 280                          fd_data->pctype, ret);
 281                 err = true;
 282         } else {
 283                 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
 284                           fd_data->pctype, ret);
 285         }
 286
 287         return err ? -EOPNOTSUPP : 0;
 288 }
 289
 290 /**
 291  * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
 292  * a specific flow spec
 293  * @vsi: pointer to the targeted VSI
 294  * @fd_data: the flow director data required for the FDir descriptor
 295  * @raw_packet: the pre-allocated packet buffer for FDir
 296  * @add: true adds a filter, false removes it
 297  *
 298  * Always returns -EOPNOTSUPP
 299  **/
 300 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
 301                                     struct i40e_fdir_filter *fd_data,
 302                                     u8 *raw_packet, bool add)
 303 {
 304         return -EOPNOTSUPP;
 305 }
 306
 307 #define I40E_IP_DUMMY_PACKET_LEN 34
 308 /**
 309  * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
 310  * a specific flow spec
 311  * @vsi: pointer to the targeted VSI
 312  * @fd_data: the flow director data required for the FDir descriptor
 313  * @raw_packet: the pre-allocated packet buffer for FDir
 314  * @add: true adds a filter, false removes it
 315  *
 316  * Returns 0 if the filters were successfully added or removed
 317  **/
 318 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
 319                                   struct i40e_fdir_filter *fd_data,
 320                                   u8 *raw_packet, bool add)
 321 {
 322         struct i40e_pf *pf = vsi->back;
 323         struct iphdr *ip;
 324         bool err = false;
 325         int ret;
 326         int i;
 327         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 328                 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
 329                 0, 0, 0, 0};
 330
 331         memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
 332         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 333
 334         ip->saddr = fd_data->src_ip[0];
 335         ip->daddr = fd_data->dst_ip[0];
 336         ip->protocol = 0;
 337
 338         for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
 339              i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
 340                 fd_data->pctype = i;
 341                 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 342
 343                 if (ret) {
 344                         dev_info(&pf->pdev->dev,
 345                                  "Filter command send failed for PCTYPE %d (ret = %d)\n",
 346                                  fd_data->pctype, ret);
 347                         err = true;
 348                 } else {
 349                         dev_info(&pf->pdev->dev,
 350                                  "Filter OK for PCTYPE %d (ret = %d)\n",
 351                                  fd_data->pctype, ret);
 352                 }
 353         }
 354
 355         return err ? -EOPNOTSUPP : 0;
 356 }
 357
 358 /**
 359  * i40e_add_del_fdir - Build raw packets to add/del fdir filter
 360  * @vsi: pointer to the targeted VSI
 361  * @cmd: command to get or set RX flow classification rules
 362  * @add: true adds a filter, false removes it
 363  *
 364  **/
 365 int i40e_add_del_fdir(struct i40e_vsi *vsi,
 366                       struct i40e_fdir_filter *input, bool add)
 367 {
 368         struct i40e_pf *pf = vsi->back;
 369         u8 *raw_packet;
 370         int ret;
 371
 372         /* Populate the Flow Director that we have at the moment
 373          * and allocate the raw packet buffer for the calling functions
 374          */
 375         raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
 376         if (!raw_packet)
 377                 return -ENOMEM;
 378
 379         switch (input->flow_type & ~FLOW_EXT) {
 380         case TCP_V4_FLOW:
 381                 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet,
 382                                               add);
 383                 break;
 384         case UDP_V4_FLOW:
 385                 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet,
 386                                               add);
 387                 break;
 388         case SCTP_V4_FLOW:
 389                 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet,
 390                                                add);
 391                 break;
 392         case IPV4_FLOW:
 393                 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet,
 394                                              add);
 395                 break;
 396         case IP_USER_FLOW:
 397                 switch (input->ip4_proto) {
 398                 case IPPROTO_TCP:
 399                         ret = i40e_add_del_fdir_tcpv4(vsi, input,
 400                                                       raw_packet, add);
 401                         break;
 402                 case IPPROTO_UDP:
 403                         ret = i40e_add_del_fdir_udpv4(vsi, input,
 404                                                       raw_packet, add);
 405                         break;
 406                 case IPPROTO_SCTP:
 407                         ret = i40e_add_del_fdir_sctpv4(vsi, input,
 408                                                        raw_packet, add);
 409                         break;
 410                 default:
 411                         ret = i40e_add_del_fdir_ipv4(vsi, input,
 412                                                      raw_packet, add);
 413                         break;
 414                 }
 415                 break;
 416         default:
 417                 dev_info(&pf->pdev->dev, "Could not specify spec type %d\n",
 418                          input->flow_type);
 419                 ret = -EINVAL;
 420         }
 421
 422         kfree(raw_packet);
 423         return ret;
 424 }
 425
 426 /**
 427  * i40e_fd_handle_status - check the Programming Status for FD
 428  * @rx_ring: the Rx ring for this descriptor
 429  * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
 430  * @prog_id: the id originally used for programming
 431  *
 432  * This is used to verify if the FD programming or invalidation
 433  * requested by SW to the HW is successful or not and take actions accordingly.
 434  **/
 435 static void i40e_fd_handle_status(struct i40e_ring *rx_ring,
 436                                   union i40e_rx_desc *rx_desc, u8 prog_id)
 437 {
 438         struct i40e_pf *pf = rx_ring->vsi->back;
 439         struct pci_dev *pdev = pf->pdev;
 440         u32 fcnt_prog, fcnt_avail;
 441         u32 error;
 442         u64 qw;
 443
 444         qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
 445         error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
 446                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
 447
 448         if (error == (0x1 << I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
 449                 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
 450                          rx_desc->wb.qword0.hi_dword.fd_id);
 451
 452                 /* filter programming failed most likely due to table full */
 453                 fcnt_prog = i40e_get_current_fd_count(pf);
 454                 fcnt_avail = i40e_get_fd_cnt_all(pf);
 455                 /* If ATR is running fcnt_prog can quickly change,
 456                  * if we are very close to full, it makes sense to disable
 457                  * FD ATR/SB and then re-enable it when there is room.
 458                  */
 459                 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
 460                         /* Turn off ATR first */
 461                         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
 462                                 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 463                                 dev_warn(&pdev->dev, "FD filter space full, ATR for further flows will be turned off\n");
 464                                 pf->auto_disable_flags |=
 465                                                        I40E_FLAG_FD_ATR_ENABLED;
 466                                 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
 467                         } else if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
 468                                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
 469                                 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
 470                                 pf->auto_disable_flags |=
 471                                                         I40E_FLAG_FD_SB_ENABLED;
 472                                 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
 473                         }
 474                 } else {
 475                         dev_info(&pdev->dev, "FD filter programming error\n");
 476                 }
 477         } else if (error ==
 478                           (0x1 << I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
 479                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
 480                         dev_info(&pdev->dev, "ntuple filter loc = %d, could not be removed\n",
 481                                  rx_desc->wb.qword0.hi_dword.fd_id);
 482         }
 483 }
 484
 485 /**
 486  * i40e_unmap_and_free_tx_resource - Release a Tx buffer
 487  * @ring:      the ring that owns the buffer
 488  * @tx_buffer: the buffer to free
 489  **/
 490 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 491                                             struct i40e_tx_buffer *tx_buffer)
 492 {
 493         if (tx_buffer->skb) {
 494                 dev_kfree_skb_any(tx_buffer->skb);
 495                 if (dma_unmap_len(tx_buffer, len))
 496                         dma_unmap_single(ring->dev,
 497                                          dma_unmap_addr(tx_buffer, dma),
 498                                          dma_unmap_len(tx_buffer, len),
 499                                          DMA_TO_DEVICE);
 500         } else if (dma_unmap_len(tx_buffer, len)) {
 501                 dma_unmap_page(ring->dev,
 502                                dma_unmap_addr(tx_buffer, dma),
 503                                dma_unmap_len(tx_buffer, len),
 504                                DMA_TO_DEVICE);
 505         }
 506         tx_buffer->next_to_watch = NULL;
 507         tx_buffer->skb = NULL;
 508         dma_unmap_len_set(tx_buffer, len, 0);
 509         /* tx_buffer must be completely set up in the transmit path */
 510 }
 511
 512 /**
 513  * i40e_clean_tx_ring - Free any empty Tx buffers
 514  * @tx_ring: ring to be cleaned
 515  **/
 516 void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
 517 {
 518         unsigned long bi_size;
 519         u16 i;
 520
 521         /* ring already cleared, nothing to do */
 522         if (!tx_ring->tx_bi)
 523                 return;
 524
 525         /* Free all the Tx ring sk_buffs */
 526         for (i = 0; i < tx_ring->count; i++)
 527                 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
 528
 529         bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
 530         memset(tx_ring->tx_bi, 0, bi_size);
 531
 532         /* Zero out the descriptor ring */
 533         memset(tx_ring->desc, 0, tx_ring->size);
 534
 535         tx_ring->next_to_use = 0;
 536         tx_ring->next_to_clean = 0;
 537
 538         if (!tx_ring->netdev)
 539                 return;
 540
 541         /* cleanup Tx queue statistics */
 542         netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
 543                                                   tx_ring->queue_index));
 544 }
 545
 546 /**
 547  * i40e_free_tx_resources - Free Tx resources per queue
 548  * @tx_ring: Tx descriptor ring for a specific queue
 549  *
 550  * Free all transmit software resources
 551  **/
 552 void i40e_free_tx_resources(struct i40e_ring *tx_ring)
 553 {
 554         i40e_clean_tx_ring(tx_ring);
 555         kfree(tx_ring->tx_bi);
 556         tx_ring->tx_bi = NULL;
 557
 558         if (tx_ring->desc) {
 559                 dma_free_coherent(tx_ring->dev, tx_ring->size,
 560                                   tx_ring->desc, tx_ring->dma);
 561                 tx_ring->desc = NULL;
 562         }
 563 }
 564
 565 /**
 566  * i40e_get_tx_pending - how many tx descriptors not processed
 567  * @tx_ring: the ring of descriptors
 568  *
 569  * Since there is no access to the ring head register
 570  * in XL710, we need to use our local copies
 571  **/
 572 static u32 i40e_get_tx_pending(struct i40e_ring *ring)
 573 {
 574         u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
 575                         ? ring->next_to_use
 576                         : ring->next_to_use + ring->count);
 577         return ntu - ring->next_to_clean;
 578 }
 579
 580 /**
 581  * i40e_check_tx_hang - Is there a hang in the Tx queue
 582  * @tx_ring: the ring of descriptors
 583  **/
 584 static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
 585 {
 586         u32 tx_pending = i40e_get_tx_pending(tx_ring);
 587         bool ret = false;
 588
 589         clear_check_for_tx_hang(tx_ring);
 590
 591         /* Check for a hung queue, but be thorough. This verifies
 592          * that a transmit has been completed since the previous
 593          * check AND there is at least one packet pending. The
 594          * ARMED bit is set to indicate a potential hang. The
 595          * bit is cleared if a pause frame is received to remove
 596          * false hang detection due to PFC or 802.3x frames. By
 597          * requiring this to fail twice we avoid races with
 598          * PFC clearing the ARMED bit and conditions where we
 599          * run the check_tx_hang logic with a transmit completion
 600          * pending but without time to complete it yet.
 601          */
 602         if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
 603             tx_pending) {
 604                 /* make sure it is true for two checks in a row */
 605                 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
 606                                        &tx_ring->state);
 607         } else {
 608                 /* update completed stats and disarm the hang check */
 609                 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
 610                 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
 611         }
 612
 613         return ret;
 614 }
 615
 616 /**
 617  * i40e_get_head - Retrieve head from head writeback
 618  * @tx_ring:  tx ring to fetch head of
 619  *
 620  * Returns value of Tx ring head based on value stored
 621  * in head write-back location
 622  **/
 623 static inline u32 i40e_get_head(struct i40e_ring *tx_ring)
 624 {
 625         void *head = (struct i40e_tx_desc *)tx_ring->desc + tx_ring->count;
 626
 627         return le32_to_cpu(*(volatile __le32 *)head);
 628 }
 629
 630 /**
 631  * i40e_clean_tx_irq - Reclaim resources after transmit completes
 632  * @tx_ring:  tx ring to clean
 633  * @budget:   how many cleans we're allowed
 634  *
 635  * Returns true if there's any budget left (e.g. the clean is finished)
 636  **/
 637 static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
 638 {
 639         u16 i = tx_ring->next_to_clean;
 640         struct i40e_tx_buffer *tx_buf;
 641         struct i40e_tx_desc *tx_head;
 642         struct i40e_tx_desc *tx_desc;
 643         unsigned int total_packets = 0;
 644         unsigned int total_bytes = 0;
 645
 646         tx_buf = &tx_ring->tx_bi[i];
 647         tx_desc = I40E_TX_DESC(tx_ring, i);
 648         i -= tx_ring->count;
 649
 650         tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));
 651
 652         do {
 653                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
 654
 655                 /* if next_to_watch is not set then there is no work pending */
 656                 if (!eop_desc)
 657                         break;
 658
 659                 /* prevent any other reads prior to eop_desc */
 660                 read_barrier_depends();
 661
 662                 /* we have caught up to head, no work left to do */
 663                 if (tx_head == tx_desc)
 664                         break;
 665
 666                 /* clear next_to_watch to prevent false hangs */
 667                 tx_buf->next_to_watch = NULL;
 668
 669                 /* update the statistics for this packet */
 670                 total_bytes += tx_buf->bytecount;
 671                 total_packets += tx_buf->gso_segs;
 672
 673                 /* free the skb */
 674                 dev_kfree_skb_any(tx_buf->skb);
 675
 676                 /* unmap skb header data */
 677                 dma_unmap_single(tx_ring->dev,
 678                                  dma_unmap_addr(tx_buf, dma),
 679                                  dma_unmap_len(tx_buf, len),
 680                                  DMA_TO_DEVICE);
 681
 682                 /* clear tx_buffer data */
 683                 tx_buf->skb = NULL;
 684                 dma_unmap_len_set(tx_buf, len, 0);
 685
 686                 /* unmap remaining buffers */
 687                 while (tx_desc != eop_desc) {
 688
 689                         tx_buf++;
 690                         tx_desc++;
 691                         i++;
 692                         if (unlikely(!i)) {
 693                                 i -= tx_ring->count;
 694                                 tx_buf = tx_ring->tx_bi;
 695                                 tx_desc = I40E_TX_DESC(tx_ring, 0);
 696                         }
 697
 698                         /* unmap any remaining paged data */
 699                         if (dma_unmap_len(tx_buf, len)) {
 700                                 dma_unmap_page(tx_ring->dev,
 701                                                dma_unmap_addr(tx_buf, dma),
 702                                                dma_unmap_len(tx_buf, len),
 703                                                DMA_TO_DEVICE);
 704                                 dma_unmap_len_set(tx_buf, len, 0);
 705                         }
 706                 }
 707
 708                 /* move us one more past the eop_desc for start of next pkt */
 709                 tx_buf++;
 710                 tx_desc++;
 711                 i++;
 712                 if (unlikely(!i)) {
 713                         i -= tx_ring->count;
 714                         tx_buf = tx_ring->tx_bi;
 715                         tx_desc = I40E_TX_DESC(tx_ring, 0);
 716                 }
 717
 718                 /* update budget accounting */
 719                 budget--;
 720         } while (likely(budget));
 721
 722         i += tx_ring->count;
 723         tx_ring->next_to_clean = i;
 724         u64_stats_update_begin(&tx_ring->syncp);
 725         tx_ring->stats.bytes += total_bytes;
 726         tx_ring->stats.packets += total_packets;
 727         u64_stats_update_end(&tx_ring->syncp);
 728         tx_ring->q_vector->tx.total_bytes += total_bytes;
 729         tx_ring->q_vector->tx.total_packets += total_packets;
 730
 731         if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
 732                 /* schedule immediate reset if we believe we hung */
 733                 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
 734                          "  VSI                  <%d>\n"
 735                          "  Tx Queue             <%d>\n"
 736                          "  next_to_use          <%x>\n"
 737                          "  next_to_clean        <%x>\n",
 738                          tx_ring->vsi->seid,
 739                          tx_ring->queue_index,
 740                          tx_ring->next_to_use, i);
 741                 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
 742                          "  time_stamp           <%lx>\n"
 743                          "  jiffies              <%lx>\n",
 744                          tx_ring->tx_bi[i].time_stamp, jiffies);
 745
 746                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 747
 748                 dev_info(tx_ring->dev,
 749                          "tx hang detected on queue %d, resetting adapter\n",
 750                          tx_ring->queue_index);
 751
 752                 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
 753
 754                 /* the adapter is about to reset, no point in enabling stuff */
 755                 return true;
 756         }
 757
 758         netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
 759                                                       tx_ring->queue_index),
 760                                   total_packets, total_bytes);
 761
 762 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 763         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 764                      (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 765                 /* Make sure that anybody stopping the queue after this
 766                  * sees the new next_to_clean.
 767                  */
 768                 smp_mb();
 769                 if (__netif_subqueue_stopped(tx_ring->netdev,
 770                                              tx_ring->queue_index) &&
 771                    !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
 772                         netif_wake_subqueue(tx_ring->netdev,
 773                                             tx_ring->queue_index);
 774                         ++tx_ring->tx_stats.restart_queue;
 775                 }
 776         }
 777
 778         return budget > 0;
 779 }
 780
 781 /**
 782  * i40e_set_new_dynamic_itr - Find new ITR level
 783  * @rc: structure containing ring performance data
 784  *
 785  * Stores a new ITR value based on packets and byte counts during
 786  * the last interrupt.  The advantage of per interrupt computation
 787  * is faster updates and more accurate ITR for the current traffic
 788  * pattern.  Constants in this function were computed based on
 789  * theoretical maximum wire speed and thresholds were set based on
 790  * testing data as well as attempting to minimize response time
 791  * while increasing bulk throughput.
 792  **/
 793 static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 794 {
 795         enum i40e_latency_range new_latency_range = rc->latency_range;
 796         u32 new_itr = rc->itr;
 797         int bytes_per_int;
 798
 799         if (rc->total_packets == 0 || !rc->itr)
 800                 return;
 801
 802         /* simple throttlerate management
 803          *   0-10MB/s   lowest (100000 ints/s)
 804          *  10-20MB/s   low    (20000 ints/s)
 805          *  20-1249MB/s bulk   (8000 ints/s)
 806          */
 807         bytes_per_int = rc->total_bytes / rc->itr;
 808         switch (rc->itr) {
 809         case I40E_LOWEST_LATENCY:
 810                 if (bytes_per_int > 10)
 811                         new_latency_range = I40E_LOW_LATENCY;
 812                 break;
 813         case I40E_LOW_LATENCY:
 814                 if (bytes_per_int > 20)
 815                         new_latency_range = I40E_BULK_LATENCY;
 816                 else if (bytes_per_int <= 10)
 817                         new_latency_range = I40E_LOWEST_LATENCY;
 818                 break;
 819         case I40E_BULK_LATENCY:
 820                 if (bytes_per_int <= 20)
 821                         rc->latency_range = I40E_LOW_LATENCY;
 822                 break;
 823         }
 824
 825         switch (new_latency_range) {
 826         case I40E_LOWEST_LATENCY:
 827                 new_itr = I40E_ITR_100K;
 828                 break;
 829         case I40E_LOW_LATENCY:
 830                 new_itr = I40E_ITR_20K;
 831                 break;
 832         case I40E_BULK_LATENCY:
 833                 new_itr = I40E_ITR_8K;
 834                 break;
 835         default:
 836                 break;
 837         }
 838
 839         if (new_itr != rc->itr) {
 840                 /* do an exponential smoothing */
 841                 new_itr = (10 * new_itr * rc->itr) /
 842                           ((9 * new_itr) + rc->itr);
 843                 rc->itr = new_itr & I40E_MAX_ITR;
 844         }
 845
 846         rc->total_bytes = 0;
 847         rc->total_packets = 0;
 848 }
 849
 850 /**
 851  * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
 852  * @q_vector: the vector to adjust
 853  **/
 854 static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
 855 {
 856         u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
 857         struct i40e_hw *hw = &q_vector->vsi->back->hw;
 858         u32 reg_addr;
 859         u16 old_itr;
 860
 861         reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
 862         old_itr = q_vector->rx.itr;
 863         i40e_set_new_dynamic_itr(&q_vector->rx);
 864         if (old_itr != q_vector->rx.itr)
 865                 wr32(hw, reg_addr, q_vector->rx.itr);
 866
 867         reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
 868         old_itr = q_vector->tx.itr;
 869         i40e_set_new_dynamic_itr(&q_vector->tx);
 870         if (old_itr != q_vector->tx.itr)
 871                 wr32(hw, reg_addr, q_vector->tx.itr);
 872 }
 873
 874 /**
 875  * i40e_clean_programming_status - clean the programming status descriptor
 876  * @rx_ring: the rx ring that has this descriptor
 877  * @rx_desc: the rx descriptor written back by HW
 878  *
 879  * Flow director should handle FD_FILTER_STATUS to check its filter programming
 880  * status being successful or not and take actions accordingly. FCoE should
 881  * handle its context/filter programming/invalidation status and take actions.
 882  *
 883  **/
 884 static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
 885                                           union i40e_rx_desc *rx_desc)
 886 {
 887         u64 qw;
 888         u8 id;
 889
 890         qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
 891         id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
 892                   I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
 893
 894         if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
 895                 i40e_fd_handle_status(rx_ring, rx_desc, id);
 896 }
 897
 898 /**
 899  * i40e_setup_tx_descriptors - Allocate the Tx descriptors
 900  * @tx_ring: the tx ring to set up
 901  *
 902  * Return 0 on success, negative on error
 903  **/
 904 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
 905 {
 906         struct device *dev = tx_ring->dev;
 907         int bi_size;
 908
 909         if (!dev)
 910                 return -ENOMEM;
 911
 912         bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
 913         tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
 914         if (!tx_ring->tx_bi)
 915                 goto err;
 916
 917         /* round up to nearest 4K */
 918         tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
 919         /* add u32 for head writeback, align after this takes care of
 920          * guaranteeing this is at least one cache line in size
 921          */
 922         tx_ring->size += sizeof(u32);
 923         tx_ring->size = ALIGN(tx_ring->size, 4096);
 924         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 925                                            &tx_ring->dma, GFP_KERNEL);
 926         if (!tx_ring->desc) {
 927                 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
 928                          tx_ring->size);
 929                 goto err;
 930         }
 931
 932         tx_ring->next_to_use = 0;
 933         tx_ring->next_to_clean = 0;
 934         return 0;
 935
 936 err:
 937         kfree(tx_ring->tx_bi);
 938         tx_ring->tx_bi = NULL;
 939         return -ENOMEM;
 940 }
 941
 942 /**
 943  * i40e_clean_rx_ring - Free Rx buffers
 944  * @rx_ring: ring to be cleaned
 945  **/
 946 void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
 947 {
 948         struct device *dev = rx_ring->dev;
 949         struct i40e_rx_buffer *rx_bi;
 950         unsigned long bi_size;
 951         u16 i;
 952
 953         /* ring already cleared, nothing to do */
 954         if (!rx_ring->rx_bi)
 955                 return;
 956
 957         /* Free all the Rx ring sk_buffs */
 958         for (i = 0; i < rx_ring->count; i++) {
 959                 rx_bi = &rx_ring->rx_bi[i];
 960                 if (rx_bi->dma) {
 961                         dma_unmap_single(dev,
 962                                          rx_bi->dma,
 963                                          rx_ring->rx_buf_len,
 964                                          DMA_FROM_DEVICE);
 965                         rx_bi->dma = 0;
 966                 }
 967                 if (rx_bi->skb) {
 968                         dev_kfree_skb(rx_bi->skb);
 969                         rx_bi->skb = NULL;
 970                 }
 971                 if (rx_bi->page) {
 972                         if (rx_bi->page_dma) {
 973                                 dma_unmap_page(dev,
 974                                                rx_bi->page_dma,
 975                                                PAGE_SIZE / 2,
 976                                                DMA_FROM_DEVICE);
 977                                 rx_bi->page_dma = 0;
 978                         }
 979                         __free_page(rx_bi->page);
 980                         rx_bi->page = NULL;
 981                         rx_bi->page_offset = 0;
 982                 }
 983         }
 984
 985         bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
 986         memset(rx_ring->rx_bi, 0, bi_size);
 987
 988         /* Zero out the descriptor ring */
 989         memset(rx_ring->desc, 0, rx_ring->size);
 990
 991         rx_ring->next_to_clean = 0;
 992         rx_ring->next_to_use = 0;
 993 }
 994
 995 /**
 996  * i40e_free_rx_resources - Free Rx resources
 997  * @rx_ring: ring to clean the resources from
 998  *
 999  * Free all receive software resources
1000  **/
1001 void i40e_free_rx_resources(struct i40e_ring *rx_ring)
1002 {
1003         i40e_clean_rx_ring(rx_ring);
1004         kfree(rx_ring->rx_bi);
1005         rx_ring->rx_bi = NULL;
1006
1007         if (rx_ring->desc) {
1008                 dma_free_coherent(rx_ring->dev, rx_ring->size,
1009                                   rx_ring->desc, rx_ring->dma);
1010                 rx_ring->desc = NULL;
1011         }
1012 }
1013
1014 /**
1015  * i40e_setup_rx_descriptors - Allocate Rx descriptors
1016  * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1017  *
1018  * Returns 0 on success, negative on failure
1019  **/
1020 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1021 {
1022         struct device *dev = rx_ring->dev;
1023         int bi_size;
1024
1025         bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1026         rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
1027         if (!rx_ring->rx_bi)
1028                 goto err;
1029
1030         /* Round up to nearest 4K */
1031         rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
1032                 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
1033                 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
1034         rx_ring->size = ALIGN(rx_ring->size, 4096);
1035         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1036                                            &rx_ring->dma, GFP_KERNEL);
1037
1038         if (!rx_ring->desc) {
1039                 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1040                          rx_ring->size);
1041                 goto err;
1042         }
1043
1044         rx_ring->next_to_clean = 0;
1045         rx_ring->next_to_use = 0;
1046
1047         return 0;
1048 err:
1049         kfree(rx_ring->rx_bi);
1050         rx_ring->rx_bi = NULL;
1051         return -ENOMEM;
1052 }
1053
1054 /**
1055  * i40e_release_rx_desc - Store the new tail and head values
1056  * @rx_ring: ring to bump
1057  * @val: new head index
1058  **/
1059 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1060 {
1061         rx_ring->next_to_use = val;
1062         /* Force memory writes to complete before letting h/w
1063          * know there are new descriptors to fetch.  (Only
1064          * applicable for weak-ordered memory model archs,
1065          * such as IA-64).
1066          */
1067         wmb();
1068         writel(val, rx_ring->tail);
1069 }
1070
1071 /**
1072  * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1073  * @rx_ring: ring to place buffers on
1074  * @cleaned_count: number of buffers to replace
1075  **/
1076 void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1077 {
1078         u16 i = rx_ring->next_to_use;
1079         union i40e_rx_desc *rx_desc;
1080         struct i40e_rx_buffer *bi;
1081         struct sk_buff *skb;
1082
1083         /* do nothing if no valid netdev defined */
1084         if (!rx_ring->netdev || !cleaned_count)
1085                 return;
1086
1087         while (cleaned_count--) {
1088                 rx_desc = I40E_RX_DESC(rx_ring, i);
1089                 bi = &rx_ring->rx_bi[i];
1090                 skb = bi->skb;
1091
1092                 if (!skb) {
1093                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1094                                                         rx_ring->rx_buf_len);
1095                         if (!skb) {
1096                                 rx_ring->rx_stats.alloc_buff_failed++;
1097                                 goto no_buffers;
1098                         }
1099                         /* initialize queue mapping */
1100                         skb_record_rx_queue(skb, rx_ring->queue_index);
1101                         bi->skb = skb;
1102                 }
1103
1104                 if (!bi->dma) {
1105                         bi->dma = dma_map_single(rx_ring->dev,
1106                                                  skb->data,
1107                                                  rx_ring->rx_buf_len,
1108                                                  DMA_FROM_DEVICE);
1109                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1110                                 rx_ring->rx_stats.alloc_buff_failed++;
1111                                 bi->dma = 0;
1112                                 goto no_buffers;
1113                         }
1114                 }
1115
1116                 if (ring_is_ps_enabled(rx_ring)) {
1117                         if (!bi->page) {
1118                                 bi->page = alloc_page(GFP_ATOMIC);
1119                                 if (!bi->page) {
1120                                         rx_ring->rx_stats.alloc_page_failed++;
1121                                         goto no_buffers;
1122                                 }
1123                         }
1124
1125                         if (!bi->page_dma) {
1126                                 /* use a half page if we're re-using */
1127                                 bi->page_offset ^= PAGE_SIZE / 2;
1128                                 bi->page_dma = dma_map_page(rx_ring->dev,
1129                                                             bi->page,
1130                                                             bi->page_offset,
1131                                                             PAGE_SIZE / 2,
1132                                                             DMA_FROM_DEVICE);
1133                                 if (dma_mapping_error(rx_ring->dev,
1134                                                       bi->page_dma)) {
1135                                         rx_ring->rx_stats.alloc_page_failed++;
1136                                         bi->page_dma = 0;
1137                                         goto no_buffers;
1138                                 }
1139                         }
1140
1141                         /* Refresh the desc even if buffer_addrs didn't change
1142                          * because each write-back erases this info.
1143                          */
1144                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1145                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1146                 } else {
1147                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1148                         rx_desc->read.hdr_addr = 0;
1149                 }
1150                 i++;
1151                 if (i == rx_ring->count)
1152                         i = 0;
1153         }
1154
1155 no_buffers:
1156         if (rx_ring->next_to_use != i)
1157                 i40e_release_rx_desc(rx_ring, i);
1158 }
1159
1160 /**
1161  * i40e_receive_skb - Send a completed packet up the stack
1162  * @rx_ring:  rx ring in play
1163  * @skb: packet to send up
1164  * @vlan_tag: vlan tag for packet
1165  **/
1166 static void i40e_receive_skb(struct i40e_ring *rx_ring,
1167                              struct sk_buff *skb, u16 vlan_tag)
1168 {
1169         struct i40e_q_vector *q_vector = rx_ring->q_vector;
1170         struct i40e_vsi *vsi = rx_ring->vsi;
1171         u64 flags = vsi->back->flags;
1172
1173         if (vlan_tag & VLAN_VID_MASK)
1174                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1175
1176         if (flags & I40E_FLAG_IN_NETPOLL)
1177                 netif_rx(skb);
1178         else
1179                 napi_gro_receive(&q_vector->napi, skb);
1180 }
1181
1182 /**
1183  * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1184  * @vsi: the VSI we care about
1185  * @skb: skb currently being received and modified
1186  * @rx_status: status value of last descriptor in packet
1187  * @rx_error: error value of last descriptor in packet
1188  * @rx_ptype: ptype value of last descriptor in packet
1189  **/
1190 static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1191                                     struct sk_buff *skb,
1192                                     u32 rx_status,
1193                                     u32 rx_error,
1194                                     u16 rx_ptype)
1195 {
1196         struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(rx_ptype);
1197         bool ipv4 = false, ipv6 = false;
1198         bool ipv4_tunnel, ipv6_tunnel;
1199         __wsum rx_udp_csum;
1200         struct iphdr *iph;
1201         __sum16 csum;
1202
1203         ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1204                       (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1205         ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1206                       (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1207
1208         skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
1209         skb->ip_summed = CHECKSUM_NONE;
1210
1211         /* Rx csum enabled and ip headers found? */
1212         if (!(vsi->netdev->features & NETIF_F_RXCSUM))
1213                 return;
1214
1215         /* did the hardware decode the packet and checksum? */
1216         if (!(rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1217                 return;
1218
1219         /* both known and outer_ip must be set for the below code to work */
1220         if (!(decoded.known && decoded.outer_ip))
1221                 return;
1222
1223         if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1224             decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4)
1225                 ipv4 = true;
1226         else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1227                  decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1228                 ipv6 = true;
1229
1230         if (ipv4 &&
1231             (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1232                          (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))))
1233                 goto checksum_fail;
1234
1235         /* likely incorrect csum if alternate IP extension headers found */
1236         if (ipv6 &&
1237             decoded.inner_prot == I40E_RX_PTYPE_INNER_PROT_TCP &&
1238             rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) &&
1239             rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1240                 /* don't increment checksum err here, non-fatal err */
1241                 return;
1242
1243         /* there was some L4 error, count error and punt packet to the stack */
1244         if (rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))
1245                 goto checksum_fail;
1246
1247         /* handle packets that were not able to be checksummed due
1248          * to arrival speed, in this case the stack can compute
1249          * the csum.
1250          */
1251         if (rx_error & (1 << I40E_RX_DESC_ERROR_PPRS_SHIFT))
1252                 return;
1253
1254         /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1255          * it in the driver, hardware does not do it for us.
1256          * Since L3L4P bit was set we assume a valid IHL value (>=5)
1257          * so the total length of IPv4 header is IHL*4 bytes
1258          * The UDP_0 bit *may* bet set if the *inner* header is UDP
1259          */
1260         if (ipv4_tunnel &&
1261             (decoded.inner_prot != I40E_RX_PTYPE_INNER_PROT_UDP) &&
1262             !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
1263                 skb->transport_header = skb->mac_header +
1264                                         sizeof(struct ethhdr) +
1265                                         (ip_hdr(skb)->ihl * 4);
1266
1267                 /* Add 4 bytes for VLAN tagged packets */
1268                 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1269                                           skb->protocol == htons(ETH_P_8021AD))
1270                                           ? VLAN_HLEN : 0;
1271
1272                 rx_udp_csum = udp_csum(skb);
1273                 iph = ip_hdr(skb);
1274                 csum = csum_tcpudp_magic(
1275                                 iph->saddr, iph->daddr,
1276                                 (skb->len - skb_transport_offset(skb)),
1277                                 IPPROTO_UDP, rx_udp_csum);
1278
1279                 if (udp_hdr(skb)->check != csum)
1280                         goto checksum_fail;
1281         }
1282
1283         skb->ip_summed = CHECKSUM_UNNECESSARY;
1284
1285         return;
1286
1287 checksum_fail:
1288         vsi->back->hw_csum_rx_error++;
1289 }
1290
1291 /**
1292  * i40e_rx_hash - returns the hash value from the Rx descriptor
1293  * @ring: descriptor ring
1294  * @rx_desc: specific descriptor
1295  **/
1296 static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1297                                union i40e_rx_desc *rx_desc)
1298 {
1299         const __le64 rss_mask =
1300                 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1301                             I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1302
1303         if ((ring->netdev->features & NETIF_F_RXHASH) &&
1304             (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1305                 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1306         else
1307                 return 0;
1308 }
1309
1310 /**
1311  * i40e_ptype_to_hash - get a hash type
1312  * @ptype: the ptype value from the descriptor
1313  *
1314  * Returns a hash type to be used by skb_set_hash
1315  **/
1316 static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype)
1317 {
1318         struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1319
1320         if (!decoded.known)
1321                 return PKT_HASH_TYPE_NONE;
1322
1323         if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1324             decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1325                 return PKT_HASH_TYPE_L4;
1326         else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1327                  decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1328                 return PKT_HASH_TYPE_L3;
1329         else
1330                 return PKT_HASH_TYPE_L2;
1331 }
1332
1333 /**
1334  * i40e_clean_rx_irq - Reclaim resources after receive completes
1335  * @rx_ring:  rx ring to clean
1336  * @budget:   how many cleans we're allowed
1337  *
1338  * Returns true if there's any budget left (e.g. the clean is finished)
1339  **/
1340 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1341 {
1342         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1343         u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1344         u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1345         const int current_node = numa_node_id();
1346         struct i40e_vsi *vsi = rx_ring->vsi;
1347         u16 i = rx_ring->next_to_clean;
1348         union i40e_rx_desc *rx_desc;
1349         u32 rx_error, rx_status;
1350         u8 rx_ptype;
1351         u64 qword;
1352
1353         if (budget <= 0)
1354                 return 0;
1355
1356         rx_desc = I40E_RX_DESC(rx_ring, i);
1357         qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1358         rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1359                     I40E_RXD_QW1_STATUS_SHIFT;
1360
1361         while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1362                 union i40e_rx_desc *next_rxd;
1363                 struct i40e_rx_buffer *rx_bi;
1364                 struct sk_buff *skb;
1365                 u16 vlan_tag;
1366                 if (i40e_rx_is_programming_status(qword)) {
1367                         i40e_clean_programming_status(rx_ring, rx_desc);
1368                         I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1369                         goto next_desc;
1370                 }
1371                 rx_bi = &rx_ring->rx_bi[i];
1372                 skb = rx_bi->skb;
1373                 prefetch(skb->data);
1374
1375                 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1376                                 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1377                 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1378                                 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1379                 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1380                          I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1381
1382                 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1383                            I40E_RXD_QW1_ERROR_SHIFT;
1384                 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1385                 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1386
1387                 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1388                            I40E_RXD_QW1_PTYPE_SHIFT;
1389                 rx_bi->skb = NULL;
1390
1391                 /* This memory barrier is needed to keep us from reading
1392                  * any other fields out of the rx_desc until we know the
1393                  * STATUS_DD bit is set
1394                  */
1395                 rmb();
1396
1397                 /* Get the header and possibly the whole packet
1398                  * If this is an skb from previous receive dma will be 0
1399                  */
1400                 if (rx_bi->dma) {
1401                         u16 len;
1402
1403                         if (rx_hbo)
1404                                 len = I40E_RX_HDR_SIZE;
1405                         else if (rx_sph)
1406                                 len = rx_header_len;
1407                         else if (rx_packet_len)
1408                                 len = rx_packet_len;   /* 1buf/no split found */
1409                         else
1410                                 len = rx_header_len;   /* split always mode */
1411
1412                         skb_put(skb, len);
1413                         dma_unmap_single(rx_ring->dev,
1414                                          rx_bi->dma,
1415                                          rx_ring->rx_buf_len,
1416                                          DMA_FROM_DEVICE);
1417                         rx_bi->dma = 0;
1418                 }
1419
1420                 /* Get the rest of the data if this was a header split */
1421                 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1422
1423                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1424                                            rx_bi->page,
1425                                            rx_bi->page_offset,
1426                                            rx_packet_len);
1427
1428                         skb->len += rx_packet_len;
1429                         skb->data_len += rx_packet_len;
1430                         skb->truesize += rx_packet_len;
1431
1432                         if ((page_count(rx_bi->page) == 1) &&
1433                             (page_to_nid(rx_bi->page) == current_node))
1434                                 get_page(rx_bi->page);
1435                         else
1436                                 rx_bi->page = NULL;
1437
1438                         dma_unmap_page(rx_ring->dev,
1439                                        rx_bi->page_dma,
1440                                        PAGE_SIZE / 2,
1441                                        DMA_FROM_DEVICE);
1442                         rx_bi->page_dma = 0;
1443                 }
1444                 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1445
1446                 if (unlikely(
1447                     !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1448                         struct i40e_rx_buffer *next_buffer;
1449
1450                         next_buffer = &rx_ring->rx_bi[i];
1451
1452                         if (ring_is_ps_enabled(rx_ring)) {
1453                                 rx_bi->skb = next_buffer->skb;
1454                                 rx_bi->dma = next_buffer->dma;
1455                                 next_buffer->skb = skb;
1456                                 next_buffer->dma = 0;
1457                         }
1458                         rx_ring->rx_stats.non_eop_descs++;
1459                         goto next_desc;
1460                 }
1461
1462                 /* ERR_MASK will only have valid bits if EOP set */
1463                 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1464                         dev_kfree_skb_any(skb);
1465                         /* TODO: shouldn't we increment a counter indicating the
1466                          * drop?
1467                          */
1468                         goto next_desc;
1469                 }
1470
1471                 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc),
1472                              i40e_ptype_to_hash(rx_ptype));
1473                 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1474                         i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1475                                            I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1476                                            I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1477                         rx_ring->last_rx_timestamp = jiffies;
1478                 }
1479
1480                 /* probably a little skewed due to removing CRC */
1481                 total_rx_bytes += skb->len;
1482                 total_rx_packets++;
1483
1484                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1485
1486                 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1487
1488                 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1489                          ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1490                          : 0;
1491                 i40e_receive_skb(rx_ring, skb, vlan_tag);
1492
1493                 rx_ring->netdev->last_rx = jiffies;
1494                 budget--;
1495 next_desc:
1496                 rx_desc->wb.qword1.status_error_len = 0;
1497                 if (!budget)
1498                         break;
1499
1500                 cleaned_count++;
1501                 /* return some buffers to hardware, one at a time is too slow */
1502                 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1503                         i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1504                         cleaned_count = 0;
1505                 }
1506
1507                 /* use prefetched values */
1508                 rx_desc = next_rxd;
1509                 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1510                 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1511                             I40E_RXD_QW1_STATUS_SHIFT;
1512         }
1513
1514         rx_ring->next_to_clean = i;
1515         u64_stats_update_begin(&rx_ring->syncp);
1516         rx_ring->stats.packets += total_rx_packets;
1517         rx_ring->stats.bytes += total_rx_bytes;
1518         u64_stats_update_end(&rx_ring->syncp);
1519         rx_ring->q_vector->rx.total_packets += total_rx_packets;
1520         rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1521
1522         if (cleaned_count)
1523                 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1524
1525         return budget > 0;
1526 }
1527
1528 /**
1529  * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1530  * @napi: napi struct with our devices info in it
1531  * @budget: amount of work driver is allowed to do this pass, in packets
1532  *
1533  * This function will clean all queues associated with a q_vector.
1534  *
1535  * Returns the amount of work done
1536  **/
1537 int i40e_napi_poll(struct napi_struct *napi, int budget)
1538 {
1539         struct i40e_q_vector *q_vector =
1540                                container_of(napi, struct i40e_q_vector, napi);
1541         struct i40e_vsi *vsi = q_vector->vsi;
1542         struct i40e_ring *ring;
1543         bool clean_complete = true;
1544         int budget_per_ring;
1545
1546         if (test_bit(__I40E_DOWN, &vsi->state)) {
1547                 napi_complete(napi);
1548                 return 0;
1549         }
1550
1551         /* Since the actual Tx work is minimal, we can give the Tx a larger
1552          * budget and be more aggressive about cleaning up the Tx descriptors.
1553          */
1554         i40e_for_each_ring(ring, q_vector->tx)
1555                 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1556
1557         /* We attempt to distribute budget to each Rx queue fairly, but don't
1558          * allow the budget to go below 1 because that would exit polling early.
1559          */
1560         budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1561
1562         i40e_for_each_ring(ring, q_vector->rx)
1563                 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
1564
1565         /* If work not completed, return budget and polling will return */
1566         if (!clean_complete)
1567                 return budget;
1568
1569         /* Work is done so exit the polling mode and re-enable the interrupt */
1570         napi_complete(napi);
1571         if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1572             ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1573                 i40e_update_dynamic_itr(q_vector);
1574
1575         if (!test_bit(__I40E_DOWN, &vsi->state)) {
1576                 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1577                         i40e_irq_dynamic_enable(vsi,
1578                                         q_vector->v_idx + vsi->base_vector);
1579                 } else {
1580                         struct i40e_hw *hw = &vsi->back->hw;
1581                         /* We re-enable the queue 0 cause, but
1582                          * don't worry about dynamic_enable
1583                          * because we left it on for the other
1584                          * possible interrupts during napi
1585                          */
1586                         u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1587                         qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1588                         wr32(hw, I40E_QINT_RQCTL(0), qval);
1589
1590                         qval = rd32(hw, I40E_QINT_TQCTL(0));
1591                         qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1592                         wr32(hw, I40E_QINT_TQCTL(0), qval);
1593
1594                         i40e_irq_dynamic_enable_icr0(vsi->back);
1595                 }
1596         }
1597
1598         return 0;
1599 }
1600
1601 /**
1602  * i40e_atr - Add a Flow Director ATR filter
1603  * @tx_ring:  ring to add programming descriptor to
1604  * @skb:      send buffer
1605  * @flags:    send flags
1606  * @protocol: wire protocol
1607  **/
1608 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1609                      u32 flags, __be16 protocol)
1610 {
1611         struct i40e_filter_program_desc *fdir_desc;
1612         struct i40e_pf *pf = tx_ring->vsi->back;
1613         union {
1614                 unsigned char *network;
1615                 struct iphdr *ipv4;
1616                 struct ipv6hdr *ipv6;
1617         } hdr;
1618         struct tcphdr *th;
1619         unsigned int hlen;
1620         u32 flex_ptype, dtype_cmd;
1621         u16 i;
1622
1623         /* make sure ATR is enabled */
1624         if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
1625                 return;
1626
1627         /* if sampling is disabled do nothing */
1628         if (!tx_ring->atr_sample_rate)
1629                 return;
1630
1631         /* snag network header to get L4 type and address */
1632         hdr.network = skb_network_header(skb);
1633
1634         /* Currently only IPv4/IPv6 with TCP is supported */
1635         if (protocol == htons(ETH_P_IP)) {
1636                 if (hdr.ipv4->protocol != IPPROTO_TCP)
1637                         return;
1638
1639                 /* access ihl as a u8 to avoid unaligned access on ia64 */
1640                 hlen = (hdr.network[0] & 0x0F) << 2;
1641         } else if (protocol == htons(ETH_P_IPV6)) {
1642                 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1643                         return;
1644
1645                 hlen = sizeof(struct ipv6hdr);
1646         } else {
1647                 return;
1648         }
1649
1650         th = (struct tcphdr *)(hdr.network + hlen);
1651
1652         /* Due to lack of space, no more new filters can be programmed */
1653         if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1654                 return;
1655
1656         tx_ring->atr_count++;
1657
1658         /* sample on all syn/fin/rst packets or once every atr sample rate */
1659         if (!th->fin &&
1660             !th->syn &&
1661             !th->rst &&
1662             (tx_ring->atr_count < tx_ring->atr_sample_rate))
1663                 return;
1664
1665         tx_ring->atr_count = 0;
1666
1667         /* grab the next descriptor */
1668         i = tx_ring->next_to_use;
1669         fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1670
1671         i++;
1672         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1673
1674         flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1675                       I40E_TXD_FLTR_QW0_QINDEX_MASK;
1676         flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1677                       (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1678                        I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1679                       (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1680                        I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1681
1682         flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1683
1684         dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1685
1686         dtype_cmd |= (th->fin || th->rst) ?
1687                      (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1688                       I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1689                      (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1690                       I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1691
1692         dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1693                      I40E_TXD_FLTR_QW1_DEST_SHIFT;
1694
1695         dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1696                      I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1697
1698         dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
1699         dtype_cmd |=
1700                 ((u32)pf->fd_atr_cnt_idx << I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
1701                 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
1702
1703         fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1704         fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1705 }
1706
1707 /**
1708  * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1709  * @skb:     send buffer
1710  * @tx_ring: ring to send buffer on
1711  * @flags:   the tx flags to be set
1712  *
1713  * Checks the skb and set up correspondingly several generic transmit flags
1714  * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1715  *
1716  * Returns error code indicate the frame should be dropped upon error and the
1717  * otherwise  returns 0 to indicate the flags has been set properly.
1718  **/
1719 static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1720                                       struct i40e_ring *tx_ring,
1721                                       u32 *flags)
1722 {
1723         __be16 protocol = skb->protocol;
1724         u32  tx_flags = 0;
1725
1726         /* if we have a HW VLAN tag being added, default to the HW one */
1727         if (vlan_tx_tag_present(skb)) {
1728                 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1729                 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1730         /* else if it is a SW VLAN, check the next protocol and store the tag */
1731         } else if (protocol == htons(ETH_P_8021Q)) {
1732                 struct vlan_hdr *vhdr, _vhdr;
1733                 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1734                 if (!vhdr)
1735                         return -EINVAL;
1736
1737                 protocol = vhdr->h_vlan_encapsulated_proto;
1738                 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1739                 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1740         }
1741
1742         /* Insert 802.1p priority into VLAN header */
1743         if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1744             ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1745              (skb->priority != TC_PRIO_CONTROL))) {
1746                 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1747                 tx_flags |= (skb->priority & 0x7) <<
1748                                 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1749                 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1750                         struct vlan_ethhdr *vhdr;
1751                         int rc;
1752
1753                         rc = skb_cow_head(skb, 0);
1754                         if (rc < 0)
1755                                 return rc;
1756                         vhdr = (struct vlan_ethhdr *)skb->data;
1757                         vhdr->h_vlan_TCI = htons(tx_flags >>
1758                                                  I40E_TX_FLAGS_VLAN_SHIFT);
1759                 } else {
1760                         tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1761                 }
1762         }
1763         *flags = tx_flags;
1764         return 0;
1765 }
1766
1767 /**
1768  * i40e_tso - set up the tso context descriptor
1769  * @tx_ring:  ptr to the ring to send
1770  * @skb:      ptr to the skb we're sending
1771  * @tx_flags: the collected send information
1772  * @protocol: the send protocol
1773  * @hdr_len:  ptr to the size of the packet header
1774  * @cd_tunneling: ptr to context descriptor bits
1775  *
1776  * Returns 0 if no TSO can happen, 1 if tso is going, or error
1777  **/
1778 static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1779                     u32 tx_flags, __be16 protocol, u8 *hdr_len,
1780                     u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1781 {
1782         u32 cd_cmd, cd_tso_len, cd_mss;
1783         struct ipv6hdr *ipv6h;
1784         struct tcphdr *tcph;
1785         struct iphdr *iph;
1786         u32 l4len;
1787         int err;
1788
1789         if (!skb_is_gso(skb))
1790                 return 0;
1791
1792         err = skb_cow_head(skb, 0);
1793         if (err < 0)
1794                 return err;
1795
1796         if (protocol == htons(ETH_P_IP)) {
1797                 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1798                 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1799                 iph->tot_len = 0;
1800                 iph->check = 0;
1801                 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1802                                                  0, IPPROTO_TCP, 0);
1803         } else if (skb_is_gso_v6(skb)) {
1804
1805                 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1806                                            : ipv6_hdr(skb);
1807                 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1808                 ipv6h->payload_len = 0;
1809                 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1810                                                0, IPPROTO_TCP, 0);
1811         }
1812
1813         l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1814         *hdr_len = (skb->encapsulation
1815                     ? (skb_inner_transport_header(skb) - skb->data)
1816                     : skb_transport_offset(skb)) + l4len;
1817
1818         /* find the field values */
1819         cd_cmd = I40E_TX_CTX_DESC_TSO;
1820         cd_tso_len = skb->len - *hdr_len;
1821         cd_mss = skb_shinfo(skb)->gso_size;
1822         *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1823                                 ((u64)cd_tso_len <<
1824                                  I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1825                                 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1826         return 1;
1827 }
1828
1829 /**
1830  * i40e_tsyn - set up the tsyn context descriptor
1831  * @tx_ring:  ptr to the ring to send
1832  * @skb:      ptr to the skb we're sending
1833  * @tx_flags: the collected send information
1834  *
1835  * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1836  **/
1837 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1838                      u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1839 {
1840         struct i40e_pf *pf;
1841
1842         if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1843                 return 0;
1844
1845         /* Tx timestamps cannot be sampled when doing TSO */
1846         if (tx_flags & I40E_TX_FLAGS_TSO)
1847                 return 0;
1848
1849         /* only timestamp the outbound packet if the user has requested it and
1850          * we are not already transmitting a packet to be timestamped
1851          */
1852         pf = i40e_netdev_to_pf(tx_ring->netdev);
1853         if (pf->ptp_tx && !pf->ptp_tx_skb) {
1854                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1855                 pf->ptp_tx_skb = skb_get(skb);
1856         } else {
1857                 return 0;
1858         }
1859
1860         *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1861                                 I40E_TXD_CTX_QW1_CMD_SHIFT;
1862
1863         return 1;
1864 }
1865
1866 /**
1867  * i40e_tx_enable_csum - Enable Tx checksum offloads
1868  * @skb: send buffer
1869  * @tx_flags: Tx flags currently set
1870  * @td_cmd: Tx descriptor command bits to set
1871  * @td_offset: Tx descriptor header offsets to set
1872  * @cd_tunneling: ptr to context desc bits
1873  **/
1874 static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1875                                 u32 *td_cmd, u32 *td_offset,
1876                                 struct i40e_ring *tx_ring,
1877                                 u32 *cd_tunneling)
1878 {
1879         struct ipv6hdr *this_ipv6_hdr;
1880         unsigned int this_tcp_hdrlen;
1881         struct iphdr *this_ip_hdr;
1882         u32 network_hdr_len;
1883         u8 l4_hdr = 0;
1884
1885         if (skb->encapsulation) {
1886                 network_hdr_len = skb_inner_network_header_len(skb);
1887                 this_ip_hdr = inner_ip_hdr(skb);
1888                 this_ipv6_hdr = inner_ipv6_hdr(skb);
1889                 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1890
1891                 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1892
1893                         if (tx_flags & I40E_TX_FLAGS_TSO) {
1894                                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1895                                 ip_hdr(skb)->check = 0;
1896                         } else {
1897                                 *cd_tunneling |=
1898                                          I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1899                         }
1900                 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1901                         if (tx_flags & I40E_TX_FLAGS_TSO) {
1902                                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1903                                 ip_hdr(skb)->check = 0;
1904                         } else {
1905                                 *cd_tunneling |=
1906                                          I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1907                         }
1908                 }
1909
1910                 /* Now set the ctx descriptor fields */
1911                 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1912                                         I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1913                                    I40E_TXD_CTX_UDP_TUNNELING            |
1914                                    ((skb_inner_network_offset(skb) -
1915                                         skb_transport_offset(skb)) >> 1) <<
1916                                    I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1917
1918         } else {
1919                 network_hdr_len = skb_network_header_len(skb);
1920                 this_ip_hdr = ip_hdr(skb);
1921                 this_ipv6_hdr = ipv6_hdr(skb);
1922                 this_tcp_hdrlen = tcp_hdrlen(skb);
1923         }
1924
1925         /* Enable IP checksum offloads */
1926         if (tx_flags & I40E_TX_FLAGS_IPV4) {
1927                 l4_hdr = this_ip_hdr->protocol;
1928                 /* the stack computes the IP header already, the only time we
1929                  * need the hardware to recompute it is in the case of TSO.
1930                  */
1931                 if (tx_flags & I40E_TX_FLAGS_TSO) {
1932                         *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1933                         this_ip_hdr->check = 0;
1934                 } else {
1935                         *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1936                 }
1937                 /* Now set the td_offset for IP header length */
1938                 *td_offset = (network_hdr_len >> 2) <<
1939                               I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1940         } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1941                 l4_hdr = this_ipv6_hdr->nexthdr;
1942                 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1943                 /* Now set the td_offset for IP header length */
1944                 *td_offset = (network_hdr_len >> 2) <<
1945                               I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1946         }
1947         /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1948         *td_offset |= (skb_network_offset(skb) >> 1) <<
1949                        I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1950
1951         /* Enable L4 checksum offloads */
1952         switch (l4_hdr) {
1953         case IPPROTO_TCP:
1954                 /* enable checksum offloads */
1955                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1956                 *td_offset |= (this_tcp_hdrlen >> 2) <<
1957                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1958                 break;
1959         case IPPROTO_SCTP:
1960                 /* enable SCTP checksum offload */
1961                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1962                 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1963                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1964                 break;
1965         case IPPROTO_UDP:
1966                 /* enable UDP checksum offload */
1967                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1968                 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1969                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1970                 break;
1971         default:
1972                 break;
1973         }
1974 }
1975
1976 /**
1977  * i40e_create_tx_ctx Build the Tx context descriptor
1978  * @tx_ring:  ring to create the descriptor on
1979  * @cd_type_cmd_tso_mss: Quad Word 1
1980  * @cd_tunneling: Quad Word 0 - bits 0-31
1981  * @cd_l2tag2: Quad Word 0 - bits 32-63
1982  **/
1983 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1984                                const u64 cd_type_cmd_tso_mss,
1985                                const u32 cd_tunneling, const u32 cd_l2tag2)
1986 {
1987         struct i40e_tx_context_desc *context_desc;
1988         int i = tx_ring->next_to_use;
1989
1990         if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
1991             !cd_tunneling && !cd_l2tag2)
1992                 return;
1993
1994         /* grab the next descriptor */
1995         context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1996
1997         i++;
1998         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1999
2000         /* cpu_to_le32 and assign to struct fields */
2001         context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
2002         context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
2003         context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
2004 }
2005
2006 /**
2007  * i40e_tx_map - Build the Tx descriptor
2008  * @tx_ring:  ring to send buffer on
2009  * @skb:      send buffer
2010  * @first:    first buffer info buffer to use
2011  * @tx_flags: collected send information
2012  * @hdr_len:  size of the packet header
2013  * @td_cmd:   the command field in the descriptor
2014  * @td_offset: offset for checksum or crc
2015  **/
2016 static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2017                         struct i40e_tx_buffer *first, u32 tx_flags,
2018                         const u8 hdr_len, u32 td_cmd, u32 td_offset)
2019 {
2020         unsigned int data_len = skb->data_len;
2021         unsigned int size = skb_headlen(skb);
2022         struct skb_frag_struct *frag;
2023         struct i40e_tx_buffer *tx_bi;
2024         struct i40e_tx_desc *tx_desc;
2025         u16 i = tx_ring->next_to_use;
2026         u32 td_tag = 0;
2027         dma_addr_t dma;
2028         u16 gso_segs;
2029
2030         if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2031                 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2032                 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2033                          I40E_TX_FLAGS_VLAN_SHIFT;
2034         }
2035
2036         if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
2037                 gso_segs = skb_shinfo(skb)->gso_segs;
2038         else
2039                 gso_segs = 1;
2040
2041         /* multiply data chunks by size of headers */
2042         first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
2043         first->gso_segs = gso_segs;
2044         first->skb = skb;
2045         first->tx_flags = tx_flags;
2046
2047         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2048
2049         tx_desc = I40E_TX_DESC(tx_ring, i);
2050         tx_bi = first;
2051
2052         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2053                 if (dma_mapping_error(tx_ring->dev, dma))
2054                         goto dma_error;
2055
2056                 /* record length, and DMA address */
2057                 dma_unmap_len_set(tx_bi, len, size);
2058                 dma_unmap_addr_set(tx_bi, dma, dma);
2059
2060                 tx_desc->buffer_addr = cpu_to_le64(dma);
2061
2062                 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2063                         tx_desc->cmd_type_offset_bsz =
2064                                 build_ctob(td_cmd, td_offset,
2065                                            I40E_MAX_DATA_PER_TXD, td_tag);
2066
2067                         tx_desc++;
2068                         i++;
2069                         if (i == tx_ring->count) {
2070                                 tx_desc = I40E_TX_DESC(tx_ring, 0);
2071                                 i = 0;
2072                         }
2073
2074                         dma += I40E_MAX_DATA_PER_TXD;
2075                         size -= I40E_MAX_DATA_PER_TXD;
2076
2077                         tx_desc->buffer_addr = cpu_to_le64(dma);
2078                 }
2079
2080                 if (likely(!data_len))
2081                         break;
2082
2083                 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2084                                                           size, td_tag);
2085
2086                 tx_desc++;
2087                 i++;
2088                 if (i == tx_ring->count) {
2089                         tx_desc = I40E_TX_DESC(tx_ring, 0);
2090                         i = 0;
2091                 }
2092
2093                 size = skb_frag_size(frag);
2094                 data_len -= size;
2095
2096                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2097                                        DMA_TO_DEVICE);
2098
2099                 tx_bi = &tx_ring->tx_bi[i];
2100         }
2101
2102         /* Place RS bit on last descriptor of any packet that spans across the
2103          * 4th descriptor (WB_STRIDE aka 0x3) in a 64B cacheline.
2104          */
2105 #define WB_STRIDE 0x3
2106         if (((i & WB_STRIDE) != WB_STRIDE) &&
2107             (first <= &tx_ring->tx_bi[i]) &&
2108             (first >= &tx_ring->tx_bi[i & ~WB_STRIDE])) {
2109                 tx_desc->cmd_type_offset_bsz =
2110                         build_ctob(td_cmd, td_offset, size, td_tag) |
2111                         cpu_to_le64((u64)I40E_TX_DESC_CMD_EOP <<
2112                                          I40E_TXD_QW1_CMD_SHIFT);
2113         } else {
2114                 tx_desc->cmd_type_offset_bsz =
2115                         build_ctob(td_cmd, td_offset, size, td_tag) |
2116                         cpu_to_le64((u64)I40E_TXD_CMD <<
2117                                          I40E_TXD_QW1_CMD_SHIFT);
2118         }
2119
2120         netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2121                                                  tx_ring->queue_index),
2122                              first->bytecount);
2123
2124         /* set the timestamp */
2125         first->time_stamp = jiffies;
2126
2127         /* Force memory writes to complete before letting h/w
2128          * know there are new descriptors to fetch.  (Only
2129          * applicable for weak-ordered memory model archs,
2130          * such as IA-64).
2131          */
2132         wmb();
2133
2134         /* set next_to_watch value indicating a packet is present */
2135         first->next_to_watch = tx_desc;
2136
2137         i++;
2138         if (i == tx_ring->count)
2139                 i = 0;
2140
2141         tx_ring->next_to_use = i;
2142
2143         /* notify HW of packet */
2144         writel(i, tx_ring->tail);
2145
2146         return;
2147
2148 dma_error:
2149         dev_info(tx_ring->dev, "TX DMA map failed\n");
2150
2151         /* clear dma mappings for failed tx_bi map */
2152         for (;;) {
2153                 tx_bi = &tx_ring->tx_bi[i];
2154                 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2155                 if (tx_bi == first)
2156                         break;
2157                 if (i == 0)
2158                         i = tx_ring->count;
2159                 i--;
2160         }
2161
2162         tx_ring->next_to_use = i;
2163 }
2164
2165 /**
2166  * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2167  * @tx_ring: the ring to be checked
2168  * @size:    the size buffer we want to assure is available
2169  *
2170  * Returns -EBUSY if a stop is needed, else 0
2171  **/
2172 static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2173 {
2174         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2175         /* Memory barrier before checking head and tail */
2176         smp_mb();
2177
2178         /* Check again in a case another CPU has just made room available. */
2179         if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2180                 return -EBUSY;
2181
2182         /* A reprieve! - use start_queue because it doesn't call schedule */
2183         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2184         ++tx_ring->tx_stats.restart_queue;
2185         return 0;
2186 }
2187
2188 /**
2189  * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2190  * @tx_ring: the ring to be checked
2191  * @size:    the size buffer we want to assure is available
2192  *
2193  * Returns 0 if stop is not needed
2194  **/
2195 static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2196 {
2197         if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2198                 return 0;
2199         return __i40e_maybe_stop_tx(tx_ring, size);
2200 }
2201
2202 /**
2203  * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2204  * @skb:     send buffer
2205  * @tx_ring: ring to send buffer on
2206  *
2207  * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2208  * there is not enough descriptors available in this ring since we need at least
2209  * one descriptor.
2210  **/
2211 static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2212                                       struct i40e_ring *tx_ring)
2213 {
2214         unsigned int f;
2215         int count = 0;
2216
2217         /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2218          *       + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
2219          *       + 4 desc gap to avoid the cache line where head is,
2220          *       + 1 desc for context descriptor,
2221          * otherwise try next time
2222          */
2223         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2224                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2225
2226         count += TXD_USE_COUNT(skb_headlen(skb));
2227         if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2228                 tx_ring->tx_stats.tx_busy++;
2229                 return 0;
2230         }
2231         return count;
2232 }
2233
2234 /**
2235  * i40e_xmit_frame_ring - Sends buffer on Tx ring
2236  * @skb:     send buffer
2237  * @tx_ring: ring to send buffer on
2238  *
2239  * Returns NETDEV_TX_OK if sent, else an error code
2240  **/
2241 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2242                                         struct i40e_ring *tx_ring)
2243 {
2244         u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2245         u32 cd_tunneling = 0, cd_l2tag2 = 0;
2246         struct i40e_tx_buffer *first;
2247         u32 td_offset = 0;
2248         u32 tx_flags = 0;
2249         __be16 protocol;
2250         u32 td_cmd = 0;
2251         u8 hdr_len = 0;
2252         int tsyn;
2253         int tso;
2254         if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2255                 return NETDEV_TX_BUSY;
2256
2257         /* prepare the xmit flags */
2258         if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2259                 goto out_drop;
2260
2261         /* obtain protocol of skb */
2262         protocol = skb->protocol;
2263
2264         /* record the location of the first descriptor for this packet */
2265         first = &tx_ring->tx_bi[tx_ring->next_to_use];
2266
2267         /* setup IPv4/IPv6 offloads */
2268         if (protocol == htons(ETH_P_IP))
2269                 tx_flags |= I40E_TX_FLAGS_IPV4;
2270         else if (protocol == htons(ETH_P_IPV6))
2271                 tx_flags |= I40E_TX_FLAGS_IPV6;
2272
2273         tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2274                        &cd_type_cmd_tso_mss, &cd_tunneling);
2275
2276         if (tso < 0)
2277                 goto out_drop;
2278         else if (tso)
2279                 tx_flags |= I40E_TX_FLAGS_TSO;
2280
2281         skb_tx_timestamp(skb);
2282
2283         tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2284
2285         if (tsyn)
2286                 tx_flags |= I40E_TX_FLAGS_TSYN;
2287
2288         /* always enable CRC insertion offload */
2289         td_cmd |= I40E_TX_DESC_CMD_ICRC;
2290
2291         /* Always offload the checksum, since it's in the data descriptor */
2292         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2293                 tx_flags |= I40E_TX_FLAGS_CSUM;
2294
2295                 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2296                                     tx_ring, &cd_tunneling);
2297         }
2298
2299         i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2300                            cd_tunneling, cd_l2tag2);
2301
2302         /* Add Flow Director ATR if it's enabled.
2303          *
2304          * NOTE: this must always be directly before the data descriptor.
2305          */
2306         i40e_atr(tx_ring, skb, tx_flags, protocol);
2307
2308         i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2309                     td_cmd, td_offset);
2310
2311         i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2312
2313         return NETDEV_TX_OK;
2314
2315 out_drop:
2316         dev_kfree_skb_any(skb);
2317         return NETDEV_TX_OK;
2318 }
2319
2320 /**
2321  * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2322  * @skb:    send buffer
2323  * @netdev: network interface device structure
2324  *
2325  * Returns NETDEV_TX_OK if sent, else an error code
2326  **/
2327 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2328 {
2329         struct i40e_netdev_priv *np = netdev_priv(netdev);
2330         struct i40e_vsi *vsi = np->vsi;
2331         struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
2332
2333         /* hardware can't handle really short frames, hardware padding works
2334          * beyond this point
2335          */
2336         if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2337                 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2338                         return NETDEV_TX_OK;
2339                 skb->len = I40E_MIN_TX_LEN;
2340                 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2341         }
2342
2343         return i40e_xmit_frame_ring(skb, tx_ring);
2344 }