]> Git Repo - linux.git/blame - drivers/net/ethernet/cavium/thunder/nicvf_main.c
projects / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux.git] / drivers / net / ethernet / cavium / thunder / nicvf_main.c
CommitLineData
4863dea3
SG		 1/*
2 * Copyright (C) 2015 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9#include <linux/module.h>
10#include <linux/interrupt.h>
11#include <linux/pci.h>
12#include <linux/netdevice.h>
aa2e259b 13#include <linux/if_vlan.h>
4863dea3
SG		 14#include <linux/etherdevice.h>
15#include <linux/ethtool.h>
16#include <linux/log2.h>
17#include <linux/prefetch.h>
18#include <linux/irq.h>
83abb7d7 19#include <linux/iommu.h>
05c773f5 20#include <linux/bpf.h>
c56d91ce 21#include <linux/bpf_trace.h>
05c773f5 22#include <linux/filter.h>
4a875509 23#include <linux/net_tstamp.h>
37c3347e 24#include <linux/workqueue.h>
4863dea3
SG		 25
26#include "nic_reg.h"
27#include "nic.h"
28#include "nicvf_queues.h"
29#include "thunder_bgx.h"
4a875509 30#include "../common/cavium_ptp.h"
4863dea3 31
6b9e6547 32#define DRV_NAME "nicvf"
4863dea3
SG		 33#define DRV_VERSION "1.0"
34
35/* Supported devices */
36static const struct pci_device_id nicvf_id_table[] = {
37 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
38 PCI_DEVICE_ID_THUNDER_NIC_VF,
f7ff0ae8
SG		 39 PCI_VENDOR_ID_CAVIUM,
40 PCI_SUBSYS_DEVID_88XX_NIC_VF) },
4863dea3
SG		 41 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
42 PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF,
f7ff0ae8
SG		 43 PCI_VENDOR_ID_CAVIUM,
44 PCI_SUBSYS_DEVID_88XX_PASS1_NIC_VF) },
45 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
46 PCI_DEVICE_ID_THUNDER_NIC_VF,
47 PCI_VENDOR_ID_CAVIUM,
48 PCI_SUBSYS_DEVID_81XX_NIC_VF) },
49 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
50 PCI_DEVICE_ID_THUNDER_NIC_VF,
51 PCI_VENDOR_ID_CAVIUM,
52 PCI_SUBSYS_DEVID_83XX_NIC_VF) },
4863dea3
SG		 53 { 0, } /* end of table */
54};
55
56MODULE_AUTHOR("Sunil Goutham");
57MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver");
58MODULE_LICENSE("GPL v2");
59MODULE_VERSION(DRV_VERSION);
60MODULE_DEVICE_TABLE(pci, nicvf_id_table);
61
62static int debug = 0x00;
63module_param(debug, int, 0644);
64MODULE_PARM_DESC(debug, "Debug message level bitmap");
65
66static int cpi_alg = CPI_ALG_NONE;
d3757ba4 67module_param(cpi_alg, int, 0444);
4863dea3
SG		 68MODULE_PARM_DESC(cpi_alg,
69 "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)");
70
37c3347e
VL		 71/* workqueue for handling kernel ndo_set_rx_mode() calls */
72static struct workqueue_struct *nicvf_rx_mode_wq;
73
92dc8769
SG		 74static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx)
75{
76 if (nic->sqs_mode)
77 return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS);
78 else
79 return qidx;
80}
81
4863dea3
SG		 82/* The Cavium ThunderX network controller can *only* be found in SoCs
83 * containing the ThunderX ARM64 CPU implementation. All accesses to the device
84 * registers on this platform are implicitly strongly ordered with respect
85 * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
86 * with no memory barriers in this driver. The readq()/writeq() functions add
87 * explicit ordering operation which in this case are redundant, and only
88 * add overhead.
89 */
90
91/* Register read/write APIs */
92void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val)
93{
94 writeq_relaxed(val, nic->reg_base + offset);
95}
96
97u64 nicvf_reg_read(struct nicvf *nic, u64 offset)
98{
99 return readq_relaxed(nic->reg_base + offset);
100}
101
102void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
103 u64 qidx, u64 val)
104{
105 void __iomem *addr = nic->reg_base + offset;
106
107 writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT));
108}
109
110u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx)
111{
112 void __iomem *addr = nic->reg_base + offset;
113
114 return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT));
115}
116
117/* VF -> PF mailbox communication */
2cd2a196
AM		 118static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
119{
120 u64 *msg = (u64 *)mbx;
121
122 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
123 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
124}
125
4863dea3
SG		 126int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
127{
128 int timeout = NIC_MBOX_MSG_TIMEOUT;
129 int sleep = 10;
4863dea3
SG		 130
131 nic->pf_acked = false;
132 nic->pf_nacked = false;
133
2cd2a196 134 nicvf_write_to_mbx(nic, mbx);
4863dea3
SG		 135
136 /* Wait for previous message to be acked, timeout 2sec */
137 while (!nic->pf_acked) {
ecae29cb
RB		 138 if (nic->pf_nacked) {
139 netdev_err(nic->netdev,
140 "PF NACK to mbox msg 0x%02x from VF%d\n",
141 (mbx->msg.msg & 0xFF), nic->vf_id);
4863dea3 142 return -EINVAL;
ecae29cb 143 }
4863dea3
SG		 144 msleep(sleep);
145 if (nic->pf_acked)
146 break;
147 timeout -= sleep;
148 if (!timeout) {
149 netdev_err(nic->netdev,
ecae29cb 150 "PF didn't ACK to mbox msg 0x%02x from VF%d\n",
4863dea3
SG		 151 (mbx->msg.msg & 0xFF), nic->vf_id);
152 return -EBUSY;
153 }
154 }
155 return 0;
156}
157
158/* Checks if VF is able to comminicate with PF
159* and also gets the VNIC number this VF is associated to.
160*/
161static int nicvf_check_pf_ready(struct nicvf *nic)
162{
2cd2a196
AM		 163 union nic_mbx mbx = {};
164
165 mbx.msg.msg = NIC_MBOX_MSG_READY;
6051cba7
SG		 166 if (nicvf_send_msg_to_pf(nic, &mbx)) {
167 netdev_err(nic->netdev,
168 "PF didn't respond to READY msg\n");
169 return 0;
4863dea3 170 }
6051cba7 171
4863dea3
SG		 172 return 1;
173}
174
fd7ec062
AM		 175static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
176{
177 if (bgx->rx)
178 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
179 else
180 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
181}
182
4863dea3
SG		 183static void nicvf_handle_mbx_intr(struct nicvf *nic)
184{
185 union nic_mbx mbx = {};
186 u64 *mbx_data;
187 u64 mbx_addr;
188 int i;
189
190 mbx_addr = NIC_VF_PF_MAILBOX_0_1;
191 mbx_data = (u64 *)&mbx;
192
193 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
194 *mbx_data = nicvf_reg_read(nic, mbx_addr);
195 mbx_data++;
196 mbx_addr += sizeof(u64);
197 }
198
199 netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg);
200 switch (mbx.msg.msg) {
201 case NIC_MBOX_MSG_READY:
6051cba7 202 nic->pf_acked = true;
4863dea3
SG		 203 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
204 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
205 nic->node = mbx.nic_cfg.node_id;
bd049a90
PF		 206 if (!nic->set_mac_pending)
207 ether_addr_copy(nic->netdev->dev_addr,
208 mbx.nic_cfg.mac_addr);
92dc8769 209 nic->sqs_mode = mbx.nic_cfg.sqs_mode;
d77a2384 210 nic->loopback_supported = mbx.nic_cfg.loopback_supported;
4863dea3
SG		 211 nic->link_up = false;
212 nic->duplex = 0;
213 nic->speed = 0;
214 break;
215 case NIC_MBOX_MSG_ACK:
216 nic->pf_acked = true;
217 break;
218 case NIC_MBOX_MSG_NACK:
219 nic->pf_nacked = true;
220 break;
221 case NIC_MBOX_MSG_RSS_SIZE:
222 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
223 nic->pf_acked = true;
224 break;
225 case NIC_MBOX_MSG_BGX_STATS:
226 nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
227 nic->pf_acked = true;
4863dea3
SG		 228 break;
229 case NIC_MBOX_MSG_BGX_LINK_CHANGE:
230 nic->pf_acked = true;
231 nic->link_up = mbx.link_status.link_up;
232 nic->duplex = mbx.link_status.duplex;
233 nic->speed = mbx.link_status.speed;
1cc70259 234 nic->mac_type = mbx.link_status.mac_type;
4863dea3 235 if (nic->link_up) {
bf24e136
JP		 236 netdev_info(nic->netdev, "Link is Up %d Mbps %s duplex\n",
237 nic->speed,
4863dea3 238 nic->duplex == DUPLEX_FULL ?
bf24e136 239 "Full" : "Half");
4863dea3 240 netif_carrier_on(nic->netdev);
b49087dd 241 netif_tx_start_all_queues(nic->netdev);
4863dea3 242 } else {
bf24e136 243 netdev_info(nic->netdev, "Link is Down\n");
4863dea3
SG		 244 netif_carrier_off(nic->netdev);
245 netif_tx_stop_all_queues(nic->netdev);
246 }
247 break;
92dc8769
SG		 248 case NIC_MBOX_MSG_ALLOC_SQS:
249 nic->sqs_count = mbx.sqs_alloc.qs_count;
250 nic->pf_acked = true;
251 break;
252 case NIC_MBOX_MSG_SNICVF_PTR:
253 /* Primary VF: make note of secondary VF's pointer
254 * to be used while packet transmission.
255 */
256 nic->snicvf[mbx.nicvf.sqs_id] =
257 (struct nicvf *)mbx.nicvf.nicvf;
258 nic->pf_acked = true;
259 break;
260 case NIC_MBOX_MSG_PNICVF_PTR:
261 /* Secondary VF/Qset: make note of primary VF's pointer
262 * to be used while packet reception, to handover packet
263 * to primary VF's netdev.
264 */
265 nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf;
266 nic->pf_acked = true;
267 break;
430da208
SG		 268 case NIC_MBOX_MSG_PFC:
269 nic->pfc.autoneg = mbx.pfc.autoneg;
270 nic->pfc.fc_rx = mbx.pfc.fc_rx;
271 nic->pfc.fc_tx = mbx.pfc.fc_tx;
272 nic->pf_acked = true;
273 break;
4863dea3
SG		 274 default:
275 netdev_err(nic->netdev,
276 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
277 break;
278 }
279 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
280}
281
282static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev)
283{
284 union nic_mbx mbx = {};
4863dea3
SG		 285
286 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
287 mbx.mac.vf_id = nic->vf_id;
e610cb32 288 ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr);
4863dea3
SG		 289
290 return nicvf_send_msg_to_pf(nic, &mbx);
291}
292
fd7ec062 293static void nicvf_config_cpi(struct nicvf *nic)
4863dea3
SG		 294{
295 union nic_mbx mbx = {};
296
297 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
298 mbx.cpi_cfg.vf_id = nic->vf_id;
299 mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
300 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
301
302 nicvf_send_msg_to_pf(nic, &mbx);
303}
304
fd7ec062 305static void nicvf_get_rss_size(struct nicvf *nic)
4863dea3
SG		 306{
307 union nic_mbx mbx = {};
308
309 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
310 mbx.rss_size.vf_id = nic->vf_id;
311 nicvf_send_msg_to_pf(nic, &mbx);
312}
313
314void nicvf_config_rss(struct nicvf *nic)
315{
316 union nic_mbx mbx = {};
317 struct nicvf_rss_info *rss = &nic->rss_info;
318 int ind_tbl_len = rss->rss_size;
319 int i, nextq = 0;
320
321 mbx.rss_cfg.vf_id = nic->vf_id;
322 mbx.rss_cfg.hash_bits = rss->hash_bits;
323 while (ind_tbl_len) {
324 mbx.rss_cfg.tbl_offset = nextq;
325 mbx.rss_cfg.tbl_len = min(ind_tbl_len,
326 RSS_IND_TBL_LEN_PER_MBX_MSG);
327 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
328 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
329
330 for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
331 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
332
333 nicvf_send_msg_to_pf(nic, &mbx);
334
335 ind_tbl_len -= mbx.rss_cfg.tbl_len;
336 }
337}
338
339void nicvf_set_rss_key(struct nicvf *nic)
340{
341 struct nicvf_rss_info *rss = &nic->rss_info;
342 u64 key_addr = NIC_VNIC_RSS_KEY_0_4;
343 int idx;
344
345 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
346 nicvf_reg_write(nic, key_addr, rss->key[idx]);
347 key_addr += sizeof(u64);
348 }
349}
350
351static int nicvf_rss_init(struct nicvf *nic)
352{
353 struct nicvf_rss_info *rss = &nic->rss_info;
354 int idx;
355
356 nicvf_get_rss_size(nic);
357
38bb5d4f 358 if (cpi_alg != CPI_ALG_NONE) {
4863dea3
SG		 359 rss->enable = false;
360 rss->hash_bits = 0;
361 return 0;
362 }
363
364 rss->enable = true;
365
0052c92f 366 netdev_rss_key_fill(rss->key, RSS_HASH_KEY_SIZE * sizeof(u64));
4863dea3
SG		 367 nicvf_set_rss_key(nic);
368
369 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
370 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
371
372 rss->hash_bits = ilog2(rounddown_pow_of_two(rss->rss_size));
373
374 for (idx = 0; idx < rss->rss_size; idx++)
375 rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx,
92dc8769 376 nic->rx_queues);
4863dea3
SG		 377 nicvf_config_rss(nic);
378 return 1;
379}
380
92dc8769
SG		 381/* Request PF to allocate additional Qsets */
382static void nicvf_request_sqs(struct nicvf *nic)
383{
384 union nic_mbx mbx = {};
385 int sqs;
386 int sqs_count = nic->sqs_count;
387 int rx_queues = 0, tx_queues = 0;
388
389 /* Only primary VF should request */
390 if (nic->sqs_mode || !nic->sqs_count)
391 return;
392
393 mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
394 mbx.sqs_alloc.vf_id = nic->vf_id;
395 mbx.sqs_alloc.qs_count = nic->sqs_count;
396 if (nicvf_send_msg_to_pf(nic, &mbx)) {
397 /* No response from PF */
398 nic->sqs_count = 0;
399 return;
400 }
401
402 /* Return if no Secondary Qsets available */
403 if (!nic->sqs_count)
404 return;
405
406 if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
407 rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
05c773f5
SG		 408
409 tx_queues = nic->tx_queues + nic->xdp_tx_queues;
410 if (tx_queues > MAX_SND_QUEUES_PER_QS)
411 tx_queues = tx_queues - MAX_SND_QUEUES_PER_QS;
92dc8769
SG		 412
413 /* Set no of Rx/Tx queues in each of the SQsets */
414 for (sqs = 0; sqs < nic->sqs_count; sqs++) {
415 mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
416 mbx.nicvf.vf_id = nic->vf_id;
417 mbx.nicvf.sqs_id = sqs;
418 nicvf_send_msg_to_pf(nic, &mbx);
419
420 nic->snicvf[sqs]->sqs_id = sqs;
421 if (rx_queues > MAX_RCV_QUEUES_PER_QS) {
422 nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS;
423 rx_queues -= MAX_RCV_QUEUES_PER_QS;
424 } else {
425 nic->snicvf[sqs]->qs->rq_cnt = rx_queues;
426 rx_queues = 0;
427 }
428
429 if (tx_queues > MAX_SND_QUEUES_PER_QS) {
430 nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS;
431 tx_queues -= MAX_SND_QUEUES_PER_QS;
432 } else {
433 nic->snicvf[sqs]->qs->sq_cnt = tx_queues;
434 tx_queues = 0;
435 }
436
437 nic->snicvf[sqs]->qs->cq_cnt =
438 max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt);
439
440 /* Initialize secondary Qset's queues and its interrupts */
441 nicvf_open(nic->snicvf[sqs]->netdev);
442 }
443
444 /* Update stack with actual Rx/Tx queue count allocated */
445 if (sqs_count != nic->sqs_count)
446 nicvf_set_real_num_queues(nic->netdev,
447 nic->tx_queues, nic->rx_queues);
448}
449
450/* Send this Qset's nicvf pointer to PF.
451 * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs
452 * so that packets received by these Qsets can use primary VF's netdev
453 */
454static void nicvf_send_vf_struct(struct nicvf *nic)
455{
456 union nic_mbx mbx = {};
457
458 mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR;
459 mbx.nicvf.sqs_mode = nic->sqs_mode;
460 mbx.nicvf.nicvf = (u64)nic;
461 nicvf_send_msg_to_pf(nic, &mbx);
462}
463
464static void nicvf_get_primary_vf_struct(struct nicvf *nic)
465{
466 union nic_mbx mbx = {};
467
468 mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
469 nicvf_send_msg_to_pf(nic, &mbx);
470}
471
4863dea3
SG		 472int nicvf_set_real_num_queues(struct net_device *netdev,
473 int tx_queues, int rx_queues)
474{
475 int err = 0;
476
477 err = netif_set_real_num_tx_queues(netdev, tx_queues);
478 if (err) {
479 netdev_err(netdev,
480 "Failed to set no of Tx queues: %d\n", tx_queues);
481 return err;
482 }
483
484 err = netif_set_real_num_rx_queues(netdev, rx_queues);
485 if (err)
486 netdev_err(netdev,
487 "Failed to set no of Rx queues: %d\n", rx_queues);
488 return err;
489}
490
491static int nicvf_init_resources(struct nicvf *nic)
492{
493 int err;
4863dea3
SG		 494
495 /* Enable Qset */
496 nicvf_qset_config(nic, true);
497
498 /* Initialize queues and HW for data transfer */
499 err = nicvf_config_data_transfer(nic, true);
500 if (err) {
501 netdev_err(nic->netdev,
502 "Failed to alloc/config VF's QSet resources\n");
503 return err;
504 }
505
4863dea3
SG		 506 return 0;
507}
508
16f2bccd 509static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog,
e3d06ff9 510 struct cqe_rx_t *cqe_rx, struct snd_queue *sq,
27e95e36 511 struct rcv_queue *rq, struct sk_buff **skb)
05c773f5
SG		 512{
513 struct xdp_buff xdp;
c56d91ce 514 struct page *page;
05c773f5 515 u32 action;
e6dbe939 516 u16 len, offset = 0;
05c773f5 517 u64 dma_addr, cpu_addr;
e3d06ff9 518 void *orig_data;
05c773f5
SG		 519
520 /* Retrieve packet buffer's DMA address and length */
521 len = *((u16 *)((void *)cqe_rx + (3 * sizeof(u64))));
522 dma_addr = *((u64 *)((void *)cqe_rx + (7 * sizeof(u64))));
523
524 cpu_addr = nicvf_iova_to_phys(nic, dma_addr);
525 if (!cpu_addr)
526 return false;
e3d06ff9
SG		 527 cpu_addr = (u64)phys_to_virt(cpu_addr);
528 page = virt_to_page((void *)cpu_addr);
05c773f5 529
e6dbe939 530 xdp.data_hard_start = page_address(page);
e3d06ff9 531 xdp.data = (void *)cpu_addr;
de8f3a83 532 xdp_set_data_meta_invalid(&xdp);
05c773f5 533 xdp.data_end = xdp.data + len;
27e95e36 534 xdp.rxq = &rq->xdp_rxq;
e3d06ff9 535 orig_data = xdp.data;
05c773f5
SG		 536
537 rcu_read_lock();
538 action = bpf_prog_run_xdp(prog, &xdp);
539 rcu_read_unlock();
540
a48ce00f 541 len = xdp.data_end - xdp.data;
e3d06ff9
SG		 542 /* Check if XDP program has changed headers */
543 if (orig_data != xdp.data) {
e3d06ff9
SG		 544 offset = orig_data - xdp.data;
545 dma_addr -= offset;
546 }
547
05c773f5
SG		 548 switch (action) {
549 case XDP_PASS:
e6dbe939
JDB		 550 /* Check if it's a recycled page, if not
551 * unmap the DMA mapping.
552 *
553 * Recycled page holds an extra reference.
554 */
555 if (page_ref_count(page) == 1) {
556 dma_addr &= PAGE_MASK;
557 dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
558 RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
559 DMA_FROM_DEVICE,
560 DMA_ATTR_SKIP_CPU_SYNC);
561 }
e3d06ff9
SG		 562
563 /* Build SKB and pass on packet to network stack */
564 *skb = build_skb(xdp.data,
565 RCV_FRAG_LEN - cqe_rx->align_pad + offset);
566 if (!*skb)
567 put_page(page);
568 else
569 skb_put(*skb, len);
05c773f5 570 return false;
16f2bccd
SG		 571 case XDP_TX:
572 nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len);
573 return true;
05c773f5
SG		 574 default:
575 bpf_warn_invalid_xdp_action(action);
1a2ace56 576 /* fall through */
c56d91ce
SG		 577 case XDP_ABORTED:
578 trace_xdp_exception(nic->netdev, prog, action);
1a2ace56 579 /* fall through */
c56d91ce 580 case XDP_DROP:
e6dbe939
JDB		 581 /* Check if it's a recycled page, if not
582 * unmap the DMA mapping.
583 *
584 * Recycled page holds an extra reference.
585 */
586 if (page_ref_count(page) == 1) {
587 dma_addr &= PAGE_MASK;
588 dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
589 RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
590 DMA_FROM_DEVICE,
591 DMA_ATTR_SKIP_CPU_SYNC);
592 }
c56d91ce
SG		 593 put_page(page);
594 return true;
05c773f5
SG		 595 }
596 return false;
597}
598
4a875509
SG		 599static void nicvf_snd_ptp_handler(struct net_device *netdev,
600 struct cqe_send_t *cqe_tx)
601{
602 struct nicvf *nic = netdev_priv(netdev);
603 struct skb_shared_hwtstamps ts;
604 u64 ns;
605
606 nic = nic->pnicvf;
607
608 /* Sync for 'ptp_skb' */
609 smp_rmb();
610
611 /* New timestamp request can be queued now */
612 atomic_set(&nic->tx_ptp_skbs, 0);
613
614 /* Check for timestamp requested skb */
615 if (!nic->ptp_skb)
616 return;
617
618 /* Check if timestamping is timedout, which is set to 10us */
619 if (cqe_tx->send_status == CQ_TX_ERROP_TSTMP_TIMEOUT ||
620 cqe_tx->send_status == CQ_TX_ERROP_TSTMP_CONFLICT)
621 goto no_tstamp;
622
623 /* Get the timestamp */
624 memset(&ts, 0, sizeof(ts));
625 ns = cavium_ptp_tstamp2time(nic->ptp_clock, cqe_tx->ptp_timestamp);
626 ts.hwtstamp = ns_to_ktime(ns);
627 skb_tstamp_tx(nic->ptp_skb, &ts);
628
629no_tstamp:
630 /* Free the original skb */
631 dev_kfree_skb_any(nic->ptp_skb);
632 nic->ptp_skb = NULL;
633 /* Sync 'ptp_skb' */
634 smp_wmb();
635}
636
4863dea3 637static void nicvf_snd_pkt_handler(struct net_device *netdev,
c43548d2 638 struct cqe_send_t *cqe_tx,
0dada88b 639 int budget, int *subdesc_cnt,
2c204c2b 640 unsigned int *tx_pkts, unsigned int *tx_bytes)
4863dea3
SG		 641{
642 struct sk_buff *skb = NULL;
16f2bccd 643 struct page *page;
4863dea3
SG		 644 struct nicvf *nic = netdev_priv(netdev);
645 struct snd_queue *sq;
646 struct sq_hdr_subdesc *hdr;
7ceb8a13 647 struct sq_hdr_subdesc *tso_sqe;
4863dea3
SG		 648
649 sq = &nic->qs->sq[cqe_tx->sq_idx];
650
651 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
652 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
653 return;
654
0dada88b
SG		 655 /* Check for errors */
656 if (cqe_tx->send_status)
657 nicvf_check_cqe_tx_errs(nic->pnicvf, cqe_tx);
4863dea3 658
16f2bccd
SG		 659 /* Is this a XDP designated Tx queue */
660 if (sq->is_xdp) {
661 page = (struct page *)sq->xdp_page[cqe_tx->sqe_ptr];
662 /* Check if it's recycled page or else unmap DMA mapping */
663 if (page && (page_ref_count(page) == 1))
664 nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
665 hdr->subdesc_cnt);
666
667 /* Release page reference for recycling */
668 if (page)
669 put_page(page);
670 sq->xdp_page[cqe_tx->sqe_ptr] = (u64)NULL;
671 *subdesc_cnt += hdr->subdesc_cnt + 1;
672 return;
673 }
674
4863dea3 675 skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
4863dea3 676 if (skb) {
7ceb8a13
SG		 677 /* Check for dummy descriptor used for HW TSO offload on 88xx */
678 if (hdr->dont_send) {
679 /* Get actual TSO descriptors and free them */
680 tso_sqe =
681 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
83abb7d7
SG		 682 nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2,
683 tso_sqe->subdesc_cnt);
0dada88b 684 *subdesc_cnt += tso_sqe->subdesc_cnt + 1;
83abb7d7
SG		 685 } else {
686 nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
687 hdr->subdesc_cnt);
7ceb8a13 688 }
0dada88b 689 *subdesc_cnt += hdr->subdesc_cnt + 1;
4863dea3 690 prefetch(skb);
2c204c2b
SG		 691 (*tx_pkts)++;
692 *tx_bytes += skb->len;
4a875509
SG		 693 /* If timestamp is requested for this skb, don't free it */
694 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS &&
695 !nic->pnicvf->ptp_skb)
696 nic->pnicvf->ptp_skb = skb;
697 else
698 napi_consume_skb(skb, budget);
143ceb0b 699 sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
40fb5f8a 700 } else {
7ceb8a13
SG		 701 /* In case of SW TSO on 88xx, only last segment will have
702 * a SKB attached, so just free SQEs here.
40fb5f8a
SG		 703 */
704 if (!nic->hw_tso)
0dada88b 705 *subdesc_cnt += hdr->subdesc_cnt + 1;
4863dea3
SG		 706 }
707}
708
38bb5d4f
SG		 709static inline void nicvf_set_rxhash(struct net_device *netdev,
710 struct cqe_rx_t *cqe_rx,
711 struct sk_buff *skb)
712{
713 u8 hash_type;
714 u32 hash;
715
716 if (!(netdev->features & NETIF_F_RXHASH))
717 return;
718
719 switch (cqe_rx->rss_alg) {
720 case RSS_ALG_TCP_IP:
721 case RSS_ALG_UDP_IP:
722 hash_type = PKT_HASH_TYPE_L4;
723 hash = cqe_rx->rss_tag;
724 break;
725 case RSS_ALG_IP:
726 hash_type = PKT_HASH_TYPE_L3;
727 hash = cqe_rx->rss_tag;
728 break;
729 default:
730 hash_type = PKT_HASH_TYPE_NONE;
731 hash = 0;
732 }
733
734 skb_set_hash(skb, hash, hash_type);
735}
736
4a875509
SG		 737static inline void nicvf_set_rxtstamp(struct nicvf *nic, struct sk_buff *skb)
738{
739 u64 ns;
740
741 if (!nic->ptp_clock || !nic->hw_rx_tstamp)
742 return;
743
744 /* The first 8 bytes is the timestamp */
745 ns = cavium_ptp_tstamp2time(nic->ptp_clock,
746 be64_to_cpu(*(__be64 *)skb->data));
747 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
748
749 __skb_pull(skb, 8);
750}
751
4863dea3
SG		 752static void nicvf_rcv_pkt_handler(struct net_device *netdev,
753 struct napi_struct *napi,
27e95e36
JDB		 754 struct cqe_rx_t *cqe_rx,
755 struct snd_queue *sq, struct rcv_queue *rq)
4863dea3 756{
e3d06ff9 757 struct sk_buff *skb = NULL;
4863dea3 758 struct nicvf *nic = netdev_priv(netdev);
83abb7d7 759 struct nicvf *snic = nic;
4863dea3 760 int err = 0;
92dc8769
SG		 761 int rq_idx;
762
763 rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx);
764
765 if (nic->sqs_mode) {
766 /* Use primary VF's 'nicvf' struct */
767 nic = nic->pnicvf;
768 netdev = nic->netdev;
769 }
4863dea3
SG		 770
771 /* Check for errors */
0dada88b
SG		 772 if (cqe_rx->err_level || cqe_rx->err_opcode) {
773 err = nicvf_check_cqe_rx_errs(nic, cqe_rx);
774 if (err && !cqe_rx->rb_cnt)
775 return;
776 }
4863dea3 777
05c773f5 778 /* For XDP, ignore pkts spanning multiple pages */
e3d06ff9
SG		 779 if (nic->xdp_prog && (cqe_rx->rb_cnt == 1)) {
780 /* Packet consumed by XDP */
27e95e36 781 if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, rq, &skb))
05c773f5 782 return;
e3d06ff9
SG		 783 } else {
784 skb = nicvf_get_rcv_skb(snic, cqe_rx,
785 nic->xdp_prog ? true : false);
786 }
05c773f5 787
e3d06ff9 788 if (!skb)
4863dea3 789 return;
4863dea3
SG		 790
791 if (netif_msg_pktdata(nic)) {
bf24e136 792 netdev_info(nic->netdev, "skb 0x%p, len=%d\n", skb, skb->len);
4863dea3
SG		 793 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
794 skb->data, skb->len, true);
795 }
796
a2dc5ded
SG		 797 /* If error packet, drop it here */
798 if (err) {
799 dev_kfree_skb_any(skb);
800 return;
801 }
802
4a875509 803 nicvf_set_rxtstamp(nic, skb);
38bb5d4f
SG		 804 nicvf_set_rxhash(netdev, cqe_rx, skb);
805
92dc8769 806 skb_record_rx_queue(skb, rq_idx);
4863dea3
SG		 807 if (netdev->hw_features & NETIF_F_RXCSUM) {
808 /* HW by default verifies TCP/UDP/SCTP checksums */
809 skb->ip_summed = CHECKSUM_UNNECESSARY;
810 } else {
811 skb_checksum_none_assert(skb);
812 }
813
814 skb->protocol = eth_type_trans(skb, netdev);
815
aa2e259b
SG		 816 /* Check for stripped VLAN */
817 if (cqe_rx->vlan_found && cqe_rx->vlan_stripped)
818 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
819 ntohs((__force __be16)cqe_rx->vlan_tci));
820
4863dea3
SG		 821 if (napi && (netdev->features & NETIF_F_GRO))
822 napi_gro_receive(napi, skb);
823 else
824 netif_receive_skb(skb);
825}
826
827static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
828 struct napi_struct *napi, int budget)
829{
74840b83 830 int processed_cqe, work_done = 0, tx_done = 0;
4863dea3 831 int cqe_count, cqe_head;
0dada88b 832 int subdesc_cnt = 0;
4863dea3
SG		 833 struct nicvf *nic = netdev_priv(netdev);
834 struct queue_set *qs = nic->qs;
835 struct cmp_queue *cq = &qs->cq[cq_idx];
836 struct cqe_rx_t *cq_desc;
74840b83 837 struct netdev_queue *txq;
16f2bccd 838 struct snd_queue *sq = &qs->sq[cq_idx];
27e95e36 839 struct rcv_queue *rq = &qs->rq[cq_idx];
16f2bccd 840 unsigned int tx_pkts = 0, tx_bytes = 0, txq_idx;
4863dea3
SG		 841
842 spin_lock_bh(&cq->lock);
843loop:
844 processed_cqe = 0;
845 /* Get no of valid CQ entries to process */
846 cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
847 cqe_count &= CQ_CQE_COUNT;
848 if (!cqe_count)
849 goto done;
850
851 /* Get head of the valid CQ entries */
852 cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
853 cqe_head &= 0xFFFF;
854
4863dea3
SG		 855 while (processed_cqe < cqe_count) {
856 /* Get the CQ descriptor */
857 cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
858 cqe_head++;
859 cqe_head &= (cq->dmem.q_len - 1);
860 /* Initiate prefetch for next descriptor */
861 prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head));
862
863 if ((work_done >= budget) && napi &&
864 (cq_desc->cqe_type != CQE_TYPE_SEND)) {
865 break;
866 }
867
4863dea3
SG		 868 switch (cq_desc->cqe_type) {
869 case CQE_TYPE_RX:
27e95e36 870 nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq, rq);
4863dea3
SG		 871 work_done++;
872 break;
873 case CQE_TYPE_SEND:
0dada88b
SG		 874 nicvf_snd_pkt_handler(netdev, (void *)cq_desc,
875 budget, &subdesc_cnt,
876 &tx_pkts, &tx_bytes);
74840b83 877 tx_done++;
4863dea3 878 break;
4a875509
SG		 879 case CQE_TYPE_SEND_PTP:
880 nicvf_snd_ptp_handler(netdev, (void *)cq_desc);
881 break;
4863dea3
SG		 882 case CQE_TYPE_INVALID:
883 case CQE_TYPE_RX_SPLIT:
884 case CQE_TYPE_RX_TCP:
4863dea3
SG		 885 /* Ignore for now */
886 break;
887 }
888 processed_cqe++;
889 }
4863dea3
SG		 890
891 /* Ring doorbell to inform H/W to reuse processed CQEs */
892 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
893 cq_idx, processed_cqe);
894
895 if ((work_done < budget) && napi)
896 goto loop;
897
898done:
0dada88b
SG		 899 /* Update SQ's descriptor free count */
900 if (subdesc_cnt)
901 nicvf_put_sq_desc(sq, subdesc_cnt);
902
16f2bccd
SG		 903 txq_idx = nicvf_netdev_qidx(nic, cq_idx);
904 /* Handle XDP TX queues */
905 if (nic->pnicvf->xdp_prog) {
906 if (txq_idx < nic->pnicvf->xdp_tx_queues) {
907 nicvf_xdp_sq_doorbell(nic, sq, cq_idx);
908 goto out;
909 }
910 nic = nic->pnicvf;
911 txq_idx -= nic->pnicvf->xdp_tx_queues;
912 }
913
0dada88b 914 /* Wakeup TXQ if its stopped earlier due to SQ full */
bd3ad7d3
SG		 915 if (tx_done ||
916 (atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) {
92dc8769 917 netdev = nic->pnicvf->netdev;
16f2bccd 918 txq = netdev_get_tx_queue(netdev, txq_idx);
2c204c2b
SG		 919 if (tx_pkts)
920 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
921
bd3ad7d3
SG		 922 /* To read updated queue and carrier status */
923 smp_mb();
92dc8769 924 if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) {
bd3ad7d3
SG		 925 netif_tx_wake_queue(txq);
926 nic = nic->pnicvf;
964cb69b 927 this_cpu_inc(nic->drv_stats->txq_wake);
bf24e136
JP		 928 netif_warn(nic, tx_err, netdev,
929 "Transmit queue wakeup SQ%d\n", txq_idx);
74840b83
SG		 930 }
931 }
932
16f2bccd 933out:
4863dea3
SG		 934 spin_unlock_bh(&cq->lock);
935 return work_done;
936}
937
938static int nicvf_poll(struct napi_struct *napi, int budget)
939{
940 u64 cq_head;
941 int work_done = 0;
942 struct net_device *netdev = napi->dev;
943 struct nicvf *nic = netdev_priv(netdev);
944 struct nicvf_cq_poll *cq;
4863dea3
SG		 945
946 cq = container_of(napi, struct nicvf_cq_poll, napi);
947 work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget);
948
4863dea3
SG		 949 if (work_done < budget) {
950 /* Slow packet rate, exit polling */
6ad20165 951 napi_complete_done(napi, work_done);
4863dea3
SG		 952 /* Re-enable interrupts */
953 cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD,
954 cq->cq_idx);
955 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
956 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD,
957 cq->cq_idx, cq_head);
958 nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
959 }
960 return work_done;
961}
962
963/* Qset error interrupt handler
964 *
965 * As of now only CQ errors are handled
966 */
fd7ec062 967static void nicvf_handle_qs_err(unsigned long data)
4863dea3
SG		 968{
969 struct nicvf *nic = (struct nicvf *)data;
970 struct queue_set *qs = nic->qs;
971 int qidx;
972 u64 status;
973
974 netif_tx_disable(nic->netdev);
975
976 /* Check if it is CQ err */
977 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
978 status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
979 qidx);
980 if (!(status & CQ_ERR_MASK))
981 continue;
982 /* Process already queued CQEs and reconfig CQ */
983 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
984 nicvf_sq_disable(nic, qidx);
985 nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0);
986 nicvf_cmp_queue_config(nic, qs, qidx, true);
987 nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx);
988 nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
989
990 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
991 }
992
993 netif_tx_start_all_queues(nic->netdev);
994 /* Re-enable Qset error interrupt */
995 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
996}
997
39ad6eea
SG		 998static void nicvf_dump_intr_status(struct nicvf *nic)
999{
bf24e136
JP		 1000 netif_info(nic, intr, nic->netdev, "interrupt status 0x%llx\n",
1001 nicvf_reg_read(nic, NIC_VF_INT));
39ad6eea
SG		 1002}
1003
4863dea3
SG		 1004static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq)
1005{
1006 struct nicvf *nic = (struct nicvf *)nicvf_irq;
1007 u64 intr;
1008
39ad6eea
SG		 1009 nicvf_dump_intr_status(nic);
1010
4863dea3
SG		 1011 intr = nicvf_reg_read(nic, NIC_VF_INT);
1012 /* Check for spurious interrupt */
1013 if (!(intr & NICVF_INTR_MBOX_MASK))
1014 return IRQ_HANDLED;
1015
1016 nicvf_handle_mbx_intr(nic);
1017
1018 return IRQ_HANDLED;
1019}
1020
39ad6eea
SG		 1021static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq)
1022{
1023 struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq;
1024 struct nicvf *nic = cq_poll->nicvf;
1025 int qidx = cq_poll->cq_idx;
1026
1027 nicvf_dump_intr_status(nic);
1028
1029 /* Disable interrupts */
1030 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1031
1032 /* Schedule NAPI */
ef0a4d86 1033 napi_schedule_irqoff(&cq_poll->napi);
39ad6eea
SG		 1034
1035 /* Clear interrupt */
1036 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
1037
1038 return IRQ_HANDLED;
1039}
1040
1041static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq)
4863dea3 1042{
4863dea3 1043 struct nicvf *nic = (struct nicvf *)nicvf_irq;
39ad6eea 1044 u8 qidx;
4863dea3 1045
4863dea3 1046
39ad6eea
SG		 1047 nicvf_dump_intr_status(nic);
1048
1049 /* Disable RBDR interrupt and schedule softirq */
1050 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
1051 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
4863dea3 1052 continue;
39ad6eea
SG		 1053 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1054 tasklet_hi_schedule(&nic->rbdr_task);
1055 /* Clear interrupt */
1056 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1057 }
4863dea3 1058
39ad6eea
SG		 1059 return IRQ_HANDLED;
1060}
4863dea3 1061
39ad6eea
SG		 1062static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq)
1063{
1064 struct nicvf *nic = (struct nicvf *)nicvf_irq;
4863dea3 1065
39ad6eea
SG		 1066 nicvf_dump_intr_status(nic);
1067
1068 /* Disable Qset err interrupt and schedule softirq */
1069 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1070 tasklet_hi_schedule(&nic->qs_err_task);
1071 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
4863dea3 1072
4863dea3
SG		 1073 return IRQ_HANDLED;
1074}
1075
fb4b7d98
SG		 1076static void nicvf_set_irq_affinity(struct nicvf *nic)
1077{
1078 int vec, cpu;
fb4b7d98
SG		 1079
1080 for (vec = 0; vec < nic->num_vec; vec++) {
1081 if (!nic->irq_allocated[vec])
1082 continue;
1083
1084 if (!zalloc_cpumask_var(&nic->affinity_mask[vec], GFP_KERNEL))
1085 return;
1086 /* CQ interrupts */
1087 if (vec < NICVF_INTR_ID_SQ)
1088 /* Leave CPU0 for RBDR and other interrupts */
1089 cpu = nicvf_netdev_qidx(nic, vec) + 1;
1090 else
1091 cpu = 0;
1092
1093 cpumask_set_cpu(cpumask_local_spread(cpu, nic->node),
1094 nic->affinity_mask[vec]);
ce211b17
TS		 1095 irq_set_affinity_hint(pci_irq_vector(nic->pdev, vec),
1096 nic->affinity_mask[vec]);
fb4b7d98
SG		 1097 }
1098}
1099
4863dea3
SG		 1100static int nicvf_register_interrupts(struct nicvf *nic)
1101{
39ad6eea 1102 int irq, ret = 0;
4863dea3
SG		 1103
1104 for_each_cq_irq(irq)
e4126213
SG		 1105 sprintf(nic->irq_name[irq], "%s-rxtx-%d",
1106 nic->pnicvf->netdev->name,
1107 nicvf_netdev_qidx(nic, irq));
4863dea3
SG		 1108
1109 for_each_sq_irq(irq)
e4126213
SG		 1110 sprintf(nic->irq_name[irq], "%s-sq-%d",
1111 nic->pnicvf->netdev->name,
1112 nicvf_netdev_qidx(nic, irq - NICVF_INTR_ID_SQ));
4863dea3
SG		 1113
1114 for_each_rbdr_irq(irq)
e4126213
SG		 1115 sprintf(nic->irq_name[irq], "%s-rbdr-%d",
1116 nic->pnicvf->netdev->name,
1117 nic->sqs_mode ? (nic->sqs_id + 1) : 0);
4863dea3 1118
39ad6eea
SG		 1119 /* Register CQ interrupts */
1120 for (irq = 0; irq < nic->qs->cq_cnt; irq++) {
ce211b17
TS		 1121 ret = request_irq(pci_irq_vector(nic->pdev, irq),
1122 nicvf_intr_handler,
39ad6eea 1123 0, nic->irq_name[irq], nic->napi[irq]);
4863dea3 1124 if (ret)
39ad6eea 1125 goto err;
4863dea3
SG		 1126 nic->irq_allocated[irq] = true;
1127 }
1128
39ad6eea
SG		 1129 /* Register RBDR interrupt */
1130 for (irq = NICVF_INTR_ID_RBDR;
1131 irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) {
ce211b17
TS		 1132 ret = request_irq(pci_irq_vector(nic->pdev, irq),
1133 nicvf_rbdr_intr_handler,
4863dea3
SG		 1134 0, nic->irq_name[irq], nic);
1135 if (ret)
39ad6eea 1136 goto err;
4863dea3
SG		 1137 nic->irq_allocated[irq] = true;
1138 }
1139
39ad6eea 1140 /* Register QS error interrupt */
e4126213
SG		 1141 sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR], "%s-qset-err-%d",
1142 nic->pnicvf->netdev->name,
1143 nic->sqs_mode ? (nic->sqs_id + 1) : 0);
39ad6eea 1144 irq = NICVF_INTR_ID_QS_ERR;
ce211b17 1145 ret = request_irq(pci_irq_vector(nic->pdev, irq),
39ad6eea
SG		 1146 nicvf_qs_err_intr_handler,
1147 0, nic->irq_name[irq], nic);
fb4b7d98
SG		 1148 if (ret)
1149 goto err;
1150
1151 nic->irq_allocated[irq] = true;
1152
1153 /* Set IRQ affinities */
1154 nicvf_set_irq_affinity(nic);
4863dea3 1155
39ad6eea
SG		 1156err:
1157 if (ret)
1158 netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq);
4863dea3 1159
39ad6eea 1160 return ret;
4863dea3
SG		 1161}
1162
1163static void nicvf_unregister_interrupts(struct nicvf *nic)
1164{
ce211b17 1165 struct pci_dev *pdev = nic->pdev;
4863dea3
SG		 1166 int irq;
1167
1168 /* Free registered interrupts */
1169 for (irq = 0; irq < nic->num_vec; irq++) {
39ad6eea
SG		 1170 if (!nic->irq_allocated[irq])
1171 continue;
1172
ce211b17 1173 irq_set_affinity_hint(pci_irq_vector(pdev, irq), NULL);
fb4b7d98
SG		 1174 free_cpumask_var(nic->affinity_mask[irq]);
1175
39ad6eea 1176 if (irq < NICVF_INTR_ID_SQ)
ce211b17 1177 free_irq(pci_irq_vector(pdev, irq), nic->napi[irq]);
39ad6eea 1178 else
ce211b17 1179 free_irq(pci_irq_vector(pdev, irq), nic);
39ad6eea 1180
4863dea3
SG		 1181 nic->irq_allocated[irq] = false;
1182 }
1183
1184 /* Disable MSI-X */
ce211b17
TS		 1185 pci_free_irq_vectors(pdev);
1186 nic->num_vec = 0;
4863dea3
SG		 1187}
1188
1189/* Initialize MSIX vectors and register MISC interrupt.
1190 * Send READY message to PF to check if its alive
1191 */
1192static int nicvf_register_misc_interrupt(struct nicvf *nic)
1193{
1194 int ret = 0;
1195 int irq = NICVF_INTR_ID_MISC;
1196
1197 /* Return if mailbox interrupt is already registered */
ce211b17 1198 if (nic->pdev->msix_enabled)
4863dea3
SG		 1199 return 0;
1200
1201 /* Enable MSI-X */
ce211b17
TS		 1202 nic->num_vec = pci_msix_vec_count(nic->pdev);
1203 ret = pci_alloc_irq_vectors(nic->pdev, nic->num_vec, nic->num_vec,
1204 PCI_IRQ_MSIX);
1205 if (ret < 0) {
1206 netdev_err(nic->netdev,
1207 "Req for #%d msix vectors failed\n", nic->num_vec);
4863dea3 1208 return 1;
ce211b17 1209 }
4863dea3
SG		 1210
1211 sprintf(nic->irq_name[irq], "%s Mbox", "NICVF");
1212 /* Register Misc interrupt */
ce211b17 1213 ret = request_irq(pci_irq_vector(nic->pdev, irq),
4863dea3
SG		 1214 nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic);
1215
1216 if (ret)
1217 return ret;
1218 nic->irq_allocated[irq] = true;
1219
1220 /* Enable mailbox interrupt */
1221 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
1222
1223 /* Check if VF is able to communicate with PF */
1224 if (!nicvf_check_pf_ready(nic)) {
1225 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1226 nicvf_unregister_interrupts(nic);
1227 return 1;
1228 }
1229
1230 return 0;
1231}
1232
1233static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev)
1234{
1235 struct nicvf *nic = netdev_priv(netdev);
1236 int qid = skb_get_queue_mapping(skb);
1237 struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid);
bd3ad7d3
SG		 1238 struct nicvf *snic;
1239 struct snd_queue *sq;
1240 int tmp;
4863dea3
SG		 1241
1242 /* Check for minimum packet length */
1243 if (skb->len <= ETH_HLEN) {
1244 dev_kfree_skb(skb);
1245 return NETDEV_TX_OK;
1246 }
1247
16f2bccd
SG		 1248 /* In XDP case, initial HW tx queues are used for XDP,
1249 * but stack's queue mapping starts at '0', so skip the
1250 * Tx queues attached to Rx queues for XDP.
1251 */
1252 if (nic->xdp_prog)
1253 qid += nic->xdp_tx_queues;
1254
bd3ad7d3
SG		 1255 snic = nic;
1256 /* Get secondary Qset's SQ structure */
1257 if (qid >= MAX_SND_QUEUES_PER_QS) {
1258 tmp = qid / MAX_SND_QUEUES_PER_QS;
1259 snic = (struct nicvf *)nic->snicvf[tmp - 1];
1260 if (!snic) {
1261 netdev_warn(nic->netdev,
1262 "Secondary Qset#%d's ptr not initialized\n",
1263 tmp - 1);
1264 dev_kfree_skb(skb);
1265 return NETDEV_TX_OK;
1266 }
1267 qid = qid % MAX_SND_QUEUES_PER_QS;
1268 }
1269
1270 sq = &snic->qs->sq[qid];
1271 if (!netif_tx_queue_stopped(txq) &&
1272 !nicvf_sq_append_skb(snic, sq, skb, qid)) {
4863dea3 1273 netif_tx_stop_queue(txq);
bd3ad7d3
SG		 1274
1275 /* Barrier, so that stop_queue visible to other cpus */
1276 smp_mb();
1277
1278 /* Check again, incase another cpu freed descriptors */
1279 if (atomic_read(&sq->free_cnt) > MIN_SQ_DESC_PER_PKT_XMIT) {
1280 netif_tx_wake_queue(txq);
1281 } else {
1282 this_cpu_inc(nic->drv_stats->txq_stop);
bf24e136
JP		 1283 netif_warn(nic, tx_err, netdev,
1284 "Transmit ring full, stopping SQ%d\n", qid);
bd3ad7d3 1285 }
4863dea3
SG		 1286 return NETDEV_TX_BUSY;
1287 }
1288
1289 return NETDEV_TX_OK;
1290}
1291
39ad6eea
SG		 1292static inline void nicvf_free_cq_poll(struct nicvf *nic)
1293{
1294 struct nicvf_cq_poll *cq_poll;
1295 int qidx;
1296
1297 for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1298 cq_poll = nic->napi[qidx];
1299 if (!cq_poll)
1300 continue;
1301 nic->napi[qidx] = NULL;
1302 kfree(cq_poll);
1303 }
1304}
1305
4863dea3
SG		 1306int nicvf_stop(struct net_device *netdev)
1307{
1308 int irq, qidx;
1309 struct nicvf *nic = netdev_priv(netdev);
1310 struct queue_set *qs = nic->qs;
1311 struct nicvf_cq_poll *cq_poll = NULL;
1312 union nic_mbx mbx = {};
1313
1314 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
1315 nicvf_send_msg_to_pf(nic, &mbx);
1316
1317 netif_carrier_off(netdev);
92dc8769 1318 netif_tx_stop_all_queues(nic->netdev);
0b72a9a1 1319 nic->link_up = false;
92dc8769
SG		 1320
1321 /* Teardown secondary qsets first */
1322 if (!nic->sqs_mode) {
1323 for (qidx = 0; qidx < nic->sqs_count; qidx++) {
1324 if (!nic->snicvf[qidx])
1325 continue;
1326 nicvf_stop(nic->snicvf[qidx]->netdev);
1327 nic->snicvf[qidx] = NULL;
1328 }
1329 }
4863dea3
SG		 1330
1331 /* Disable RBDR & QS error interrupts */
1332 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
1333 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1334 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1335 }
1336 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1337 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
1338
1339 /* Wait for pending IRQ handlers to finish */
1340 for (irq = 0; irq < nic->num_vec; irq++)
ce211b17 1341 synchronize_irq(pci_irq_vector(nic->pdev, irq));
4863dea3
SG		 1342
1343 tasklet_kill(&nic->rbdr_task);
1344 tasklet_kill(&nic->qs_err_task);
1345 if (nic->rb_work_scheduled)
1346 cancel_delayed_work_sync(&nic->rbdr_work);
1347
1348 for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1349 cq_poll = nic->napi[qidx];
1350 if (!cq_poll)
1351 continue;
4863dea3
SG		 1352 napi_synchronize(&cq_poll->napi);
1353 /* CQ intr is enabled while napi_complete,
1354 * so disable it now
1355 */
1356 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1357 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
1358 napi_disable(&cq_poll->napi);
1359 netif_napi_del(&cq_poll->napi);
4863dea3
SG		 1360 }
1361
b49087dd
SG		 1362 netif_tx_disable(netdev);
1363
2c204c2b
SG		 1364 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
1365 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
1366
4863dea3
SG		 1367 /* Free resources */
1368 nicvf_config_data_transfer(nic, false);
1369
1370 /* Disable HW Qset */
1371 nicvf_qset_config(nic, false);
1372
1373 /* disable mailbox interrupt */
1374 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1375
1376 nicvf_unregister_interrupts(nic);
1377
39ad6eea
SG		 1378 nicvf_free_cq_poll(nic);
1379
4a875509
SG		 1380 /* Free any pending SKB saved to receive timestamp */
1381 if (nic->ptp_skb) {
1382 dev_kfree_skb_any(nic->ptp_skb);
1383 nic->ptp_skb = NULL;
1384 }
1385
92dc8769
SG		 1386 /* Clear multiqset info */
1387 nic->pnicvf = nic;
92dc8769 1388
4863dea3
SG		 1389 return 0;
1390}
1391
4a875509
SG		 1392static int nicvf_config_hw_rx_tstamp(struct nicvf *nic, bool enable)
1393{
1394 union nic_mbx mbx = {};
1395
1396 mbx.ptp.msg = NIC_MBOX_MSG_PTP_CFG;
1397 mbx.ptp.enable = enable;
1398
1399 return nicvf_send_msg_to_pf(nic, &mbx);
1400}
1401
712c3185
SG		 1402static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
1403{
1404 union nic_mbx mbx = {};
1405
1406 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
1407 mbx.frs.max_frs = mtu;
1408 mbx.frs.vf_id = nic->vf_id;
1409
1410 return nicvf_send_msg_to_pf(nic, &mbx);
1411}
1412
4863dea3
SG		 1413int nicvf_open(struct net_device *netdev)
1414{
964cb69b 1415 int cpu, err, qidx;
4863dea3
SG		 1416 struct nicvf *nic = netdev_priv(netdev);
1417 struct queue_set *qs = nic->qs;
1418 struct nicvf_cq_poll *cq_poll = NULL;
c94acf80 1419 union nic_mbx mbx = {};
4863dea3
SG		 1420
1421 netif_carrier_off(netdev);
1422
1423 err = nicvf_register_misc_interrupt(nic);
1424 if (err)
1425 return err;
1426
1427 /* Register NAPI handler for processing CQEs */
1428 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1429 cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL);
1430 if (!cq_poll) {
1431 err = -ENOMEM;
1432 goto napi_del;
1433 }
1434 cq_poll->cq_idx = qidx;
39ad6eea 1435 cq_poll->nicvf = nic;
4863dea3
SG		 1436 netif_napi_add(netdev, &cq_poll->napi, nicvf_poll,
1437 NAPI_POLL_WEIGHT);
1438 napi_enable(&cq_poll->napi);
1439 nic->napi[qidx] = cq_poll;
1440 }
1441
1442 /* Check if we got MAC address from PF or else generate a radom MAC */
a3a8ce4c 1443 if (!nic->sqs_mode && is_zero_ether_addr(netdev->dev_addr)) {
4863dea3
SG		 1444 eth_hw_addr_random(netdev);
1445 nicvf_hw_set_mac_addr(nic, netdev);
1446 }
1447
bd049a90
PF		 1448 if (nic->set_mac_pending) {
1449 nic->set_mac_pending = false;
1450 nicvf_hw_set_mac_addr(nic, netdev);
1451 }
1452
4863dea3
SG		 1453 /* Init tasklet for handling Qset err interrupt */
1454 tasklet_init(&nic->qs_err_task, nicvf_handle_qs_err,
1455 (unsigned long)nic);
1456
1457 /* Init RBDR tasklet which will refill RBDR */
1458 tasklet_init(&nic->rbdr_task, nicvf_rbdr_task,
1459 (unsigned long)nic);
1460 INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work);
1461
1462 /* Configure CPI alorithm */
1463 nic->cpi_alg = cpi_alg;
92dc8769
SG		 1464 if (!nic->sqs_mode)
1465 nicvf_config_cpi(nic);
1466
1467 nicvf_request_sqs(nic);
1468 if (nic->sqs_mode)
1469 nicvf_get_primary_vf_struct(nic);
4863dea3 1470
4a875509
SG		 1471 /* Configure PTP timestamp */
1472 if (nic->ptp_clock)
1473 nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp);
1474 atomic_set(&nic->tx_ptp_skbs, 0);
1475 nic->ptp_skb = NULL;
1476
712c3185
SG		 1477 /* Configure receive side scaling and MTU */
1478 if (!nic->sqs_mode) {
92dc8769 1479 nicvf_rss_init(nic);
60dce04b
WY		 1480 err = nicvf_update_hw_max_frs(nic, netdev->mtu);
1481 if (err)
712c3185 1482 goto cleanup;
964cb69b
SG		 1483
1484 /* Clear percpu stats */
1485 for_each_possible_cpu(cpu)
1486 memset(per_cpu_ptr(nic->drv_stats, cpu), 0,
1487 sizeof(struct nicvf_drv_stats));
712c3185 1488 }
4863dea3
SG		 1489
1490 err = nicvf_register_interrupts(nic);
1491 if (err)
1492 goto cleanup;
1493
1494 /* Initialize the queues */
1495 err = nicvf_init_resources(nic);
1496 if (err)
1497 goto cleanup;
1498
1499 /* Make sure queue initialization is written */
1500 wmb();
1501
1502 nicvf_reg_write(nic, NIC_VF_INT, -1);
1503 /* Enable Qset err interrupt */
1504 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
1505
1506 /* Enable completion queue interrupt */
1507 for (qidx = 0; qidx < qs->cq_cnt; qidx++)
1508 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
1509
1510 /* Enable RBDR threshold interrupt */
1511 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
1512 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
1513
c94acf80
SG		 1514 /* Send VF config done msg to PF */
1515 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
1516 nicvf_write_to_mbx(nic, &mbx);
74840b83 1517
4863dea3
SG		 1518 return 0;
1519cleanup:
1520 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1521 nicvf_unregister_interrupts(nic);
39ad6eea
SG		 1522 tasklet_kill(&nic->qs_err_task);
1523 tasklet_kill(&nic->rbdr_task);
4863dea3
SG		 1524napi_del:
1525 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1526 cq_poll = nic->napi[qidx];
1527 if (!cq_poll)
1528 continue;
1529 napi_disable(&cq_poll->napi);
1530 netif_napi_del(&cq_poll->napi);
4863dea3 1531 }
39ad6eea 1532 nicvf_free_cq_poll(nic);
4863dea3
SG		 1533 return err;
1534}
1535
4863dea3
SG		 1536static int nicvf_change_mtu(struct net_device *netdev, int new_mtu)
1537{
1538 struct nicvf *nic = netdev_priv(netdev);
f9aa9dc7 1539 int orig_mtu = netdev->mtu;
4863dea3 1540
4863dea3 1541 netdev->mtu = new_mtu;
712c3185
SG		 1542
1543 if (!netif_running(netdev))
1544 return 0;
1545
f9aa9dc7
DM		 1546 if (nicvf_update_hw_max_frs(nic, new_mtu)) {
1547 netdev->mtu = orig_mtu;
4863dea3 1548 return -EINVAL;
f9aa9dc7 1549 }
4863dea3
SG		 1550
1551 return 0;
1552}
1553
1554static int nicvf_set_mac_address(struct net_device *netdev, void *p)
1555{
1556 struct sockaddr *addr = p;
1557 struct nicvf *nic = netdev_priv(netdev);
1558
1559 if (!is_valid_ether_addr(addr->sa_data))
1560 return -EADDRNOTAVAIL;
1561
1562 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1563
ce211b17 1564 if (nic->pdev->msix_enabled) {
4863dea3
SG		 1565 if (nicvf_hw_set_mac_addr(nic, netdev))
1566 return -EBUSY;
bd049a90
PF		 1567 } else {
1568 nic->set_mac_pending = true;
1569 }
4863dea3
SG		 1570
1571 return 0;
1572}
1573
4863dea3
SG		 1574void nicvf_update_lmac_stats(struct nicvf *nic)
1575{
1576 int stat = 0;
1577 union nic_mbx mbx = {};
4863dea3
SG		 1578
1579 if (!netif_running(nic->netdev))
1580 return;
1581
1582 mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS;
1583 mbx.bgx_stats.vf_id = nic->vf_id;
1584 /* Rx stats */
1585 mbx.bgx_stats.rx = 1;
1586 while (stat < BGX_RX_STATS_COUNT) {
4863dea3 1587 mbx.bgx_stats.idx = stat;
6051cba7
SG		 1588 if (nicvf_send_msg_to_pf(nic, &mbx))
1589 return;
4863dea3
SG		 1590 stat++;
1591 }
1592
1593 stat = 0;
1594
1595 /* Tx stats */
1596 mbx.bgx_stats.rx = 0;
1597 while (stat < BGX_TX_STATS_COUNT) {
4863dea3 1598 mbx.bgx_stats.idx = stat;
6051cba7
SG		 1599 if (nicvf_send_msg_to_pf(nic, &mbx))
1600 return;
4863dea3
SG		 1601 stat++;
1602 }
1603}
1604
1605void nicvf_update_stats(struct nicvf *nic)
1606{
964cb69b
SG		 1607 int qidx, cpu;
1608 u64 tmp_stats = 0;
a2dc5ded 1609 struct nicvf_hw_stats *stats = &nic->hw_stats;
964cb69b 1610 struct nicvf_drv_stats *drv_stats;
4863dea3
SG		 1611 struct queue_set *qs = nic->qs;
1612
1613#define GET_RX_STATS(reg) \
1614 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3))
1615#define GET_TX_STATS(reg) \
1616 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3))
1617
a2dc5ded
SG		 1618 stats->rx_bytes = GET_RX_STATS(RX_OCTS);
1619 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
1620 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
1621 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
4863dea3
SG		 1622 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
1623 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
1624 stats->rx_drop_red = GET_RX_STATS(RX_RED);
a2dc5ded 1625 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
4863dea3 1626 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
a2dc5ded 1627 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
4863dea3
SG		 1628 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
1629 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
1630 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
1631 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
1632
964cb69b
SG		 1633 stats->tx_bytes = GET_TX_STATS(TX_OCTS);
1634 stats->tx_ucast_frames = GET_TX_STATS(TX_UCAST);
1635 stats->tx_bcast_frames = GET_TX_STATS(TX_BCAST);
1636 stats->tx_mcast_frames = GET_TX_STATS(TX_MCAST);
4863dea3
SG		 1637 stats->tx_drops = GET_TX_STATS(TX_DROP);
1638
964cb69b
SG		 1639 /* On T88 pass 2.0, the dummy SQE added for TSO notification
1640 * via CQE has 'dont_send' set. Hence HW drops the pkt pointed
1641 * pointed by dummy SQE and results in tx_drops counter being
1642 * incremented. Subtracting it from tx_tso counter will give
1643 * exact tx_drops counter.
1644 */
1645 if (nic->t88 && nic->hw_tso) {
1646 for_each_possible_cpu(cpu) {
1647 drv_stats = per_cpu_ptr(nic->drv_stats, cpu);
1648 tmp_stats += drv_stats->tx_tso;
1649 }
1650 stats->tx_drops = tmp_stats - stats->tx_drops;
1651 }
1652 stats->tx_frames = stats->tx_ucast_frames +
1653 stats->tx_bcast_frames +
1654 stats->tx_mcast_frames;
1655 stats->rx_frames = stats->rx_ucast_frames +
1656 stats->rx_bcast_frames +
1657 stats->rx_mcast_frames;
1658 stats->rx_drops = stats->rx_drop_red +
1659 stats->rx_drop_overrun;
4863dea3
SG		 1660
1661 /* Update RQ and SQ stats */
1662 for (qidx = 0; qidx < qs->rq_cnt; qidx++)
1663 nicvf_update_rq_stats(nic, qidx);
1664 for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1665 nicvf_update_sq_stats(nic, qidx);
1666}
1667
bc1f4470 1668static void nicvf_get_stats64(struct net_device *netdev,
1669 struct rtnl_link_stats64 *stats)
4863dea3
SG		 1670{
1671 struct nicvf *nic = netdev_priv(netdev);
a2dc5ded 1672 struct nicvf_hw_stats *hw_stats = &nic->hw_stats;
4863dea3
SG		 1673
1674 nicvf_update_stats(nic);
1675
a2dc5ded 1676 stats->rx_bytes = hw_stats->rx_bytes;
964cb69b
SG		 1677 stats->rx_packets = hw_stats->rx_frames;
1678 stats->rx_dropped = hw_stats->rx_drops;
a2dc5ded 1679 stats->multicast = hw_stats->rx_mcast_frames;
4863dea3 1680
964cb69b
SG		 1681 stats->tx_bytes = hw_stats->tx_bytes;
1682 stats->tx_packets = hw_stats->tx_frames;
1683 stats->tx_dropped = hw_stats->tx_drops;
4863dea3 1684
4863dea3
SG		 1685}
1686
1687static void nicvf_tx_timeout(struct net_device *dev)
1688{
1689 struct nicvf *nic = netdev_priv(dev);
1690
bf24e136 1691 netif_warn(nic, tx_err, dev, "Transmit timed out, resetting\n");
4863dea3 1692
964cb69b 1693 this_cpu_inc(nic->drv_stats->tx_timeout);
4863dea3
SG		 1694 schedule_work(&nic->reset_task);
1695}
1696
1697static void nicvf_reset_task(struct work_struct *work)
1698{
1699 struct nicvf *nic;
1700
1701 nic = container_of(work, struct nicvf, reset_task);
1702
1703 if (!netif_running(nic->netdev))
1704 return;
1705
1706 nicvf_stop(nic->netdev);
1707 nicvf_open(nic->netdev);
860e9538 1708 netif_trans_update(nic->netdev);
4863dea3
SG		 1709}
1710
d77a2384
SG		 1711static int nicvf_config_loopback(struct nicvf *nic,
1712 netdev_features_t features)
1713{
1714 union nic_mbx mbx = {};
1715
1716 mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK;
1717 mbx.lbk.vf_id = nic->vf_id;
1718 mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0;
1719
1720 return nicvf_send_msg_to_pf(nic, &mbx);
1721}
1722
1723static netdev_features_t nicvf_fix_features(struct net_device *netdev,
1724 netdev_features_t features)
1725{
1726 struct nicvf *nic = netdev_priv(netdev);
1727
1728 if ((features & NETIF_F_LOOPBACK) &&
1729 netif_running(netdev) && !nic->loopback_supported)
1730 features &= ~NETIF_F_LOOPBACK;
1731
1732 return features;
1733}
1734
aa2e259b
SG		 1735static int nicvf_set_features(struct net_device *netdev,
1736 netdev_features_t features)
1737{
1738 struct nicvf *nic = netdev_priv(netdev);
1739 netdev_features_t changed = features ^ netdev->features;
1740
1741 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1742 nicvf_config_vlan_stripping(nic, features);
1743
d77a2384
SG		 1744 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
1745 return nicvf_config_loopback(nic, features);
1746
aa2e259b
SG		 1747 return 0;
1748}
1749
05c773f5
SG		 1750static void nicvf_set_xdp_queues(struct nicvf *nic, bool bpf_attached)
1751{
1752 u8 cq_count, txq_count;
1753
1754 /* Set XDP Tx queue count same as Rx queue count */
1755 if (!bpf_attached)
1756 nic->xdp_tx_queues = 0;
1757 else
1758 nic->xdp_tx_queues = nic->rx_queues;
1759
1760 /* If queue count > MAX_CMP_QUEUES_PER_QS, then additional qsets
1761 * needs to be allocated, check how many.
1762 */
1763 txq_count = nic->xdp_tx_queues + nic->tx_queues;
1764 cq_count = max(nic->rx_queues, txq_count);
1765 if (cq_count > MAX_CMP_QUEUES_PER_QS) {
1766 nic->sqs_count = roundup(cq_count, MAX_CMP_QUEUES_PER_QS);
1767 nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1;
1768 } else {
1769 nic->sqs_count = 0;
1770 }
1771
1772 /* Set primary Qset's resources */
1773 nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS);
1774 nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS);
1775 nic->qs->cq_cnt = max_t(u8, nic->qs->rq_cnt, nic->qs->sq_cnt);
1776
1777 /* Update stack */
1778 nicvf_set_real_num_queues(nic->netdev, nic->tx_queues, nic->rx_queues);
1779}
1780
1781static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog)
1782{
1783 struct net_device *dev = nic->netdev;
1784 bool if_up = netif_running(nic->netdev);
1785 struct bpf_prog *old_prog;
1786 bool bpf_attached = false;
6d0f60b0 1787 int ret = 0;
05c773f5
SG		 1788
1789 /* For now just support only the usual MTU sized frames */
1790 if (prog && (dev->mtu > 1500)) {
1791 netdev_warn(dev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
1792 dev->mtu);
1793 return -EOPNOTSUPP;
1794 }
1795
05c773f5
SG		 1796 /* ALL SQs attached to CQs i.e same as RQs, are treated as
1797 * XDP Tx queues and more Tx queues are allocated for
1798 * network stack to send pkts out.
1799 *
1800 * No of Tx queues are either same as Rx queues or whatever
1801 * is left in max no of queues possible.
1802 */
1803 if ((nic->rx_queues + nic->tx_queues) > nic->max_queues) {
1804 netdev_warn(dev,
1805 "Failed to attach BPF prog, RXQs + TXQs > Max %d\n",
1806 nic->max_queues);
1807 return -ENOMEM;
1808 }
1809
1810 if (if_up)
1811 nicvf_stop(nic->netdev);
1812
1813 old_prog = xchg(&nic->xdp_prog, prog);
1814 /* Detach old prog, if any */
1815 if (old_prog)
1816 bpf_prog_put(old_prog);
1817
1818 if (nic->xdp_prog) {
1819 /* Attach BPF program */
1820 nic->xdp_prog = bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);
6d0f60b0 1821 if (!IS_ERR(nic->xdp_prog)) {
05c773f5 1822 bpf_attached = true;
6d0f60b0
LB		 1823 } else {
1824 ret = PTR_ERR(nic->xdp_prog);
1825 nic->xdp_prog = NULL;
1826 }
05c773f5
SG		 1827 }
1828
1829 /* Calculate Tx queues needed for XDP and network stack */
1830 nicvf_set_xdp_queues(nic, bpf_attached);
1831
1832 if (if_up) {
1833 /* Reinitialize interface, clean slate */
1834 nicvf_open(nic->netdev);
1835 netif_trans_update(nic->netdev);
1836 }
1837
6d0f60b0 1838 return ret;
05c773f5
SG		 1839}
1840
f4e63525 1841static int nicvf_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
05c773f5
SG		 1842{
1843 struct nicvf *nic = netdev_priv(netdev);
1844
1845 /* To avoid checks while retrieving buffer address from CQE_RX,
1846 * do not support XDP for T88 pass1.x silicons which are anyway
1847 * not in use widely.
1848 */
1849 if (pass1_silicon(nic->pdev))
1850 return -EOPNOTSUPP;
1851
1852 switch (xdp->command) {
1853 case XDP_SETUP_PROG:
1854 return nicvf_xdp_setup(nic, xdp->prog);
1855 case XDP_QUERY_PROG:
1efde2b6 1856 xdp->prog_id = nic->xdp_prog ? nic->xdp_prog->aux->id : 0;
05c773f5
SG		 1857 return 0;
1858 default:
1859 return -EINVAL;
1860 }
1861}
1862
4a875509
SG		 1863static int nicvf_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr)
1864{
1865 struct hwtstamp_config config;
1866 struct nicvf *nic = netdev_priv(netdev);
1867
1868 if (!nic->ptp_clock)
1869 return -ENODEV;
1870
1871 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1872 return -EFAULT;
1873
1874 /* reserved for future extensions */
1875 if (config.flags)
1876 return -EINVAL;
1877
1878 switch (config.tx_type) {
1879 case HWTSTAMP_TX_OFF:
1880 case HWTSTAMP_TX_ON:
1881 break;
1882 default:
1883 return -ERANGE;
1884 }
1885
1886 switch (config.rx_filter) {
1887 case HWTSTAMP_FILTER_NONE:
1888 nic->hw_rx_tstamp = false;
1889 break;
1890 case HWTSTAMP_FILTER_ALL:
1891 case HWTSTAMP_FILTER_SOME:
1892 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1893 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1894 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1895 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1896 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1897 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1898 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1899 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1900 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1901 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1902 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1903 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1904 nic->hw_rx_tstamp = true;
1905 config.rx_filter = HWTSTAMP_FILTER_ALL;
1906 break;
1907 default:
1908 return -ERANGE;
1909 }
1910
1911 if (netif_running(netdev))
1912 nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp);
1913
1914 if (copy_to_user(ifr->ifr_data, &config, sizeof(config)))
1915 return -EFAULT;
1916
1917 return 0;
1918}
1919
1920static int nicvf_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
1921{
1922 switch (cmd) {
1923 case SIOCSHWTSTAMP:
1924 return nicvf_config_hwtstamp(netdev, req);
1925 default:
1926 return -EOPNOTSUPP;
1927 }
1928}
1929
469998c8
VL		 1930static void __nicvf_set_rx_mode_task(u8 mode, struct xcast_addr_list *mc_addrs,
1931 struct nicvf *nic)
37c3347e 1932{
37c3347e 1933 union nic_mbx mbx = {};
9b5c4dfb 1934 int idx;
37c3347e 1935
37c3347e
VL		 1936 /* From the inside of VM code flow we have only 128 bits memory
1937 * available to send message to host's PF, so send all mc addrs
1938 * one by one, starting from flush command in case if kernel
1939 * requests to configure specific MAC filtering
1940 */
1941
1942 /* flush DMAC filters and reset RX mode */
1943 mbx.xcast.msg = NIC_MBOX_MSG_RESET_XCAST;
1944 nicvf_send_msg_to_pf(nic, &mbx);
1945
469998c8 1946 if (mode & BGX_XCAST_MCAST_FILTER) {
37c3347e
VL		 1947 /* once enabling filtering, we need to signal to PF to add
1948 * its' own LMAC to the filter to accept packets for it.
1949 */
1950 mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST;
1951 mbx.xcast.data.mac = 0;
1952 nicvf_send_msg_to_pf(nic, &mbx);
1953 }
1954
1955 /* check if we have any specific MACs to be added to PF DMAC filter */
469998c8 1956 if (mc_addrs) {
37c3347e 1957 /* now go through kernel list of MACs and add them one by one */
469998c8 1958 for (idx = 0; idx < mc_addrs->count; idx++) {
37c3347e 1959 mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST;
469998c8 1960 mbx.xcast.data.mac = mc_addrs->mc[idx];
37c3347e 1961 nicvf_send_msg_to_pf(nic, &mbx);
37c3347e 1962 }
469998c8 1963 kfree(mc_addrs);
37c3347e
VL		 1964 }
1965
1966 /* and finally set rx mode for PF accordingly */
1967 mbx.xcast.msg = NIC_MBOX_MSG_SET_XCAST;
469998c8 1968 mbx.xcast.data.mode = mode;
37c3347e
VL		 1969
1970 nicvf_send_msg_to_pf(nic, &mbx);
1971}
1972
469998c8
VL		 1973static void nicvf_set_rx_mode_task(struct work_struct *work_arg)
1974{
1975 struct nicvf_work *vf_work = container_of(work_arg, struct nicvf_work,
1976 work.work);
1977 struct nicvf *nic = container_of(vf_work, struct nicvf, rx_mode_work);
1978 u8 mode;
1979 struct xcast_addr_list *mc;
1980
1981 if (!vf_work)
1982 return;
1983
1984 /* Save message data locally to prevent them from
1985 * being overwritten by next ndo_set_rx_mode call().
1986 */
1987 spin_lock(&nic->rx_mode_wq_lock);
1988 mode = vf_work->mode;
1989 mc = vf_work->mc;
1990 vf_work->mc = NULL;
1991 spin_unlock(&nic->rx_mode_wq_lock);
1992
1993 __nicvf_set_rx_mode_task(mode, mc, nic);
1994}
1995
37c3347e
VL		 1996static void nicvf_set_rx_mode(struct net_device *netdev)
1997{
1998 struct nicvf *nic = netdev_priv(netdev);
1999 struct netdev_hw_addr *ha;
2000 struct xcast_addr_list *mc_list = NULL;
2001 u8 mode = 0;
2002
2003 if (netdev->flags & IFF_PROMISC) {
2004 mode = BGX_XCAST_BCAST_ACCEPT | BGX_XCAST_MCAST_ACCEPT;
2005 } else {
2006 if (netdev->flags & IFF_BROADCAST)
2007 mode |= BGX_XCAST_BCAST_ACCEPT;
2008
2009 if (netdev->flags & IFF_ALLMULTI) {
2010 mode |= BGX_XCAST_MCAST_ACCEPT;
2011 } else if (netdev->flags & IFF_MULTICAST) {
2012 mode |= BGX_XCAST_MCAST_FILTER;
2013 /* here we need to copy mc addrs */
2014 if (netdev_mc_count(netdev)) {
9b5c4dfb
VL		 2015 mc_list = kmalloc(offsetof(typeof(*mc_list),
2016 mc[netdev_mc_count(netdev)]),
2017 GFP_ATOMIC);
2018 if (unlikely(!mc_list))
2019 return;
2020 mc_list->count = 0;
37c3347e 2021 netdev_hw_addr_list_for_each(ha, &netdev->mc) {
9b5c4dfb 2022 mc_list->mc[mc_list->count] =
37c3347e 2023 ether_addr_to_u64(ha->addr);
37c3347e
VL		 2024 mc_list->count++;
2025 }
2026 }
2027 }
2028 }
469998c8
VL		 2029 spin_lock(&nic->rx_mode_wq_lock);
2030 kfree(nic->rx_mode_work.mc);
37c3347e
VL		 2031 nic->rx_mode_work.mc = mc_list;
2032 nic->rx_mode_work.mode = mode;
469998c8
VL		 2033 queue_delayed_work(nicvf_rx_mode_wq, &nic->rx_mode_work.work, 0);
2034 spin_unlock(&nic->rx_mode_wq_lock);
37c3347e
VL		 2035}
2036
4863dea3
SG		 2037static const struct net_device_ops nicvf_netdev_ops = {
2038 .ndo_open = nicvf_open,
2039 .ndo_stop = nicvf_stop,
2040 .ndo_start_xmit = nicvf_xmit,
2041 .ndo_change_mtu = nicvf_change_mtu,
2042 .ndo_set_mac_address = nicvf_set_mac_address,
2043 .ndo_get_stats64 = nicvf_get_stats64,
2044 .ndo_tx_timeout = nicvf_tx_timeout,
d77a2384 2045 .ndo_fix_features = nicvf_fix_features,
aa2e259b 2046 .ndo_set_features = nicvf_set_features,
f4e63525 2047 .ndo_bpf = nicvf_xdp,
4a875509 2048 .ndo_do_ioctl = nicvf_ioctl,
37c3347e 2049 .ndo_set_rx_mode = nicvf_set_rx_mode,
4863dea3
SG		 2050};
2051
2052static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2053{
2054 struct device *dev = &pdev->dev;
2055 struct net_device *netdev;
2056 struct nicvf *nic;
92dc8769 2057 int err, qcount;
7ceb8a13 2058 u16 sdevid;
4a875509
SG		 2059 struct cavium_ptp *ptp_clock;
2060
2061 ptp_clock = cavium_ptp_get();
2062 if (IS_ERR(ptp_clock)) {
2063 if (PTR_ERR(ptp_clock) == -ENODEV)
2064 /* In virtualized environment we proceed without ptp */
2065 ptp_clock = NULL;
2066 else
2067 return PTR_ERR(ptp_clock);
2068 }
4863dea3
SG		 2069
2070 err = pci_enable_device(pdev);
2071 if (err) {
2072 dev_err(dev, "Failed to enable PCI device\n");
2073 return err;
2074 }
2075
2076 err = pci_request_regions(pdev, DRV_NAME);
2077 if (err) {
2078 dev_err(dev, "PCI request regions failed 0x%x\n", err);
2079 goto err_disable_device;
2080 }
2081
2082 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
2083 if (err) {
2084 dev_err(dev, "Unable to get usable DMA configuration\n");
2085 goto err_release_regions;
2086 }
2087
2088 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
2089 if (err) {
2090 dev_err(dev, "unable to get 48-bit DMA for consistent allocations\n");
2091 goto err_release_regions;
2092 }
2093
3a397ebe 2094 qcount = netif_get_num_default_rss_queues();
92dc8769
SG		 2095
2096 /* Restrict multiqset support only for host bound VFs */
2097 if (pdev->is_virtfn) {
2098 /* Set max number of queues per VF */
3a397ebe
SG		 2099 qcount = min_t(int, num_online_cpus(),
2100 (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
92dc8769
SG		 2101 }
2102
2103 netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount);
4863dea3
SG		 2104 if (!netdev) {
2105 err = -ENOMEM;
2106 goto err_release_regions;
2107 }
2108
2109 pci_set_drvdata(pdev, netdev);
2110
2111 SET_NETDEV_DEV(netdev, &pdev->dev);
2112
2113 nic = netdev_priv(netdev);
2114 nic->netdev = netdev;
2115 nic->pdev = pdev;
92dc8769
SG		 2116 nic->pnicvf = nic;
2117 nic->max_queues = qcount;
87de0838
SG		 2118 /* If no of CPUs are too low, there won't be any queues left
2119 * for XDP_TX, hence double it.
2120 */
2121 if (!nic->t88)
2122 nic->max_queues *= 2;
4a875509 2123 nic->ptp_clock = ptp_clock;
4863dea3
SG		 2124
2125 /* MAP VF's configuration registers */
2126 nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
2127 if (!nic->reg_base) {
2128 dev_err(dev, "Cannot map config register space, aborting\n");
2129 err = -ENOMEM;
2130 goto err_free_netdev;
2131 }
2132
964cb69b
SG		 2133 nic->drv_stats = netdev_alloc_pcpu_stats(struct nicvf_drv_stats);
2134 if (!nic->drv_stats) {
2135 err = -ENOMEM;
2136 goto err_free_netdev;
2137 }
2138
4863dea3
SG		 2139 err = nicvf_set_qset_resources(nic);
2140 if (err)
2141 goto err_free_netdev;
2142
4863dea3
SG		 2143 /* Check if PF is alive and get MAC address for this VF */
2144 err = nicvf_register_misc_interrupt(nic);
2145 if (err)
2146 goto err_free_netdev;
2147
92dc8769
SG		 2148 nicvf_send_vf_struct(nic);
2149
8d210d54
SG		 2150 if (!pass1_silicon(nic->pdev))
2151 nic->hw_tso = true;
2152
83abb7d7
SG		 2153 /* Get iommu domain for iova to physical addr conversion */
2154 nic->iommu_domain = iommu_get_domain_for_dev(dev);
2155
7ceb8a13
SG		 2156 pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid);
2157 if (sdevid == 0xA134)
2158 nic->t88 = true;
2159
92dc8769
SG		 2160 /* Check if this VF is in QS only mode */
2161 if (nic->sqs_mode)
2162 return 0;
2163
2164 err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues);
2165 if (err)
2166 goto err_unregister_interrupts;
2167
3a9024f5
TS		 2168 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_SG |
2169 NETIF_F_TSO | NETIF_F_GRO | NETIF_F_TSO6 |
2170 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
92dc8769
SG		 2171 NETIF_F_HW_VLAN_CTAG_RX);
2172
2173 netdev->hw_features |= NETIF_F_RXHASH;
aa2e259b
SG		 2174
2175 netdev->features |= netdev->hw_features;
d77a2384 2176 netdev->hw_features |= NETIF_F_LOOPBACK;
38bb5d4f 2177
3a9024f5
TS		 2178 netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM |
2179 NETIF_F_IPV6_CSUM | NETIF_F_TSO | NETIF_F_TSO6;
4863dea3
SG		 2180
2181 netdev->netdev_ops = &nicvf_netdev_ops;
3d7a8aaa 2182 netdev->watchdog_timeo = NICVF_TX_TIMEOUT;
4863dea3 2183
109cc165
JW		 2184 /* MTU range: 64 - 9200 */
2185 netdev->min_mtu = NIC_HW_MIN_FRS;
2186 netdev->max_mtu = NIC_HW_MAX_FRS;
2187
4863dea3
SG		 2188 INIT_WORK(&nic->reset_task, nicvf_reset_task);
2189
37c3347e 2190 INIT_DELAYED_WORK(&nic->rx_mode_work.work, nicvf_set_rx_mode_task);
469998c8 2191 spin_lock_init(&nic->rx_mode_wq_lock);
37c3347e 2192
4863dea3
SG		 2193 err = register_netdev(netdev);
2194 if (err) {
2195 dev_err(dev, "Failed to register netdevice\n");
2196 goto err_unregister_interrupts;
2197 }
2198
2199 nic->msg_enable = debug;
2200
2201 nicvf_set_ethtool_ops(netdev);
2202
2203 return 0;
2204
2205err_unregister_interrupts:
2206 nicvf_unregister_interrupts(nic);
2207err_free_netdev:
2208 pci_set_drvdata(pdev, NULL);
964cb69b
SG		 2209 if (nic->drv_stats)
2210 free_percpu(nic->drv_stats);
4863dea3
SG		 2211 free_netdev(netdev);
2212err_release_regions:
2213 pci_release_regions(pdev);
2214err_disable_device:
2215 pci_disable_device(pdev);
2216 return err;
2217}
2218
2219static void nicvf_remove(struct pci_dev *pdev)
2220{
2221 struct net_device *netdev = pci_get_drvdata(pdev);
7750130d
PF		 2222 struct nicvf *nic;
2223 struct net_device *pnetdev;
2224
2225 if (!netdev)
2226 return;
2227
2228 nic = netdev_priv(netdev);
2229 pnetdev = nic->pnicvf->netdev;
4863dea3 2230
37c3347e
VL		 2231 cancel_delayed_work_sync(&nic->rx_mode_work.work);
2232
92dc8769
SG		 2233 /* Check if this Qset is assigned to different VF.
2234 * If yes, clean primary and all secondary Qsets.
2235 */
2236 if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED))
2237 unregister_netdev(pnetdev);
4863dea3
SG		 2238 nicvf_unregister_interrupts(nic);
2239 pci_set_drvdata(pdev, NULL);
964cb69b
SG		 2240 if (nic->drv_stats)
2241 free_percpu(nic->drv_stats);
4a875509 2242 cavium_ptp_put(nic->ptp_clock);
4863dea3
SG		 2243 free_netdev(netdev);
2244 pci_release_regions(pdev);
2245 pci_disable_device(pdev);
2246}
2247
4adf4351
SG		 2248static void nicvf_shutdown(struct pci_dev *pdev)
2249{
2250 nicvf_remove(pdev);
2251}
2252
4863dea3
SG		 2253static struct pci_driver nicvf_driver = {
2254 .name = DRV_NAME,
2255 .id_table = nicvf_id_table,
2256 .probe = nicvf_probe,
2257 .remove = nicvf_remove,
4adf4351 2258 .shutdown = nicvf_shutdown,
4863dea3
SG		 2259};
2260
2261static int __init nicvf_init_module(void)
2262{
2263 pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
37c3347e
VL		 2264 nicvf_rx_mode_wq = alloc_ordered_workqueue("nicvf_generic",
2265 WQ_MEM_RECLAIM);
4863dea3
SG		 2266 return pci_register_driver(&nicvf_driver);
2267}
2268
2269static void __exit nicvf_cleanup_module(void)
2270{
37c3347e
VL		 2271 if (nicvf_rx_mode_wq) {
2272 destroy_workqueue(nicvf_rx_mode_wq);
2273 nicvf_rx_mode_wq = NULL;
2274 }
4863dea3
SG		 2275 pci_unregister_driver(&nicvf_driver);
2276}
2277
2278module_init(nicvf_init_module);
2279module_exit(nicvf_cleanup_module);
Empty description
This page took 0.700044 seconds and 4 git commands to generate.