drivers/net/netdevsim/netdev.c

   1 /*
   2  * Copyright (C) 2017 Netronome Systems, Inc.
   3  *
   4  * This software is licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree.
   7  *
   8  * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
   9  * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
  10  * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  11  * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
  12  * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
  13  * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
  14  */
  15
  16 #include <linux/debugfs.h>
  17 #include <linux/etherdevice.h>
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/netdevice.h>
  21 #include <linux/slab.h>
  22 #include <net/netdev_queues.h>
  23 #include <net/page_pool/helpers.h>
  24 #include <net/netlink.h>
  25 #include <net/pkt_cls.h>
  26 #include <net/rtnetlink.h>
  27 #include <net/udp_tunnel.h>
  28
  29 #include "netdevsim.h"
  30
  31 #define NSIM_RING_SIZE          256
  32
  33 static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb)
  34 {
  35         if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) {
  36                 dev_kfree_skb_any(skb);
  37                 return NET_RX_DROP;
  38         }
  39
  40         skb_queue_tail(&rq->skb_queue, skb);
  41         return NET_RX_SUCCESS;
  42 }
  43
  44 static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb,
  45                             struct nsim_rq *rq)
  46 {
  47         return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb);
  48 }
  49
  50 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
  51 {
  52         struct netdevsim *ns = netdev_priv(dev);
  53         struct net_device *peer_dev;
  54         unsigned int len = skb->len;
  55         struct netdevsim *peer_ns;
  56         struct nsim_rq *rq;
  57         int rxq;
  58
  59         rcu_read_lock();
  60         if (!nsim_ipsec_tx(ns, skb))
  61                 goto out_drop_free;
  62
  63         peer_ns = rcu_dereference(ns->peer);
  64         if (!peer_ns)
  65                 goto out_drop_free;
  66
  67         peer_dev = peer_ns->netdev;
  68         rxq = skb_get_queue_mapping(skb);
  69         if (rxq >= peer_dev->num_rx_queues)
  70                 rxq = rxq % peer_dev->num_rx_queues;
  71         rq = &peer_ns->rq[rxq];
  72
  73         skb_tx_timestamp(skb);
  74         if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP))
  75                 goto out_drop_cnt;
  76
  77         napi_schedule(&rq->napi);
  78
  79         rcu_read_unlock();
  80         u64_stats_update_begin(&ns->syncp);
  81         ns->tx_packets++;
  82         ns->tx_bytes += len;
  83         u64_stats_update_end(&ns->syncp);
  84         return NETDEV_TX_OK;
  85
  86 out_drop_free:
  87         dev_kfree_skb(skb);
  88 out_drop_cnt:
  89         rcu_read_unlock();
  90         u64_stats_update_begin(&ns->syncp);
  91         ns->tx_dropped++;
  92         u64_stats_update_end(&ns->syncp);
  93         return NETDEV_TX_OK;
  94 }
  95
  96 static void nsim_set_rx_mode(struct net_device *dev)
  97 {
  98 }
  99
 100 static int nsim_change_mtu(struct net_device *dev, int new_mtu)
 101 {
 102         struct netdevsim *ns = netdev_priv(dev);
 103
 104         if (ns->xdp.prog && new_mtu > NSIM_XDP_MAX_MTU)
 105                 return -EBUSY;
 106
 107         WRITE_ONCE(dev->mtu, new_mtu);
 108
 109         return 0;
 110 }
 111
 112 static void
 113 nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 114 {
 115         struct netdevsim *ns = netdev_priv(dev);
 116         unsigned int start;
 117
 118         do {
 119                 start = u64_stats_fetch_begin(&ns->syncp);
 120                 stats->tx_bytes = ns->tx_bytes;
 121                 stats->tx_packets = ns->tx_packets;
 122                 stats->tx_dropped = ns->tx_dropped;
 123         } while (u64_stats_fetch_retry(&ns->syncp, start));
 124 }
 125
 126 static int
 127 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
 128 {
 129         return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);
 130 }
 131
 132 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
 133 {
 134         struct netdevsim *ns = netdev_priv(dev);
 135         struct nsim_dev *nsim_dev = ns->nsim_dev;
 136
 137         /* Only refuse multicast addresses, zero address can mean unset/any. */
 138         if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac))
 139                 return -EINVAL;
 140         memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN);
 141
 142         return 0;
 143 }
 144
 145 static int nsim_set_vf_vlan(struct net_device *dev, int vf,
 146                             u16 vlan, u8 qos, __be16 vlan_proto)
 147 {
 148         struct netdevsim *ns = netdev_priv(dev);
 149         struct nsim_dev *nsim_dev = ns->nsim_dev;
 150
 151         if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7)
 152                 return -EINVAL;
 153
 154         nsim_dev->vfconfigs[vf].vlan = vlan;
 155         nsim_dev->vfconfigs[vf].qos = qos;
 156         nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto;
 157
 158         return 0;
 159 }
 160
 161 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max)
 162 {
 163         struct netdevsim *ns = netdev_priv(dev);
 164         struct nsim_dev *nsim_dev = ns->nsim_dev;
 165
 166         if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) {
 167                 pr_err("Not supported in switchdev mode. Please use devlink API.\n");
 168                 return -EOPNOTSUPP;
 169         }
 170
 171         if (vf >= nsim_dev_get_vfs(nsim_dev))
 172                 return -EINVAL;
 173
 174         nsim_dev->vfconfigs[vf].min_tx_rate = min;
 175         nsim_dev->vfconfigs[vf].max_tx_rate = max;
 176
 177         return 0;
 178 }
 179
 180 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
 181 {
 182         struct netdevsim *ns = netdev_priv(dev);
 183         struct nsim_dev *nsim_dev = ns->nsim_dev;
 184
 185         if (vf >= nsim_dev_get_vfs(nsim_dev))
 186                 return -EINVAL;
 187         nsim_dev->vfconfigs[vf].spoofchk_enabled = val;
 188
 189         return 0;
 190 }
 191
 192 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
 193 {
 194         struct netdevsim *ns = netdev_priv(dev);
 195         struct nsim_dev *nsim_dev = ns->nsim_dev;
 196
 197         if (vf >= nsim_dev_get_vfs(nsim_dev))
 198                 return -EINVAL;
 199         nsim_dev->vfconfigs[vf].rss_query_enabled = val;
 200
 201         return 0;
 202 }
 203
 204 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val)
 205 {
 206         struct netdevsim *ns = netdev_priv(dev);
 207         struct nsim_dev *nsim_dev = ns->nsim_dev;
 208
 209         if (vf >= nsim_dev_get_vfs(nsim_dev))
 210                 return -EINVAL;
 211         nsim_dev->vfconfigs[vf].trusted = val;
 212
 213         return 0;
 214 }
 215
 216 static int
 217 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi)
 218 {
 219         struct netdevsim *ns = netdev_priv(dev);
 220         struct nsim_dev *nsim_dev = ns->nsim_dev;
 221
 222         if (vf >= nsim_dev_get_vfs(nsim_dev))
 223                 return -EINVAL;
 224
 225         ivi->vf = vf;
 226         ivi->linkstate = nsim_dev->vfconfigs[vf].link_state;
 227         ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate;
 228         ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate;
 229         ivi->vlan = nsim_dev->vfconfigs[vf].vlan;
 230         ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto;
 231         ivi->qos = nsim_dev->vfconfigs[vf].qos;
 232         memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN);
 233         ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled;
 234         ivi->trusted = nsim_dev->vfconfigs[vf].trusted;
 235         ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled;
 236
 237         return 0;
 238 }
 239
 240 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
 241 {
 242         struct netdevsim *ns = netdev_priv(dev);
 243         struct nsim_dev *nsim_dev = ns->nsim_dev;
 244
 245         if (vf >= nsim_dev_get_vfs(nsim_dev))
 246                 return -EINVAL;
 247
 248         switch (state) {
 249         case IFLA_VF_LINK_STATE_AUTO:
 250         case IFLA_VF_LINK_STATE_ENABLE:
 251         case IFLA_VF_LINK_STATE_DISABLE:
 252                 break;
 253         default:
 254                 return -EINVAL;
 255         }
 256
 257         nsim_dev->vfconfigs[vf].link_state = state;
 258
 259         return 0;
 260 }
 261
 262 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats)
 263 {
 264         stats->window_drops = 0;
 265         stats->tx_overruns = 0;
 266 }
 267
 268 static int nsim_setup_tc_taprio(struct net_device *dev,
 269                                 struct tc_taprio_qopt_offload *offload)
 270 {
 271         int err = 0;
 272
 273         switch (offload->cmd) {
 274         case TAPRIO_CMD_REPLACE:
 275         case TAPRIO_CMD_DESTROY:
 276                 break;
 277         case TAPRIO_CMD_STATS:
 278                 nsim_taprio_stats(&offload->stats);
 279                 break;
 280         default:
 281                 err = -EOPNOTSUPP;
 282         }
 283
 284         return err;
 285 }
 286
 287 static LIST_HEAD(nsim_block_cb_list);
 288
 289 static int
 290 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
 291 {
 292         struct netdevsim *ns = netdev_priv(dev);
 293
 294         switch (type) {
 295         case TC_SETUP_QDISC_TAPRIO:
 296                 return nsim_setup_tc_taprio(dev, type_data);
 297         case TC_SETUP_BLOCK:
 298                 return flow_block_cb_setup_simple(type_data,
 299                                                   &nsim_block_cb_list,
 300                                                   nsim_setup_tc_block_cb,
 301                                                   ns, ns, true);
 302         default:
 303                 return -EOPNOTSUPP;
 304         }
 305 }
 306
 307 static int
 308 nsim_set_features(struct net_device *dev, netdev_features_t features)
 309 {
 310         struct netdevsim *ns = netdev_priv(dev);
 311
 312         if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))
 313                 return nsim_bpf_disable_tc(ns);
 314
 315         return 0;
 316 }
 317
 318 static int nsim_get_iflink(const struct net_device *dev)
 319 {
 320         struct netdevsim *nsim, *peer;
 321         int iflink;
 322
 323         nsim = netdev_priv(dev);
 324
 325         rcu_read_lock();
 326         peer = rcu_dereference(nsim->peer);
 327         iflink = peer ? READ_ONCE(peer->netdev->ifindex) :
 328                         READ_ONCE(dev->ifindex);
 329         rcu_read_unlock();
 330
 331         return iflink;
 332 }
 333
 334 static int nsim_rcv(struct nsim_rq *rq, int budget)
 335 {
 336         struct sk_buff *skb;
 337         int i;
 338
 339         for (i = 0; i < budget; i++) {
 340                 if (skb_queue_empty(&rq->skb_queue))
 341                         break;
 342
 343                 skb = skb_dequeue(&rq->skb_queue);
 344                 netif_receive_skb(skb);
 345         }
 346
 347         return i;
 348 }
 349
 350 static int nsim_poll(struct napi_struct *napi, int budget)
 351 {
 352         struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi);
 353         int done;
 354
 355         done = nsim_rcv(rq, budget);
 356         napi_complete(napi);
 357
 358         return done;
 359 }
 360
 361 static int nsim_create_page_pool(struct nsim_rq *rq)
 362 {
 363         struct page_pool_params p = {
 364                 .order = 0,
 365                 .pool_size = NSIM_RING_SIZE,
 366                 .nid = NUMA_NO_NODE,
 367                 .dev = &rq->napi.dev->dev,
 368                 .napi = &rq->napi,
 369                 .dma_dir = DMA_BIDIRECTIONAL,
 370                 .netdev = rq->napi.dev,
 371         };
 372
 373         rq->page_pool = page_pool_create(&p);
 374         if (IS_ERR(rq->page_pool)) {
 375                 int err = PTR_ERR(rq->page_pool);
 376
 377                 rq->page_pool = NULL;
 378                 return err;
 379         }
 380         return 0;
 381 }
 382
 383 static int nsim_init_napi(struct netdevsim *ns)
 384 {
 385         struct net_device *dev = ns->netdev;
 386         struct nsim_rq *rq;
 387         int err, i;
 388
 389         for (i = 0; i < dev->num_rx_queues; i++) {
 390                 rq = &ns->rq[i];
 391
 392                 netif_napi_add(dev, &rq->napi, nsim_poll);
 393         }
 394
 395         for (i = 0; i < dev->num_rx_queues; i++) {
 396                 rq = &ns->rq[i];
 397
 398                 err = nsim_create_page_pool(rq);
 399                 if (err)
 400                         goto err_pp_destroy;
 401         }
 402
 403         return 0;
 404
 405 err_pp_destroy:
 406         while (i--) {
 407                 page_pool_destroy(ns->rq[i].page_pool);
 408                 ns->rq[i].page_pool = NULL;
 409         }
 410
 411         for (i = 0; i < dev->num_rx_queues; i++)
 412                 __netif_napi_del(&ns->rq[i].napi);
 413
 414         return err;
 415 }
 416
 417 static void nsim_enable_napi(struct netdevsim *ns)
 418 {
 419         struct net_device *dev = ns->netdev;
 420         int i;
 421
 422         for (i = 0; i < dev->num_rx_queues; i++) {
 423                 struct nsim_rq *rq = &ns->rq[i];
 424
 425                 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi);
 426                 napi_enable(&rq->napi);
 427         }
 428 }
 429
 430 static int nsim_open(struct net_device *dev)
 431 {
 432         struct netdevsim *ns = netdev_priv(dev);
 433         int err;
 434
 435         err = nsim_init_napi(ns);
 436         if (err)
 437                 return err;
 438
 439         nsim_enable_napi(ns);
 440
 441         return 0;
 442 }
 443
 444 static void nsim_del_napi(struct netdevsim *ns)
 445 {
 446         struct net_device *dev = ns->netdev;
 447         int i;
 448
 449         for (i = 0; i < dev->num_rx_queues; i++) {
 450                 struct nsim_rq *rq = &ns->rq[i];
 451
 452                 napi_disable(&rq->napi);
 453                 __netif_napi_del(&rq->napi);
 454         }
 455         synchronize_net();
 456
 457         for (i = 0; i < dev->num_rx_queues; i++) {
 458                 page_pool_destroy(ns->rq[i].page_pool);
 459                 ns->rq[i].page_pool = NULL;
 460         }
 461 }
 462
 463 static int nsim_stop(struct net_device *dev)
 464 {
 465         struct netdevsim *ns = netdev_priv(dev);
 466         struct netdevsim *peer;
 467
 468         netif_carrier_off(dev);
 469         peer = rtnl_dereference(ns->peer);
 470         if (peer)
 471                 netif_carrier_off(peer->netdev);
 472
 473         nsim_del_napi(ns);
 474
 475         return 0;
 476 }
 477
 478 static const struct net_device_ops nsim_netdev_ops = {
 479         .ndo_start_xmit         = nsim_start_xmit,
 480         .ndo_set_rx_mode        = nsim_set_rx_mode,
 481         .ndo_set_mac_address    = eth_mac_addr,
 482         .ndo_validate_addr      = eth_validate_addr,
 483         .ndo_change_mtu         = nsim_change_mtu,
 484         .ndo_get_stats64        = nsim_get_stats64,
 485         .ndo_set_vf_mac         = nsim_set_vf_mac,
 486         .ndo_set_vf_vlan        = nsim_set_vf_vlan,
 487         .ndo_set_vf_rate        = nsim_set_vf_rate,
 488         .ndo_set_vf_spoofchk    = nsim_set_vf_spoofchk,
 489         .ndo_set_vf_trust       = nsim_set_vf_trust,
 490         .ndo_get_vf_config      = nsim_get_vf_config,
 491         .ndo_set_vf_link_state  = nsim_set_vf_link_state,
 492         .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en,
 493         .ndo_setup_tc           = nsim_setup_tc,
 494         .ndo_set_features       = nsim_set_features,
 495         .ndo_get_iflink         = nsim_get_iflink,
 496         .ndo_bpf                = nsim_bpf,
 497         .ndo_open               = nsim_open,
 498         .ndo_stop               = nsim_stop,
 499 };
 500
 501 static const struct net_device_ops nsim_vf_netdev_ops = {
 502         .ndo_start_xmit         = nsim_start_xmit,
 503         .ndo_set_rx_mode        = nsim_set_rx_mode,
 504         .ndo_set_mac_address    = eth_mac_addr,
 505         .ndo_validate_addr      = eth_validate_addr,
 506         .ndo_change_mtu         = nsim_change_mtu,
 507         .ndo_get_stats64        = nsim_get_stats64,
 508         .ndo_setup_tc           = nsim_setup_tc,
 509         .ndo_set_features       = nsim_set_features,
 510 };
 511
 512 /* We don't have true per-queue stats, yet, so do some random fakery here.
 513  * Only report stuff for queue 0.
 514  */
 515 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx,
 516                                     struct netdev_queue_stats_rx *stats)
 517 {
 518         struct rtnl_link_stats64 rtstats = {};
 519
 520         if (!idx)
 521                 nsim_get_stats64(dev, &rtstats);
 522
 523         stats->packets = rtstats.rx_packets - !!rtstats.rx_packets;
 524         stats->bytes = rtstats.rx_bytes;
 525 }
 526
 527 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx,
 528                                     struct netdev_queue_stats_tx *stats)
 529 {
 530         struct rtnl_link_stats64 rtstats = {};
 531
 532         if (!idx)
 533                 nsim_get_stats64(dev, &rtstats);
 534
 535         stats->packets = rtstats.tx_packets - !!rtstats.tx_packets;
 536         stats->bytes = rtstats.tx_bytes;
 537 }
 538
 539 static void nsim_get_base_stats(struct net_device *dev,
 540                                 struct netdev_queue_stats_rx *rx,
 541                                 struct netdev_queue_stats_tx *tx)
 542 {
 543         struct rtnl_link_stats64 rtstats = {};
 544
 545         nsim_get_stats64(dev, &rtstats);
 546
 547         rx->packets = !!rtstats.rx_packets;
 548         rx->bytes = 0;
 549         tx->packets = !!rtstats.tx_packets;
 550         tx->bytes = 0;
 551 }
 552
 553 static const struct netdev_stat_ops nsim_stat_ops = {
 554         .get_queue_stats_tx     = nsim_get_queue_stats_tx,
 555         .get_queue_stats_rx     = nsim_get_queue_stats_rx,
 556         .get_base_stats         = nsim_get_base_stats,
 557 };
 558
 559 static ssize_t
 560 nsim_pp_hold_read(struct file *file, char __user *data,
 561                   size_t count, loff_t *ppos)
 562 {
 563         struct netdevsim *ns = file->private_data;
 564         char buf[3] = "n\n";
 565
 566         if (ns->page)
 567                 buf[0] = 'y';
 568
 569         return simple_read_from_buffer(data, count, ppos, buf, 2);
 570 }
 571
 572 static ssize_t
 573 nsim_pp_hold_write(struct file *file, const char __user *data,
 574                    size_t count, loff_t *ppos)
 575 {
 576         struct netdevsim *ns = file->private_data;
 577         ssize_t ret;
 578         bool val;
 579
 580         ret = kstrtobool_from_user(data, count, &val);
 581         if (ret)
 582                 return ret;
 583
 584         rtnl_lock();
 585         ret = count;
 586         if (val == !!ns->page)
 587                 goto exit;
 588
 589         if (!netif_running(ns->netdev) && val) {
 590                 ret = -ENETDOWN;
 591         } else if (val) {
 592                 ns->page = page_pool_dev_alloc_pages(ns->rq[0].page_pool);
 593                 if (!ns->page)
 594                         ret = -ENOMEM;
 595         } else {
 596                 page_pool_put_full_page(ns->page->pp, ns->page, false);
 597                 ns->page = NULL;
 598         }
 599         rtnl_unlock();
 600
 601 exit:
 602         return count;
 603 }
 604
 605 static const struct file_operations nsim_pp_hold_fops = {
 606         .open = simple_open,
 607         .read = nsim_pp_hold_read,
 608         .write = nsim_pp_hold_write,
 609         .llseek = generic_file_llseek,
 610         .owner = THIS_MODULE,
 611 };
 612
 613 static void nsim_setup(struct net_device *dev)
 614 {
 615         ether_setup(dev);
 616         eth_hw_addr_random(dev);
 617
 618         dev->tx_queue_len = 0;
 619         dev->flags &= ~IFF_MULTICAST;
 620         dev->priv_flags |= IFF_LIVE_ADDR_CHANGE |
 621                            IFF_NO_QUEUE;
 622         dev->features |= NETIF_F_HIGHDMA |
 623                          NETIF_F_SG |
 624                          NETIF_F_FRAGLIST |
 625                          NETIF_F_HW_CSUM |
 626                          NETIF_F_TSO;
 627         dev->hw_features |= NETIF_F_HW_TC;
 628         dev->max_mtu = ETH_MAX_MTU;
 629         dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD;
 630 }
 631
 632 static int nsim_queue_init(struct netdevsim *ns)
 633 {
 634         struct net_device *dev = ns->netdev;
 635         int i;
 636
 637         ns->rq = kvcalloc(dev->num_rx_queues, sizeof(*ns->rq),
 638                           GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
 639         if (!ns->rq)
 640                 return -ENOMEM;
 641
 642         for (i = 0; i < dev->num_rx_queues; i++)
 643                 skb_queue_head_init(&ns->rq[i].skb_queue);
 644
 645         return 0;
 646 }
 647
 648 static void nsim_queue_free(struct netdevsim *ns)
 649 {
 650         struct net_device *dev = ns->netdev;
 651         int i;
 652
 653         for (i = 0; i < dev->num_rx_queues; i++)
 654                 skb_queue_purge_reason(&ns->rq[i].skb_queue,
 655                                        SKB_DROP_REASON_QUEUE_PURGE);
 656
 657         kvfree(ns->rq);
 658         ns->rq = NULL;
 659 }
 660
 661 static int nsim_init_netdevsim(struct netdevsim *ns)
 662 {
 663         struct mock_phc *phc;
 664         int err;
 665
 666         phc = mock_phc_create(&ns->nsim_bus_dev->dev);
 667         if (IS_ERR(phc))
 668                 return PTR_ERR(phc);
 669
 670         ns->phc = phc;
 671         ns->netdev->netdev_ops = &nsim_netdev_ops;
 672         ns->netdev->stat_ops = &nsim_stat_ops;
 673
 674         err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev);
 675         if (err)
 676                 goto err_phc_destroy;
 677
 678         rtnl_lock();
 679         err = nsim_queue_init(ns);
 680         if (err)
 681                 goto err_utn_destroy;
 682
 683         err = nsim_bpf_init(ns);
 684         if (err)
 685                 goto err_rq_destroy;
 686
 687         nsim_macsec_init(ns);
 688         nsim_ipsec_init(ns);
 689
 690         err = register_netdevice(ns->netdev);
 691         if (err)
 692                 goto err_ipsec_teardown;
 693         rtnl_unlock();
 694         return 0;
 695
 696 err_ipsec_teardown:
 697         nsim_ipsec_teardown(ns);
 698         nsim_macsec_teardown(ns);
 699         nsim_bpf_uninit(ns);
 700 err_rq_destroy:
 701         nsim_queue_free(ns);
 702 err_utn_destroy:
 703         rtnl_unlock();
 704         nsim_udp_tunnels_info_destroy(ns->netdev);
 705 err_phc_destroy:
 706         mock_phc_destroy(ns->phc);
 707         return err;
 708 }
 709
 710 static int nsim_init_netdevsim_vf(struct netdevsim *ns)
 711 {
 712         int err;
 713
 714         ns->netdev->netdev_ops = &nsim_vf_netdev_ops;
 715         rtnl_lock();
 716         err = register_netdevice(ns->netdev);
 717         rtnl_unlock();
 718         return err;
 719 }
 720
 721 static void nsim_exit_netdevsim(struct netdevsim *ns)
 722 {
 723         nsim_udp_tunnels_info_destroy(ns->netdev);
 724         mock_phc_destroy(ns->phc);
 725 }
 726
 727 struct netdevsim *
 728 nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port)
 729 {
 730         struct net_device *dev;
 731         struct netdevsim *ns;
 732         int err;
 733
 734         dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup,
 735                               nsim_dev->nsim_bus_dev->num_queues);
 736         if (!dev)
 737                 return ERR_PTR(-ENOMEM);
 738
 739         dev_net_set(dev, nsim_dev_net(nsim_dev));
 740         ns = netdev_priv(dev);
 741         ns->netdev = dev;
 742         u64_stats_init(&ns->syncp);
 743         ns->nsim_dev = nsim_dev;
 744         ns->nsim_dev_port = nsim_dev_port;
 745         ns->nsim_bus_dev = nsim_dev->nsim_bus_dev;
 746         SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev);
 747         SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port);
 748         nsim_ethtool_init(ns);
 749         if (nsim_dev_port_is_pf(nsim_dev_port))
 750                 err = nsim_init_netdevsim(ns);
 751         else
 752                 err = nsim_init_netdevsim_vf(ns);
 753         if (err)
 754                 goto err_free_netdev;
 755
 756         ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir,
 757                                          ns, &nsim_pp_hold_fops);
 758
 759         return ns;
 760
 761 err_free_netdev:
 762         free_netdev(dev);
 763         return ERR_PTR(err);
 764 }
 765
 766 void nsim_destroy(struct netdevsim *ns)
 767 {
 768         struct net_device *dev = ns->netdev;
 769         struct netdevsim *peer;
 770
 771         debugfs_remove(ns->pp_dfs);
 772
 773         rtnl_lock();
 774         peer = rtnl_dereference(ns->peer);
 775         if (peer)
 776                 RCU_INIT_POINTER(peer->peer, NULL);
 777         RCU_INIT_POINTER(ns->peer, NULL);
 778         unregister_netdevice(dev);
 779         if (nsim_dev_port_is_pf(ns->nsim_dev_port)) {
 780                 nsim_macsec_teardown(ns);
 781                 nsim_ipsec_teardown(ns);
 782                 nsim_bpf_uninit(ns);
 783                 nsim_queue_free(ns);
 784         }
 785         rtnl_unlock();
 786         if (nsim_dev_port_is_pf(ns->nsim_dev_port))
 787                 nsim_exit_netdevsim(ns);
 788
 789         /* Put this intentionally late to exercise the orphaning path */
 790         if (ns->page) {
 791                 page_pool_put_full_page(ns->page->pp, ns->page, false);
 792                 ns->page = NULL;
 793         }
 794
 795         free_netdev(dev);
 796 }
 797
 798 bool netdev_is_nsim(struct net_device *dev)
 799 {
 800         return dev->netdev_ops == &nsim_netdev_ops;
 801 }
 802
 803 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
 804                          struct netlink_ext_ack *extack)
 805 {
 806         NL_SET_ERR_MSG_MOD(extack,
 807                            "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device");
 808         return -EOPNOTSUPP;
 809 }
 810
 811 static struct rtnl_link_ops nsim_link_ops __read_mostly = {
 812         .kind           = DRV_NAME,
 813         .validate       = nsim_validate,
 814 };
 815
 816 static int __init nsim_module_init(void)
 817 {
 818         int err;
 819
 820         err = nsim_dev_init();
 821         if (err)
 822                 return err;
 823
 824         err = nsim_bus_init();
 825         if (err)
 826                 goto err_dev_exit;
 827
 828         err = rtnl_link_register(&nsim_link_ops);
 829         if (err)
 830                 goto err_bus_exit;
 831
 832         return 0;
 833
 834 err_bus_exit:
 835         nsim_bus_exit();
 836 err_dev_exit:
 837         nsim_dev_exit();
 838         return err;
 839 }
 840
 841 static void __exit nsim_module_exit(void)
 842 {
 843         rtnl_link_unregister(&nsim_link_ops);
 844         nsim_bus_exit();
 845         nsim_dev_exit();
 846 }
 847
 848 module_init(nsim_module_init);
 849 module_exit(nsim_module_exit);
 850 MODULE_LICENSE("GPL");
 851 MODULE_DESCRIPTION("Simulated networking device for testing");
 852 MODULE_ALIAS_RTNL_LINK(DRV_NAME);