drivers/net/netdevsim/netdev.c

   1 /*
   2  * Copyright (C) 2017 Netronome Systems, Inc.
   3  *
   4  * This software is licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree.
   7  *
   8  * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
   9  * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
  10  * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  11  * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
  12  * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
  13  * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
  14  */
  15
  16 #include <linux/debugfs.h>
  17 #include <linux/etherdevice.h>
  18 #include <linux/ethtool_netlink.h>
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/netdevice.h>
  22 #include <linux/slab.h>
  23 #include <net/netdev_queues.h>
  24 #include <net/netdev_rx_queue.h>
  25 #include <net/page_pool/helpers.h>
  26 #include <net/netlink.h>
  27 #include <net/net_shaper.h>
  28 #include <net/pkt_cls.h>
  29 #include <net/rtnetlink.h>
  30 #include <net/udp_tunnel.h>
  31
  32 #include "netdevsim.h"
  33
  34 MODULE_IMPORT_NS("NETDEV_INTERNAL");
  35
  36 #define NSIM_RING_SIZE          256
  37
  38 static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb)
  39 {
  40         if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) {
  41                 dev_kfree_skb_any(skb);
  42                 return NET_RX_DROP;
  43         }
  44
  45         skb_queue_tail(&rq->skb_queue, skb);
  46         return NET_RX_SUCCESS;
  47 }
  48
  49 static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb,
  50                             struct nsim_rq *rq)
  51 {
  52         return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb);
  53 }
  54
  55 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
  56 {
  57         struct netdevsim *ns = netdev_priv(dev);
  58         struct net_device *peer_dev;
  59         unsigned int len = skb->len;
  60         struct netdevsim *peer_ns;
  61         struct netdev_config *cfg;
  62         struct nsim_rq *rq;
  63         int rxq;
  64
  65         rcu_read_lock();
  66         if (!nsim_ipsec_tx(ns, skb))
  67                 goto out_drop_free;
  68
  69         peer_ns = rcu_dereference(ns->peer);
  70         if (!peer_ns)
  71                 goto out_drop_free;
  72
  73         peer_dev = peer_ns->netdev;
  74         rxq = skb_get_queue_mapping(skb);
  75         if (rxq >= peer_dev->num_rx_queues)
  76                 rxq = rxq % peer_dev->num_rx_queues;
  77         rq = peer_ns->rq[rxq];
  78
  79         cfg = peer_dev->cfg;
  80         if (skb_is_nonlinear(skb) &&
  81             (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED ||
  82              (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
  83               cfg->hds_thresh > len)))
  84                 skb_linearize(skb);
  85
  86         skb_tx_timestamp(skb);
  87         if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP))
  88                 goto out_drop_cnt;
  89
  90         napi_schedule(&rq->napi);
  91
  92         rcu_read_unlock();
  93         u64_stats_update_begin(&ns->syncp);
  94         ns->tx_packets++;
  95         ns->tx_bytes += len;
  96         u64_stats_update_end(&ns->syncp);
  97         return NETDEV_TX_OK;
  98
  99 out_drop_free:
 100         dev_kfree_skb(skb);
 101 out_drop_cnt:
 102         rcu_read_unlock();
 103         u64_stats_update_begin(&ns->syncp);
 104         ns->tx_dropped++;
 105         u64_stats_update_end(&ns->syncp);
 106         return NETDEV_TX_OK;
 107 }
 108
 109 static void nsim_set_rx_mode(struct net_device *dev)
 110 {
 111 }
 112
 113 static int nsim_change_mtu(struct net_device *dev, int new_mtu)
 114 {
 115         struct netdevsim *ns = netdev_priv(dev);
 116
 117         if (ns->xdp.prog && new_mtu > NSIM_XDP_MAX_MTU)
 118                 return -EBUSY;
 119
 120         WRITE_ONCE(dev->mtu, new_mtu);
 121
 122         return 0;
 123 }
 124
 125 static void
 126 nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 127 {
 128         struct netdevsim *ns = netdev_priv(dev);
 129         unsigned int start;
 130
 131         do {
 132                 start = u64_stats_fetch_begin(&ns->syncp);
 133                 stats->tx_bytes = ns->tx_bytes;
 134                 stats->tx_packets = ns->tx_packets;
 135                 stats->tx_dropped = ns->tx_dropped;
 136         } while (u64_stats_fetch_retry(&ns->syncp, start));
 137 }
 138
 139 static int
 140 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
 141 {
 142         return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);
 143 }
 144
 145 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
 146 {
 147         struct netdevsim *ns = netdev_priv(dev);
 148         struct nsim_dev *nsim_dev = ns->nsim_dev;
 149
 150         /* Only refuse multicast addresses, zero address can mean unset/any. */
 151         if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac))
 152                 return -EINVAL;
 153         memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN);
 154
 155         return 0;
 156 }
 157
 158 static int nsim_set_vf_vlan(struct net_device *dev, int vf,
 159                             u16 vlan, u8 qos, __be16 vlan_proto)
 160 {
 161         struct netdevsim *ns = netdev_priv(dev);
 162         struct nsim_dev *nsim_dev = ns->nsim_dev;
 163
 164         if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7)
 165                 return -EINVAL;
 166
 167         nsim_dev->vfconfigs[vf].vlan = vlan;
 168         nsim_dev->vfconfigs[vf].qos = qos;
 169         nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto;
 170
 171         return 0;
 172 }
 173
 174 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max)
 175 {
 176         struct netdevsim *ns = netdev_priv(dev);
 177         struct nsim_dev *nsim_dev = ns->nsim_dev;
 178
 179         if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) {
 180                 pr_err("Not supported in switchdev mode. Please use devlink API.\n");
 181                 return -EOPNOTSUPP;
 182         }
 183
 184         if (vf >= nsim_dev_get_vfs(nsim_dev))
 185                 return -EINVAL;
 186
 187         nsim_dev->vfconfigs[vf].min_tx_rate = min;
 188         nsim_dev->vfconfigs[vf].max_tx_rate = max;
 189
 190         return 0;
 191 }
 192
 193 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
 194 {
 195         struct netdevsim *ns = netdev_priv(dev);
 196         struct nsim_dev *nsim_dev = ns->nsim_dev;
 197
 198         if (vf >= nsim_dev_get_vfs(nsim_dev))
 199                 return -EINVAL;
 200         nsim_dev->vfconfigs[vf].spoofchk_enabled = val;
 201
 202         return 0;
 203 }
 204
 205 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
 206 {
 207         struct netdevsim *ns = netdev_priv(dev);
 208         struct nsim_dev *nsim_dev = ns->nsim_dev;
 209
 210         if (vf >= nsim_dev_get_vfs(nsim_dev))
 211                 return -EINVAL;
 212         nsim_dev->vfconfigs[vf].rss_query_enabled = val;
 213
 214         return 0;
 215 }
 216
 217 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val)
 218 {
 219         struct netdevsim *ns = netdev_priv(dev);
 220         struct nsim_dev *nsim_dev = ns->nsim_dev;
 221
 222         if (vf >= nsim_dev_get_vfs(nsim_dev))
 223                 return -EINVAL;
 224         nsim_dev->vfconfigs[vf].trusted = val;
 225
 226         return 0;
 227 }
 228
 229 static int
 230 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi)
 231 {
 232         struct netdevsim *ns = netdev_priv(dev);
 233         struct nsim_dev *nsim_dev = ns->nsim_dev;
 234
 235         if (vf >= nsim_dev_get_vfs(nsim_dev))
 236                 return -EINVAL;
 237
 238         ivi->vf = vf;
 239         ivi->linkstate = nsim_dev->vfconfigs[vf].link_state;
 240         ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate;
 241         ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate;
 242         ivi->vlan = nsim_dev->vfconfigs[vf].vlan;
 243         ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto;
 244         ivi->qos = nsim_dev->vfconfigs[vf].qos;
 245         memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN);
 246         ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled;
 247         ivi->trusted = nsim_dev->vfconfigs[vf].trusted;
 248         ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled;
 249
 250         return 0;
 251 }
 252
 253 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
 254 {
 255         struct netdevsim *ns = netdev_priv(dev);
 256         struct nsim_dev *nsim_dev = ns->nsim_dev;
 257
 258         if (vf >= nsim_dev_get_vfs(nsim_dev))
 259                 return -EINVAL;
 260
 261         switch (state) {
 262         case IFLA_VF_LINK_STATE_AUTO:
 263         case IFLA_VF_LINK_STATE_ENABLE:
 264         case IFLA_VF_LINK_STATE_DISABLE:
 265                 break;
 266         default:
 267                 return -EINVAL;
 268         }
 269
 270         nsim_dev->vfconfigs[vf].link_state = state;
 271
 272         return 0;
 273 }
 274
 275 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats)
 276 {
 277         stats->window_drops = 0;
 278         stats->tx_overruns = 0;
 279 }
 280
 281 static int nsim_setup_tc_taprio(struct net_device *dev,
 282                                 struct tc_taprio_qopt_offload *offload)
 283 {
 284         int err = 0;
 285
 286         switch (offload->cmd) {
 287         case TAPRIO_CMD_REPLACE:
 288         case TAPRIO_CMD_DESTROY:
 289                 break;
 290         case TAPRIO_CMD_STATS:
 291                 nsim_taprio_stats(&offload->stats);
 292                 break;
 293         default:
 294                 err = -EOPNOTSUPP;
 295         }
 296
 297         return err;
 298 }
 299
 300 static LIST_HEAD(nsim_block_cb_list);
 301
 302 static int
 303 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
 304 {
 305         struct netdevsim *ns = netdev_priv(dev);
 306
 307         switch (type) {
 308         case TC_SETUP_QDISC_TAPRIO:
 309                 return nsim_setup_tc_taprio(dev, type_data);
 310         case TC_SETUP_BLOCK:
 311                 return flow_block_cb_setup_simple(type_data,
 312                                                   &nsim_block_cb_list,
 313                                                   nsim_setup_tc_block_cb,
 314                                                   ns, ns, true);
 315         default:
 316                 return -EOPNOTSUPP;
 317         }
 318 }
 319
 320 static int
 321 nsim_set_features(struct net_device *dev, netdev_features_t features)
 322 {
 323         struct netdevsim *ns = netdev_priv(dev);
 324
 325         if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))
 326                 return nsim_bpf_disable_tc(ns);
 327
 328         return 0;
 329 }
 330
 331 static int nsim_get_iflink(const struct net_device *dev)
 332 {
 333         struct netdevsim *nsim, *peer;
 334         int iflink;
 335
 336         nsim = netdev_priv(dev);
 337
 338         rcu_read_lock();
 339         peer = rcu_dereference(nsim->peer);
 340         iflink = peer ? READ_ONCE(peer->netdev->ifindex) :
 341                         READ_ONCE(dev->ifindex);
 342         rcu_read_unlock();
 343
 344         return iflink;
 345 }
 346
 347 static int nsim_rcv(struct nsim_rq *rq, int budget)
 348 {
 349         struct sk_buff *skb;
 350         int i;
 351
 352         for (i = 0; i < budget; i++) {
 353                 if (skb_queue_empty(&rq->skb_queue))
 354                         break;
 355
 356                 skb = skb_dequeue(&rq->skb_queue);
 357                 netif_receive_skb(skb);
 358         }
 359
 360         return i;
 361 }
 362
 363 static int nsim_poll(struct napi_struct *napi, int budget)
 364 {
 365         struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi);
 366         int done;
 367
 368         done = nsim_rcv(rq, budget);
 369         napi_complete(napi);
 370
 371         return done;
 372 }
 373
 374 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi)
 375 {
 376         struct page_pool_params params = {
 377                 .order = 0,
 378                 .pool_size = NSIM_RING_SIZE,
 379                 .nid = NUMA_NO_NODE,
 380                 .dev = &napi->dev->dev,
 381                 .napi = napi,
 382                 .dma_dir = DMA_BIDIRECTIONAL,
 383                 .netdev = napi->dev,
 384         };
 385         struct page_pool *pool;
 386
 387         pool = page_pool_create(&params);
 388         if (IS_ERR(pool))
 389                 return PTR_ERR(pool);
 390
 391         *p = pool;
 392         return 0;
 393 }
 394
 395 static int nsim_init_napi(struct netdevsim *ns)
 396 {
 397         struct net_device *dev = ns->netdev;
 398         struct nsim_rq *rq;
 399         int err, i;
 400
 401         for (i = 0; i < dev->num_rx_queues; i++) {
 402                 rq = ns->rq[i];
 403
 404                 netif_napi_add_config(dev, &rq->napi, nsim_poll, i);
 405         }
 406
 407         for (i = 0; i < dev->num_rx_queues; i++) {
 408                 rq = ns->rq[i];
 409
 410                 err = nsim_create_page_pool(&rq->page_pool, &rq->napi);
 411                 if (err)
 412                         goto err_pp_destroy;
 413         }
 414
 415         return 0;
 416
 417 err_pp_destroy:
 418         while (i--) {
 419                 page_pool_destroy(ns->rq[i]->page_pool);
 420                 ns->rq[i]->page_pool = NULL;
 421         }
 422
 423         for (i = 0; i < dev->num_rx_queues; i++)
 424                 __netif_napi_del(&ns->rq[i]->napi);
 425
 426         return err;
 427 }
 428
 429 static void nsim_enable_napi(struct netdevsim *ns)
 430 {
 431         struct net_device *dev = ns->netdev;
 432         int i;
 433
 434         for (i = 0; i < dev->num_rx_queues; i++) {
 435                 struct nsim_rq *rq = ns->rq[i];
 436
 437                 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi);
 438                 napi_enable(&rq->napi);
 439         }
 440 }
 441
 442 static int nsim_open(struct net_device *dev)
 443 {
 444         struct netdevsim *ns = netdev_priv(dev);
 445         int err;
 446
 447         err = nsim_init_napi(ns);
 448         if (err)
 449                 return err;
 450
 451         nsim_enable_napi(ns);
 452
 453         return 0;
 454 }
 455
 456 static void nsim_del_napi(struct netdevsim *ns)
 457 {
 458         struct net_device *dev = ns->netdev;
 459         int i;
 460
 461         for (i = 0; i < dev->num_rx_queues; i++) {
 462                 struct nsim_rq *rq = ns->rq[i];
 463
 464                 napi_disable(&rq->napi);
 465                 __netif_napi_del(&rq->napi);
 466         }
 467         synchronize_net();
 468
 469         for (i = 0; i < dev->num_rx_queues; i++) {
 470                 page_pool_destroy(ns->rq[i]->page_pool);
 471                 ns->rq[i]->page_pool = NULL;
 472         }
 473 }
 474
 475 static int nsim_stop(struct net_device *dev)
 476 {
 477         struct netdevsim *ns = netdev_priv(dev);
 478         struct netdevsim *peer;
 479
 480         netif_carrier_off(dev);
 481         peer = rtnl_dereference(ns->peer);
 482         if (peer)
 483                 netif_carrier_off(peer->netdev);
 484
 485         nsim_del_napi(ns);
 486
 487         return 0;
 488 }
 489
 490 static int nsim_shaper_set(struct net_shaper_binding *binding,
 491                            const struct net_shaper *shaper,
 492                            struct netlink_ext_ack *extack)
 493 {
 494         return 0;
 495 }
 496
 497 static int nsim_shaper_del(struct net_shaper_binding *binding,
 498                            const struct net_shaper_handle *handle,
 499                            struct netlink_ext_ack *extack)
 500 {
 501         return 0;
 502 }
 503
 504 static int nsim_shaper_group(struct net_shaper_binding *binding,
 505                              int leaves_count,
 506                              const struct net_shaper *leaves,
 507                              const struct net_shaper *root,
 508                              struct netlink_ext_ack *extack)
 509 {
 510         return 0;
 511 }
 512
 513 static void nsim_shaper_cap(struct net_shaper_binding *binding,
 514                             enum net_shaper_scope scope,
 515                             unsigned long *flags)
 516 {
 517         *flags = ULONG_MAX;
 518 }
 519
 520 static const struct net_shaper_ops nsim_shaper_ops = {
 521         .set                    = nsim_shaper_set,
 522         .delete                 = nsim_shaper_del,
 523         .group                  = nsim_shaper_group,
 524         .capabilities           = nsim_shaper_cap,
 525 };
 526
 527 static const struct net_device_ops nsim_netdev_ops = {
 528         .ndo_start_xmit         = nsim_start_xmit,
 529         .ndo_set_rx_mode        = nsim_set_rx_mode,
 530         .ndo_set_mac_address    = eth_mac_addr,
 531         .ndo_validate_addr      = eth_validate_addr,
 532         .ndo_change_mtu         = nsim_change_mtu,
 533         .ndo_get_stats64        = nsim_get_stats64,
 534         .ndo_set_vf_mac         = nsim_set_vf_mac,
 535         .ndo_set_vf_vlan        = nsim_set_vf_vlan,
 536         .ndo_set_vf_rate        = nsim_set_vf_rate,
 537         .ndo_set_vf_spoofchk    = nsim_set_vf_spoofchk,
 538         .ndo_set_vf_trust       = nsim_set_vf_trust,
 539         .ndo_get_vf_config      = nsim_get_vf_config,
 540         .ndo_set_vf_link_state  = nsim_set_vf_link_state,
 541         .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en,
 542         .ndo_setup_tc           = nsim_setup_tc,
 543         .ndo_set_features       = nsim_set_features,
 544         .ndo_get_iflink         = nsim_get_iflink,
 545         .ndo_bpf                = nsim_bpf,
 546         .ndo_open               = nsim_open,
 547         .ndo_stop               = nsim_stop,
 548         .net_shaper_ops         = &nsim_shaper_ops,
 549 };
 550
 551 static const struct net_device_ops nsim_vf_netdev_ops = {
 552         .ndo_start_xmit         = nsim_start_xmit,
 553         .ndo_set_rx_mode        = nsim_set_rx_mode,
 554         .ndo_set_mac_address    = eth_mac_addr,
 555         .ndo_validate_addr      = eth_validate_addr,
 556         .ndo_change_mtu         = nsim_change_mtu,
 557         .ndo_get_stats64        = nsim_get_stats64,
 558         .ndo_setup_tc           = nsim_setup_tc,
 559         .ndo_set_features       = nsim_set_features,
 560 };
 561
 562 /* We don't have true per-queue stats, yet, so do some random fakery here.
 563  * Only report stuff for queue 0.
 564  */
 565 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx,
 566                                     struct netdev_queue_stats_rx *stats)
 567 {
 568         struct rtnl_link_stats64 rtstats = {};
 569
 570         if (!idx)
 571                 nsim_get_stats64(dev, &rtstats);
 572
 573         stats->packets = rtstats.rx_packets - !!rtstats.rx_packets;
 574         stats->bytes = rtstats.rx_bytes;
 575 }
 576
 577 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx,
 578                                     struct netdev_queue_stats_tx *stats)
 579 {
 580         struct rtnl_link_stats64 rtstats = {};
 581
 582         if (!idx)
 583                 nsim_get_stats64(dev, &rtstats);
 584
 585         stats->packets = rtstats.tx_packets - !!rtstats.tx_packets;
 586         stats->bytes = rtstats.tx_bytes;
 587 }
 588
 589 static void nsim_get_base_stats(struct net_device *dev,
 590                                 struct netdev_queue_stats_rx *rx,
 591                                 struct netdev_queue_stats_tx *tx)
 592 {
 593         struct rtnl_link_stats64 rtstats = {};
 594
 595         nsim_get_stats64(dev, &rtstats);
 596
 597         rx->packets = !!rtstats.rx_packets;
 598         rx->bytes = 0;
 599         tx->packets = !!rtstats.tx_packets;
 600         tx->bytes = 0;
 601 }
 602
 603 static const struct netdev_stat_ops nsim_stat_ops = {
 604         .get_queue_stats_tx     = nsim_get_queue_stats_tx,
 605         .get_queue_stats_rx     = nsim_get_queue_stats_rx,
 606         .get_base_stats         = nsim_get_base_stats,
 607 };
 608
 609 static struct nsim_rq *nsim_queue_alloc(void)
 610 {
 611         struct nsim_rq *rq;
 612
 613         rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT);
 614         if (!rq)
 615                 return NULL;
 616
 617         skb_queue_head_init(&rq->skb_queue);
 618         return rq;
 619 }
 620
 621 static void nsim_queue_free(struct nsim_rq *rq)
 622 {
 623         skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE);
 624         kfree(rq);
 625 }
 626
 627 /* Queue reset mode is controlled by ns->rq_reset_mode.
 628  * - normal - new NAPI new pool (old NAPI enabled when new added)
 629  * - mode 1 - allocate new pool (NAPI is only disabled / enabled)
 630  * - mode 2 - new NAPI new pool (old NAPI removed before new added)
 631  * - mode 3 - new NAPI new pool (old NAPI disabled when new added)
 632  */
 633 struct nsim_queue_mem {
 634         struct nsim_rq *rq;
 635         struct page_pool *pp;
 636 };
 637
 638 static int
 639 nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx)
 640 {
 641         struct nsim_queue_mem *qmem = per_queue_mem;
 642         struct netdevsim *ns = netdev_priv(dev);
 643         int err;
 644
 645         if (ns->rq_reset_mode > 3)
 646                 return -EINVAL;
 647
 648         if (ns->rq_reset_mode == 1)
 649                 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi);
 650
 651         qmem->rq = nsim_queue_alloc();
 652         if (!qmem->rq)
 653                 return -ENOMEM;
 654
 655         err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi);
 656         if (err)
 657                 goto err_free;
 658
 659         if (!ns->rq_reset_mode)
 660                 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx);
 661
 662         return 0;
 663
 664 err_free:
 665         nsim_queue_free(qmem->rq);
 666         return err;
 667 }
 668
 669 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem)
 670 {
 671         struct nsim_queue_mem *qmem = per_queue_mem;
 672         struct netdevsim *ns = netdev_priv(dev);
 673
 674         page_pool_destroy(qmem->pp);
 675         if (qmem->rq) {
 676                 if (!ns->rq_reset_mode)
 677                         netif_napi_del(&qmem->rq->napi);
 678                 page_pool_destroy(qmem->rq->page_pool);
 679                 nsim_queue_free(qmem->rq);
 680         }
 681 }
 682
 683 static int
 684 nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx)
 685 {
 686         struct nsim_queue_mem *qmem = per_queue_mem;
 687         struct netdevsim *ns = netdev_priv(dev);
 688
 689         if (ns->rq_reset_mode == 1) {
 690                 ns->rq[idx]->page_pool = qmem->pp;
 691                 napi_enable(&ns->rq[idx]->napi);
 692                 return 0;
 693         }
 694
 695         /* netif_napi_add()/_del() should normally be called from alloc/free,
 696          * here we want to test various call orders.
 697          */
 698         if (ns->rq_reset_mode == 2) {
 699                 netif_napi_del(&ns->rq[idx]->napi);
 700                 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx);
 701         } else if (ns->rq_reset_mode == 3) {
 702                 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx);
 703                 netif_napi_del(&ns->rq[idx]->napi);
 704         }
 705
 706         ns->rq[idx] = qmem->rq;
 707         napi_enable(&ns->rq[idx]->napi);
 708
 709         return 0;
 710 }
 711
 712 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx)
 713 {
 714         struct nsim_queue_mem *qmem = per_queue_mem;
 715         struct netdevsim *ns = netdev_priv(dev);
 716
 717         napi_disable(&ns->rq[idx]->napi);
 718
 719         if (ns->rq_reset_mode == 1) {
 720                 qmem->pp = ns->rq[idx]->page_pool;
 721                 page_pool_disable_direct_recycling(qmem->pp);
 722         } else {
 723                 qmem->rq = ns->rq[idx];
 724         }
 725
 726         return 0;
 727 }
 728
 729 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = {
 730         .ndo_queue_mem_size     = sizeof(struct nsim_queue_mem),
 731         .ndo_queue_mem_alloc    = nsim_queue_mem_alloc,
 732         .ndo_queue_mem_free     = nsim_queue_mem_free,
 733         .ndo_queue_start        = nsim_queue_start,
 734         .ndo_queue_stop         = nsim_queue_stop,
 735 };
 736
 737 static ssize_t
 738 nsim_qreset_write(struct file *file, const char __user *data,
 739                   size_t count, loff_t *ppos)
 740 {
 741         struct netdevsim *ns = file->private_data;
 742         unsigned int queue, mode;
 743         char buf[32];
 744         ssize_t ret;
 745
 746         if (count >= sizeof(buf))
 747                 return -EINVAL;
 748         if (copy_from_user(buf, data, count))
 749                 return -EFAULT;
 750         buf[count] = '\0';
 751
 752         ret = sscanf(buf, "%u %u", &queue, &mode);
 753         if (ret != 2)
 754                 return -EINVAL;
 755
 756         rtnl_lock();
 757         if (!netif_running(ns->netdev)) {
 758                 ret = -ENETDOWN;
 759                 goto exit_unlock;
 760         }
 761
 762         if (queue >= ns->netdev->real_num_rx_queues) {
 763                 ret = -EINVAL;
 764                 goto exit_unlock;
 765         }
 766
 767         ns->rq_reset_mode = mode;
 768         ret = netdev_rx_queue_restart(ns->netdev, queue);
 769         ns->rq_reset_mode = 0;
 770         if (ret)
 771                 goto exit_unlock;
 772
 773         ret = count;
 774 exit_unlock:
 775         rtnl_unlock();
 776         return ret;
 777 }
 778
 779 static const struct file_operations nsim_qreset_fops = {
 780         .open = simple_open,
 781         .write = nsim_qreset_write,
 782         .owner = THIS_MODULE,
 783 };
 784
 785 static ssize_t
 786 nsim_pp_hold_read(struct file *file, char __user *data,
 787                   size_t count, loff_t *ppos)
 788 {
 789         struct netdevsim *ns = file->private_data;
 790         char buf[3] = "n\n";
 791
 792         if (ns->page)
 793                 buf[0] = 'y';
 794
 795         return simple_read_from_buffer(data, count, ppos, buf, 2);
 796 }
 797
 798 static ssize_t
 799 nsim_pp_hold_write(struct file *file, const char __user *data,
 800                    size_t count, loff_t *ppos)
 801 {
 802         struct netdevsim *ns = file->private_data;
 803         ssize_t ret;
 804         bool val;
 805
 806         ret = kstrtobool_from_user(data, count, &val);
 807         if (ret)
 808                 return ret;
 809
 810         rtnl_lock();
 811         ret = count;
 812         if (val == !!ns->page)
 813                 goto exit;
 814
 815         if (!netif_running(ns->netdev) && val) {
 816                 ret = -ENETDOWN;
 817         } else if (val) {
 818                 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool);
 819                 if (!ns->page)
 820                         ret = -ENOMEM;
 821         } else {
 822                 page_pool_put_full_page(ns->page->pp, ns->page, false);
 823                 ns->page = NULL;
 824         }
 825
 826 exit:
 827         rtnl_unlock();
 828         return ret;
 829 }
 830
 831 static const struct file_operations nsim_pp_hold_fops = {
 832         .open = simple_open,
 833         .read = nsim_pp_hold_read,
 834         .write = nsim_pp_hold_write,
 835         .llseek = generic_file_llseek,
 836         .owner = THIS_MODULE,
 837 };
 838
 839 static void nsim_setup(struct net_device *dev)
 840 {
 841         ether_setup(dev);
 842         eth_hw_addr_random(dev);
 843
 844         dev->tx_queue_len = 0;
 845         dev->flags &= ~IFF_MULTICAST;
 846         dev->priv_flags |= IFF_LIVE_ADDR_CHANGE |
 847                            IFF_NO_QUEUE;
 848         dev->features |= NETIF_F_HIGHDMA |
 849                          NETIF_F_SG |
 850                          NETIF_F_FRAGLIST |
 851                          NETIF_F_HW_CSUM |
 852                          NETIF_F_TSO;
 853         dev->hw_features |= NETIF_F_HW_TC |
 854                             NETIF_F_SG |
 855                             NETIF_F_FRAGLIST |
 856                             NETIF_F_HW_CSUM |
 857                             NETIF_F_TSO;
 858         dev->max_mtu = ETH_MAX_MTU;
 859         dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD;
 860 }
 861
 862 static int nsim_queue_init(struct netdevsim *ns)
 863 {
 864         struct net_device *dev = ns->netdev;
 865         int i;
 866
 867         ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq),
 868                          GFP_KERNEL_ACCOUNT);
 869         if (!ns->rq)
 870                 return -ENOMEM;
 871
 872         for (i = 0; i < dev->num_rx_queues; i++) {
 873                 ns->rq[i] = nsim_queue_alloc();
 874                 if (!ns->rq[i])
 875                         goto err_free_prev;
 876         }
 877
 878         return 0;
 879
 880 err_free_prev:
 881         while (i--)
 882                 kfree(ns->rq[i]);
 883         kfree(ns->rq);
 884         return -ENOMEM;
 885 }
 886
 887 static void nsim_queue_uninit(struct netdevsim *ns)
 888 {
 889         struct net_device *dev = ns->netdev;
 890         int i;
 891
 892         for (i = 0; i < dev->num_rx_queues; i++)
 893                 nsim_queue_free(ns->rq[i]);
 894
 895         kfree(ns->rq);
 896         ns->rq = NULL;
 897 }
 898
 899 static int nsim_init_netdevsim(struct netdevsim *ns)
 900 {
 901         struct mock_phc *phc;
 902         int err;
 903
 904         phc = mock_phc_create(&ns->nsim_bus_dev->dev);
 905         if (IS_ERR(phc))
 906                 return PTR_ERR(phc);
 907
 908         ns->phc = phc;
 909         ns->netdev->netdev_ops = &nsim_netdev_ops;
 910         ns->netdev->stat_ops = &nsim_stat_ops;
 911         ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops;
 912
 913         err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev);
 914         if (err)
 915                 goto err_phc_destroy;
 916
 917         rtnl_lock();
 918         err = nsim_queue_init(ns);
 919         if (err)
 920                 goto err_utn_destroy;
 921
 922         err = nsim_bpf_init(ns);
 923         if (err)
 924                 goto err_rq_destroy;
 925
 926         nsim_macsec_init(ns);
 927         nsim_ipsec_init(ns);
 928
 929         err = register_netdevice(ns->netdev);
 930         if (err)
 931                 goto err_ipsec_teardown;
 932         rtnl_unlock();
 933         return 0;
 934
 935 err_ipsec_teardown:
 936         nsim_ipsec_teardown(ns);
 937         nsim_macsec_teardown(ns);
 938         nsim_bpf_uninit(ns);
 939 err_rq_destroy:
 940         nsim_queue_uninit(ns);
 941 err_utn_destroy:
 942         rtnl_unlock();
 943         nsim_udp_tunnels_info_destroy(ns->netdev);
 944 err_phc_destroy:
 945         mock_phc_destroy(ns->phc);
 946         return err;
 947 }
 948
 949 static int nsim_init_netdevsim_vf(struct netdevsim *ns)
 950 {
 951         int err;
 952
 953         ns->netdev->netdev_ops = &nsim_vf_netdev_ops;
 954         rtnl_lock();
 955         err = register_netdevice(ns->netdev);
 956         rtnl_unlock();
 957         return err;
 958 }
 959
 960 static void nsim_exit_netdevsim(struct netdevsim *ns)
 961 {
 962         nsim_udp_tunnels_info_destroy(ns->netdev);
 963         mock_phc_destroy(ns->phc);
 964 }
 965
 966 struct netdevsim *
 967 nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port)
 968 {
 969         struct net_device *dev;
 970         struct netdevsim *ns;
 971         int err;
 972
 973         dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup,
 974                               nsim_dev->nsim_bus_dev->num_queues);
 975         if (!dev)
 976                 return ERR_PTR(-ENOMEM);
 977
 978         dev_net_set(dev, nsim_dev_net(nsim_dev));
 979         ns = netdev_priv(dev);
 980         ns->netdev = dev;
 981         u64_stats_init(&ns->syncp);
 982         ns->nsim_dev = nsim_dev;
 983         ns->nsim_dev_port = nsim_dev_port;
 984         ns->nsim_bus_dev = nsim_dev->nsim_bus_dev;
 985         SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev);
 986         SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port);
 987         nsim_ethtool_init(ns);
 988         if (nsim_dev_port_is_pf(nsim_dev_port))
 989                 err = nsim_init_netdevsim(ns);
 990         else
 991                 err = nsim_init_netdevsim_vf(ns);
 992         if (err)
 993                 goto err_free_netdev;
 994
 995         ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir,
 996                                          ns, &nsim_pp_hold_fops);
 997         ns->qr_dfs = debugfs_create_file("queue_reset", 0200,
 998                                          nsim_dev_port->ddir, ns,
 999                                          &nsim_qreset_fops);
1000
1001         return ns;
1002
1003 err_free_netdev:
1004         free_netdev(dev);
1005         return ERR_PTR(err);
1006 }
1007
1008 void nsim_destroy(struct netdevsim *ns)
1009 {
1010         struct net_device *dev = ns->netdev;
1011         struct netdevsim *peer;
1012
1013         debugfs_remove(ns->qr_dfs);
1014         debugfs_remove(ns->pp_dfs);
1015
1016         rtnl_lock();
1017         peer = rtnl_dereference(ns->peer);
1018         if (peer)
1019                 RCU_INIT_POINTER(peer->peer, NULL);
1020         RCU_INIT_POINTER(ns->peer, NULL);
1021         unregister_netdevice(dev);
1022         if (nsim_dev_port_is_pf(ns->nsim_dev_port)) {
1023                 nsim_macsec_teardown(ns);
1024                 nsim_ipsec_teardown(ns);
1025                 nsim_bpf_uninit(ns);
1026                 nsim_queue_uninit(ns);
1027         }
1028         rtnl_unlock();
1029         if (nsim_dev_port_is_pf(ns->nsim_dev_port))
1030                 nsim_exit_netdevsim(ns);
1031
1032         /* Put this intentionally late to exercise the orphaning path */
1033         if (ns->page) {
1034                 page_pool_put_full_page(ns->page->pp, ns->page, false);
1035                 ns->page = NULL;
1036         }
1037
1038         free_netdev(dev);
1039 }
1040
1041 bool netdev_is_nsim(struct net_device *dev)
1042 {
1043         return dev->netdev_ops == &nsim_netdev_ops;
1044 }
1045
1046 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
1047                          struct netlink_ext_ack *extack)
1048 {
1049         NL_SET_ERR_MSG_MOD(extack,
1050                            "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device");
1051         return -EOPNOTSUPP;
1052 }
1053
1054 static struct rtnl_link_ops nsim_link_ops __read_mostly = {
1055         .kind           = DRV_NAME,
1056         .validate       = nsim_validate,
1057 };
1058
1059 static int __init nsim_module_init(void)
1060 {
1061         int err;
1062
1063         err = nsim_dev_init();
1064         if (err)
1065                 return err;
1066
1067         err = nsim_bus_init();
1068         if (err)
1069                 goto err_dev_exit;
1070
1071         err = rtnl_link_register(&nsim_link_ops);
1072         if (err)
1073                 goto err_bus_exit;
1074
1075         return 0;
1076
1077 err_bus_exit:
1078         nsim_bus_exit();
1079 err_dev_exit:
1080         nsim_dev_exit();
1081         return err;
1082 }
1083
1084 static void __exit nsim_module_exit(void)
1085 {
1086         rtnl_link_unregister(&nsim_link_ops);
1087         nsim_bus_exit();
1088         nsim_dev_exit();
1089 }
1090
1091 module_init(nsim_module_init);
1092 module_exit(nsim_module_exit);
1093 MODULE_LICENSE("GPL");
1094 MODULE_DESCRIPTION("Simulated networking device for testing");
1095 MODULE_ALIAS_RTNL_LINK(DRV_NAME);