drivers/net/netkit.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2023 Isovalent */
   3
   4 #include <linux/netdevice.h>
   5 #include <linux/ethtool.h>
   6 #include <linux/etherdevice.h>
   7 #include <linux/filter.h>
   8 #include <linux/netfilter_netdev.h>
   9 #include <linux/bpf_mprog.h>
  10 #include <linux/indirect_call_wrapper.h>
  11
  12 #include <net/netkit.h>
  13 #include <net/dst.h>
  14 #include <net/tcx.h>
  15
  16 #define DRV_NAME "netkit"
  17
  18 struct netkit {
  19         /* Needed in fast-path */
  20         struct net_device __rcu *peer;
  21         struct bpf_mprog_entry __rcu *active;
  22         enum netkit_action policy;
  23         enum netkit_scrub scrub;
  24         struct bpf_mprog_bundle bundle;
  25
  26         /* Needed in slow-path */
  27         enum netkit_mode mode;
  28         bool primary;
  29         u32 headroom;
  30 };
  31
  32 struct netkit_link {
  33         struct bpf_link link;
  34         struct net_device *dev;
  35         u32 location;
  36 };
  37
  38 static __always_inline int
  39 netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
  40            enum netkit_action ret)
  41 {
  42         const struct bpf_mprog_fp *fp;
  43         const struct bpf_prog *prog;
  44
  45         bpf_mprog_foreach_prog(entry, fp, prog) {
  46                 bpf_compute_data_pointers(skb);
  47                 ret = bpf_prog_run(prog, skb);
  48                 if (ret != NETKIT_NEXT)
  49                         break;
  50         }
  51         return ret;
  52 }
  53
  54 static void netkit_xnet(struct sk_buff *skb)
  55 {
  56         skb->priority = 0;
  57         skb->mark = 0;
  58 }
  59
  60 static void netkit_prep_forward(struct sk_buff *skb,
  61                                 bool xnet, bool xnet_scrub)
  62 {
  63         skb_scrub_packet(skb, false);
  64         nf_skip_egress(skb, true);
  65         skb_reset_mac_header(skb);
  66         if (!xnet)
  67                 return;
  68         ipvs_reset(skb);
  69         skb_clear_tstamp(skb);
  70         if (xnet_scrub)
  71                 netkit_xnet(skb);
  72 }
  73
  74 static struct netkit *netkit_priv(const struct net_device *dev)
  75 {
  76         return netdev_priv(dev);
  77 }
  78
  79 static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
  80 {
  81         struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
  82         struct netkit *nk = netkit_priv(dev);
  83         enum netkit_action ret = READ_ONCE(nk->policy);
  84         netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
  85         const struct bpf_mprog_entry *entry;
  86         struct net_device *peer;
  87         int len = skb->len;
  88
  89         bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
  90         rcu_read_lock();
  91         peer = rcu_dereference(nk->peer);
  92         if (unlikely(!peer || !(peer->flags & IFF_UP) ||
  93                      !pskb_may_pull(skb, ETH_HLEN) ||
  94                      skb_orphan_frags(skb, GFP_ATOMIC)))
  95                 goto drop;
  96         netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)),
  97                             nk->scrub);
  98         eth_skb_pkt_type(skb, peer);
  99         skb->dev = peer;
 100         entry = rcu_dereference(nk->active);
 101         if (entry)
 102                 ret = netkit_run(entry, skb, ret);
 103         switch (ret) {
 104         case NETKIT_NEXT:
 105         case NETKIT_PASS:
 106                 eth_skb_pull_mac(skb);
 107                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
 108                 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
 109                         dev_sw_netstats_tx_add(dev, 1, len);
 110                         dev_sw_netstats_rx_add(peer, len);
 111                 } else {
 112                         goto drop_stats;
 113                 }
 114                 break;
 115         case NETKIT_REDIRECT:
 116                 dev_sw_netstats_tx_add(dev, 1, len);
 117                 skb_do_redirect(skb);
 118                 break;
 119         case NETKIT_DROP:
 120         default:
 121 drop:
 122                 kfree_skb(skb);
 123 drop_stats:
 124                 dev_core_stats_tx_dropped_inc(dev);
 125                 ret_dev = NET_XMIT_DROP;
 126                 break;
 127         }
 128         rcu_read_unlock();
 129         bpf_net_ctx_clear(bpf_net_ctx);
 130         return ret_dev;
 131 }
 132
 133 static int netkit_open(struct net_device *dev)
 134 {
 135         struct netkit *nk = netkit_priv(dev);
 136         struct net_device *peer = rtnl_dereference(nk->peer);
 137
 138         if (!peer)
 139                 return -ENOTCONN;
 140         if (peer->flags & IFF_UP) {
 141                 netif_carrier_on(dev);
 142                 netif_carrier_on(peer);
 143         }
 144         return 0;
 145 }
 146
 147 static int netkit_close(struct net_device *dev)
 148 {
 149         struct netkit *nk = netkit_priv(dev);
 150         struct net_device *peer = rtnl_dereference(nk->peer);
 151
 152         netif_carrier_off(dev);
 153         if (peer)
 154                 netif_carrier_off(peer);
 155         return 0;
 156 }
 157
 158 static int netkit_get_iflink(const struct net_device *dev)
 159 {
 160         struct netkit *nk = netkit_priv(dev);
 161         struct net_device *peer;
 162         int iflink = 0;
 163
 164         rcu_read_lock();
 165         peer = rcu_dereference(nk->peer);
 166         if (peer)
 167                 iflink = READ_ONCE(peer->ifindex);
 168         rcu_read_unlock();
 169         return iflink;
 170 }
 171
 172 static void netkit_set_multicast(struct net_device *dev)
 173 {
 174         /* Nothing to do, we receive whatever gets pushed to us! */
 175 }
 176
 177 static int netkit_set_macaddr(struct net_device *dev, void *sa)
 178 {
 179         struct netkit *nk = netkit_priv(dev);
 180
 181         if (nk->mode != NETKIT_L2)
 182                 return -EOPNOTSUPP;
 183
 184         return eth_mac_addr(dev, sa);
 185 }
 186
 187 static void netkit_set_headroom(struct net_device *dev, int headroom)
 188 {
 189         struct netkit *nk = netkit_priv(dev), *nk2;
 190         struct net_device *peer;
 191
 192         if (headroom < 0)
 193                 headroom = NET_SKB_PAD;
 194
 195         rcu_read_lock();
 196         peer = rcu_dereference(nk->peer);
 197         if (unlikely(!peer))
 198                 goto out;
 199
 200         nk2 = netkit_priv(peer);
 201         nk->headroom = headroom;
 202         headroom = max(nk->headroom, nk2->headroom);
 203
 204         peer->needed_headroom = headroom;
 205         dev->needed_headroom = headroom;
 206 out:
 207         rcu_read_unlock();
 208 }
 209
 210 INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
 211 {
 212         return rcu_dereference(netkit_priv(dev)->peer);
 213 }
 214
 215 static void netkit_get_stats(struct net_device *dev,
 216                              struct rtnl_link_stats64 *stats)
 217 {
 218         dev_fetch_sw_netstats(stats, dev->tstats);
 219         stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
 220 }
 221
 222 static void netkit_uninit(struct net_device *dev);
 223
 224 static const struct net_device_ops netkit_netdev_ops = {
 225         .ndo_open               = netkit_open,
 226         .ndo_stop               = netkit_close,
 227         .ndo_start_xmit         = netkit_xmit,
 228         .ndo_set_rx_mode        = netkit_set_multicast,
 229         .ndo_set_rx_headroom    = netkit_set_headroom,
 230         .ndo_set_mac_address    = netkit_set_macaddr,
 231         .ndo_get_iflink         = netkit_get_iflink,
 232         .ndo_get_peer_dev       = netkit_peer_dev,
 233         .ndo_get_stats64        = netkit_get_stats,
 234         .ndo_uninit             = netkit_uninit,
 235         .ndo_features_check     = passthru_features_check,
 236 };
 237
 238 static void netkit_get_drvinfo(struct net_device *dev,
 239                                struct ethtool_drvinfo *info)
 240 {
 241         strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 242 }
 243
 244 static const struct ethtool_ops netkit_ethtool_ops = {
 245         .get_drvinfo            = netkit_get_drvinfo,
 246 };
 247
 248 static void netkit_setup(struct net_device *dev)
 249 {
 250         static const netdev_features_t netkit_features_hw_vlan =
 251                 NETIF_F_HW_VLAN_CTAG_TX |
 252                 NETIF_F_HW_VLAN_CTAG_RX |
 253                 NETIF_F_HW_VLAN_STAG_TX |
 254                 NETIF_F_HW_VLAN_STAG_RX;
 255         static const netdev_features_t netkit_features =
 256                 netkit_features_hw_vlan |
 257                 NETIF_F_SG |
 258                 NETIF_F_FRAGLIST |
 259                 NETIF_F_HW_CSUM |
 260                 NETIF_F_RXCSUM |
 261                 NETIF_F_SCTP_CRC |
 262                 NETIF_F_HIGHDMA |
 263                 NETIF_F_GSO_SOFTWARE |
 264                 NETIF_F_GSO_ENCAP_ALL;
 265
 266         ether_setup(dev);
 267         dev->max_mtu = ETH_MAX_MTU;
 268         dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
 269
 270         dev->flags |= IFF_NOARP;
 271         dev->priv_flags &= ~IFF_TX_SKB_SHARING;
 272         dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
 273         dev->priv_flags |= IFF_PHONY_HEADROOM;
 274         dev->priv_flags |= IFF_NO_QUEUE;
 275         dev->priv_flags |= IFF_DISABLE_NETPOLL;
 276         dev->lltx = true;
 277
 278         dev->ethtool_ops = &netkit_ethtool_ops;
 279         dev->netdev_ops  = &netkit_netdev_ops;
 280
 281         dev->features |= netkit_features;
 282         dev->hw_features = netkit_features;
 283         dev->hw_enc_features = netkit_features;
 284         dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
 285         dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
 286
 287         dev->needs_free_netdev = true;
 288
 289         netif_set_tso_max_size(dev, GSO_MAX_SIZE);
 290 }
 291
 292 static struct net *netkit_get_link_net(const struct net_device *dev)
 293 {
 294         struct netkit *nk = netkit_priv(dev);
 295         struct net_device *peer = rtnl_dereference(nk->peer);
 296
 297         return peer ? dev_net(peer) : dev_net(dev);
 298 }
 299
 300 static int netkit_check_policy(int policy, struct nlattr *tb,
 301                                struct netlink_ext_ack *extack)
 302 {
 303         switch (policy) {
 304         case NETKIT_PASS:
 305         case NETKIT_DROP:
 306                 return 0;
 307         default:
 308                 NL_SET_ERR_MSG_ATTR(extack, tb,
 309                                     "Provided default xmit policy not supported");
 310                 return -EINVAL;
 311         }
 312 }
 313
 314 static int netkit_validate(struct nlattr *tb[], struct nlattr *data[],
 315                            struct netlink_ext_ack *extack)
 316 {
 317         struct nlattr *attr = tb[IFLA_ADDRESS];
 318
 319         if (!attr)
 320                 return 0;
 321         if (nla_len(attr) != ETH_ALEN)
 322                 return -EINVAL;
 323         if (!is_valid_ether_addr(nla_data(attr)))
 324                 return -EADDRNOTAVAIL;
 325         return 0;
 326 }
 327
 328 static struct rtnl_link_ops netkit_link_ops;
 329
 330 static int netkit_new_link(struct net *peer_net, struct net_device *dev,
 331                            struct nlattr *tb[], struct nlattr *data[],
 332                            struct netlink_ext_ack *extack)
 333 {
 334         struct nlattr *peer_tb[IFLA_MAX + 1], **tbp = tb, *attr;
 335         enum netkit_action policy_prim = NETKIT_PASS;
 336         enum netkit_action policy_peer = NETKIT_PASS;
 337         enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT;
 338         enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT;
 339         enum netkit_mode mode = NETKIT_L3;
 340         unsigned char ifname_assign_type;
 341         struct ifinfomsg *ifmp = NULL;
 342         struct net_device *peer;
 343         char ifname[IFNAMSIZ];
 344         struct netkit *nk;
 345         int err;
 346
 347         if (data) {
 348                 if (data[IFLA_NETKIT_MODE])
 349                         mode = nla_get_u32(data[IFLA_NETKIT_MODE]);
 350                 if (data[IFLA_NETKIT_PEER_INFO]) {
 351                         attr = data[IFLA_NETKIT_PEER_INFO];
 352                         ifmp = nla_data(attr);
 353                         rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack);
 354                         tbp = peer_tb;
 355                 }
 356                 if (data[IFLA_NETKIT_SCRUB])
 357                         scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]);
 358                 if (data[IFLA_NETKIT_PEER_SCRUB])
 359                         scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]);
 360                 if (data[IFLA_NETKIT_POLICY]) {
 361                         attr = data[IFLA_NETKIT_POLICY];
 362                         policy_prim = nla_get_u32(attr);
 363                         err = netkit_check_policy(policy_prim, attr, extack);
 364                         if (err < 0)
 365                                 return err;
 366                 }
 367                 if (data[IFLA_NETKIT_PEER_POLICY]) {
 368                         attr = data[IFLA_NETKIT_PEER_POLICY];
 369                         policy_peer = nla_get_u32(attr);
 370                         err = netkit_check_policy(policy_peer, attr, extack);
 371                         if (err < 0)
 372                                 return err;
 373                 }
 374         }
 375
 376         if (ifmp && tbp[IFLA_IFNAME]) {
 377                 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
 378                 ifname_assign_type = NET_NAME_USER;
 379         } else {
 380                 strscpy(ifname, "nk%d", IFNAMSIZ);
 381                 ifname_assign_type = NET_NAME_ENUM;
 382         }
 383         if (mode != NETKIT_L2 &&
 384             (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS]))
 385                 return -EOPNOTSUPP;
 386
 387         peer = rtnl_create_link(peer_net, ifname, ifname_assign_type,
 388                                 &netkit_link_ops, tbp, extack);
 389         if (IS_ERR(peer))
 390                 return PTR_ERR(peer);
 391
 392         netif_inherit_tso_max(peer, dev);
 393
 394         if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS]))
 395                 eth_hw_addr_random(peer);
 396         if (ifmp && dev->ifindex)
 397                 peer->ifindex = ifmp->ifi_index;
 398
 399         nk = netkit_priv(peer);
 400         nk->primary = false;
 401         nk->policy = policy_peer;
 402         nk->scrub = scrub_peer;
 403         nk->mode = mode;
 404         bpf_mprog_bundle_init(&nk->bundle);
 405
 406         err = register_netdevice(peer);
 407         if (err < 0)
 408                 goto err_register_peer;
 409         netif_carrier_off(peer);
 410         if (mode == NETKIT_L2)
 411                 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);
 412
 413         err = rtnl_configure_link(peer, NULL, 0, NULL);
 414         if (err < 0)
 415                 goto err_configure_peer;
 416
 417         if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS])
 418                 eth_hw_addr_random(dev);
 419         if (tb[IFLA_IFNAME])
 420                 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
 421         else
 422                 strscpy(dev->name, "nk%d", IFNAMSIZ);
 423
 424         nk = netkit_priv(dev);
 425         nk->primary = true;
 426         nk->policy = policy_prim;
 427         nk->scrub = scrub_prim;
 428         nk->mode = mode;
 429         bpf_mprog_bundle_init(&nk->bundle);
 430
 431         err = register_netdevice(dev);
 432         if (err < 0)
 433                 goto err_configure_peer;
 434         netif_carrier_off(dev);
 435         if (mode == NETKIT_L2)
 436                 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);
 437
 438         rcu_assign_pointer(netkit_priv(dev)->peer, peer);
 439         rcu_assign_pointer(netkit_priv(peer)->peer, dev);
 440         return 0;
 441 err_configure_peer:
 442         unregister_netdevice(peer);
 443         return err;
 444 err_register_peer:
 445         free_netdev(peer);
 446         return err;
 447 }
 448
 449 static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev,
 450                                                   bool bundle_fallback)
 451 {
 452         struct netkit *nk = netkit_priv(dev);
 453         struct bpf_mprog_entry *entry;
 454
 455         ASSERT_RTNL();
 456         entry = rcu_dereference_rtnl(nk->active);
 457         if (entry)
 458                 return entry;
 459         if (bundle_fallback)
 460                 return &nk->bundle.a;
 461         return NULL;
 462 }
 463
 464 static void netkit_entry_update(struct net_device *dev,
 465                                 struct bpf_mprog_entry *entry)
 466 {
 467         struct netkit *nk = netkit_priv(dev);
 468
 469         ASSERT_RTNL();
 470         rcu_assign_pointer(nk->active, entry);
 471 }
 472
 473 static void netkit_entry_sync(void)
 474 {
 475         synchronize_rcu();
 476 }
 477
 478 static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which)
 479 {
 480         struct net_device *dev;
 481         struct netkit *nk;
 482
 483         ASSERT_RTNL();
 484
 485         switch (which) {
 486         case BPF_NETKIT_PRIMARY:
 487         case BPF_NETKIT_PEER:
 488                 break;
 489         default:
 490                 return ERR_PTR(-EINVAL);
 491         }
 492
 493         dev = __dev_get_by_index(net, ifindex);
 494         if (!dev)
 495                 return ERR_PTR(-ENODEV);
 496         if (dev->netdev_ops != &netkit_netdev_ops)
 497                 return ERR_PTR(-ENXIO);
 498
 499         nk = netkit_priv(dev);
 500         if (!nk->primary)
 501                 return ERR_PTR(-EACCES);
 502         if (which == BPF_NETKIT_PEER) {
 503                 dev = rcu_dereference_rtnl(nk->peer);
 504                 if (!dev)
 505                         return ERR_PTR(-ENODEV);
 506         }
 507         return dev;
 508 }
 509
 510 int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 511 {
 512         struct bpf_mprog_entry *entry, *entry_new;
 513         struct bpf_prog *replace_prog = NULL;
 514         struct net_device *dev;
 515         int ret;
 516
 517         rtnl_lock();
 518         dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
 519                                attr->attach_type);
 520         if (IS_ERR(dev)) {
 521                 ret = PTR_ERR(dev);
 522                 goto out;
 523         }
 524         entry = netkit_entry_fetch(dev, true);
 525         if (attr->attach_flags & BPF_F_REPLACE) {
 526                 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
 527                                                  prog->type);
 528                 if (IS_ERR(replace_prog)) {
 529                         ret = PTR_ERR(replace_prog);
 530                         replace_prog = NULL;
 531                         goto out;
 532                 }
 533         }
 534         ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
 535                                attr->attach_flags, attr->relative_fd,
 536                                attr->expected_revision);
 537         if (!ret) {
 538                 if (entry != entry_new) {
 539                         netkit_entry_update(dev, entry_new);
 540                         netkit_entry_sync();
 541                 }
 542                 bpf_mprog_commit(entry);
 543         }
 544 out:
 545         if (replace_prog)
 546                 bpf_prog_put(replace_prog);
 547         rtnl_unlock();
 548         return ret;
 549 }
 550
 551 int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
 552 {
 553         struct bpf_mprog_entry *entry, *entry_new;
 554         struct net_device *dev;
 555         int ret;
 556
 557         rtnl_lock();
 558         dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
 559                                attr->attach_type);
 560         if (IS_ERR(dev)) {
 561                 ret = PTR_ERR(dev);
 562                 goto out;
 563         }
 564         entry = netkit_entry_fetch(dev, false);
 565         if (!entry) {
 566                 ret = -ENOENT;
 567                 goto out;
 568         }
 569         ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
 570                                attr->relative_fd, attr->expected_revision);
 571         if (!ret) {
 572                 if (!bpf_mprog_total(entry_new))
 573                         entry_new = NULL;
 574                 netkit_entry_update(dev, entry_new);
 575                 netkit_entry_sync();
 576                 bpf_mprog_commit(entry);
 577         }
 578 out:
 579         rtnl_unlock();
 580         return ret;
 581 }
 582
 583 int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
 584 {
 585         struct net_device *dev;
 586         int ret;
 587
 588         rtnl_lock();
 589         dev = netkit_dev_fetch(current->nsproxy->net_ns,
 590                                attr->query.target_ifindex,
 591                                attr->query.attach_type);
 592         if (IS_ERR(dev)) {
 593                 ret = PTR_ERR(dev);
 594                 goto out;
 595         }
 596         ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));
 597 out:
 598         rtnl_unlock();
 599         return ret;
 600 }
 601
 602 static struct netkit_link *netkit_link(const struct bpf_link *link)
 603 {
 604         return container_of(link, struct netkit_link, link);
 605 }
 606
 607 static int netkit_link_prog_attach(struct bpf_link *link, u32 flags,
 608                                    u32 id_or_fd, u64 revision)
 609 {
 610         struct netkit_link *nkl = netkit_link(link);
 611         struct bpf_mprog_entry *entry, *entry_new;
 612         struct net_device *dev = nkl->dev;
 613         int ret;
 614
 615         ASSERT_RTNL();
 616         entry = netkit_entry_fetch(dev, true);
 617         ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
 618                                id_or_fd, revision);
 619         if (!ret) {
 620                 if (entry != entry_new) {
 621                         netkit_entry_update(dev, entry_new);
 622                         netkit_entry_sync();
 623                 }
 624                 bpf_mprog_commit(entry);
 625         }
 626         return ret;
 627 }
 628
 629 static void netkit_link_release(struct bpf_link *link)
 630 {
 631         struct netkit_link *nkl = netkit_link(link);
 632         struct bpf_mprog_entry *entry, *entry_new;
 633         struct net_device *dev;
 634         int ret = 0;
 635
 636         rtnl_lock();
 637         dev = nkl->dev;
 638         if (!dev)
 639                 goto out;
 640         entry = netkit_entry_fetch(dev, false);
 641         if (!entry) {
 642                 ret = -ENOENT;
 643                 goto out;
 644         }
 645         ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
 646         if (!ret) {
 647                 if (!bpf_mprog_total(entry_new))
 648                         entry_new = NULL;
 649                 netkit_entry_update(dev, entry_new);
 650                 netkit_entry_sync();
 651                 bpf_mprog_commit(entry);
 652                 nkl->dev = NULL;
 653         }
 654 out:
 655         WARN_ON_ONCE(ret);
 656         rtnl_unlock();
 657 }
 658
 659 static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog,
 660                               struct bpf_prog *oprog)
 661 {
 662         struct netkit_link *nkl = netkit_link(link);
 663         struct bpf_mprog_entry *entry, *entry_new;
 664         struct net_device *dev;
 665         int ret = 0;
 666
 667         rtnl_lock();
 668         dev = nkl->dev;
 669         if (!dev) {
 670                 ret = -ENOLINK;
 671                 goto out;
 672         }
 673         if (oprog && link->prog != oprog) {
 674                 ret = -EPERM;
 675                 goto out;
 676         }
 677         oprog = link->prog;
 678         if (oprog == nprog) {
 679                 bpf_prog_put(nprog);
 680                 goto out;
 681         }
 682         entry = netkit_entry_fetch(dev, false);
 683         if (!entry) {
 684                 ret = -ENOENT;
 685                 goto out;
 686         }
 687         ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
 688                                BPF_F_REPLACE | BPF_F_ID,
 689                                link->prog->aux->id, 0);
 690         if (!ret) {
 691                 WARN_ON_ONCE(entry != entry_new);
 692                 oprog = xchg(&link->prog, nprog);
 693                 bpf_prog_put(oprog);
 694                 bpf_mprog_commit(entry);
 695         }
 696 out:
 697         rtnl_unlock();
 698         return ret;
 699 }
 700
 701 static void netkit_link_dealloc(struct bpf_link *link)
 702 {
 703         kfree(netkit_link(link));
 704 }
 705
 706 static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
 707 {
 708         const struct netkit_link *nkl = netkit_link(link);
 709         u32 ifindex = 0;
 710
 711         rtnl_lock();
 712         if (nkl->dev)
 713                 ifindex = nkl->dev->ifindex;
 714         rtnl_unlock();
 715
 716         seq_printf(seq, "ifindex:\t%u\n", ifindex);
 717         seq_printf(seq, "attach_type:\t%u (%s)\n",
 718                    nkl->location,
 719                    nkl->location == BPF_NETKIT_PRIMARY ? "primary" : "peer");
 720 }
 721
 722 static int netkit_link_fill_info(const struct bpf_link *link,
 723                                  struct bpf_link_info *info)
 724 {
 725         const struct netkit_link *nkl = netkit_link(link);
 726         u32 ifindex = 0;
 727
 728         rtnl_lock();
 729         if (nkl->dev)
 730                 ifindex = nkl->dev->ifindex;
 731         rtnl_unlock();
 732
 733         info->netkit.ifindex = ifindex;
 734         info->netkit.attach_type = nkl->location;
 735         return 0;
 736 }
 737
 738 static int netkit_link_detach(struct bpf_link *link)
 739 {
 740         netkit_link_release(link);
 741         return 0;
 742 }
 743
 744 static const struct bpf_link_ops netkit_link_lops = {
 745         .release        = netkit_link_release,
 746         .detach         = netkit_link_detach,
 747         .dealloc        = netkit_link_dealloc,
 748         .update_prog    = netkit_link_update,
 749         .show_fdinfo    = netkit_link_fdinfo,
 750         .fill_link_info = netkit_link_fill_info,
 751 };
 752
 753 static int netkit_link_init(struct netkit_link *nkl,
 754                             struct bpf_link_primer *link_primer,
 755                             const union bpf_attr *attr,
 756                             struct net_device *dev,
 757                             struct bpf_prog *prog)
 758 {
 759         bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
 760                       &netkit_link_lops, prog);
 761         nkl->location = attr->link_create.attach_type;
 762         nkl->dev = dev;
 763         return bpf_link_prime(&nkl->link, link_primer);
 764 }
 765
 766 int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 767 {
 768         struct bpf_link_primer link_primer;
 769         struct netkit_link *nkl;
 770         struct net_device *dev;
 771         int ret;
 772
 773         rtnl_lock();
 774         dev = netkit_dev_fetch(current->nsproxy->net_ns,
 775                                attr->link_create.target_ifindex,
 776                                attr->link_create.attach_type);
 777         if (IS_ERR(dev)) {
 778                 ret = PTR_ERR(dev);
 779                 goto out;
 780         }
 781         nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT);
 782         if (!nkl) {
 783                 ret = -ENOMEM;
 784                 goto out;
 785         }
 786         ret = netkit_link_init(nkl, &link_primer, attr, dev, prog);
 787         if (ret) {
 788                 kfree(nkl);
 789                 goto out;
 790         }
 791         ret = netkit_link_prog_attach(&nkl->link,
 792                                       attr->link_create.flags,
 793                                       attr->link_create.netkit.relative_fd,
 794                                       attr->link_create.netkit.expected_revision);
 795         if (ret) {
 796                 nkl->dev = NULL;
 797                 bpf_link_cleanup(&link_primer);
 798                 goto out;
 799         }
 800         ret = bpf_link_settle(&link_primer);
 801 out:
 802         rtnl_unlock();
 803         return ret;
 804 }
 805
 806 static void netkit_release_all(struct net_device *dev)
 807 {
 808         struct bpf_mprog_entry *entry;
 809         struct bpf_tuple tuple = {};
 810         struct bpf_mprog_fp *fp;
 811         struct bpf_mprog_cp *cp;
 812
 813         entry = netkit_entry_fetch(dev, false);
 814         if (!entry)
 815                 return;
 816         netkit_entry_update(dev, NULL);
 817         netkit_entry_sync();
 818         bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
 819                 if (tuple.link)
 820                         netkit_link(tuple.link)->dev = NULL;
 821                 else
 822                         bpf_prog_put(tuple.prog);
 823         }
 824 }
 825
 826 static void netkit_uninit(struct net_device *dev)
 827 {
 828         netkit_release_all(dev);
 829 }
 830
 831 static void netkit_del_link(struct net_device *dev, struct list_head *head)
 832 {
 833         struct netkit *nk = netkit_priv(dev);
 834         struct net_device *peer = rtnl_dereference(nk->peer);
 835
 836         RCU_INIT_POINTER(nk->peer, NULL);
 837         unregister_netdevice_queue(dev, head);
 838         if (peer) {
 839                 nk = netkit_priv(peer);
 840                 RCU_INIT_POINTER(nk->peer, NULL);
 841                 unregister_netdevice_queue(peer, head);
 842         }
 843 }
 844
 845 static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
 846                               struct nlattr *data[],
 847                               struct netlink_ext_ack *extack)
 848 {
 849         struct netkit *nk = netkit_priv(dev);
 850         struct net_device *peer = rtnl_dereference(nk->peer);
 851         enum netkit_action policy;
 852         struct nlattr *attr;
 853         int err;
 854
 855         if (!nk->primary) {
 856                 NL_SET_ERR_MSG(extack,
 857                                "netkit link settings can be changed only through the primary device");
 858                 return -EACCES;
 859         }
 860
 861         if (data[IFLA_NETKIT_MODE]) {
 862                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_MODE],
 863                                     "netkit link operating mode cannot be changed after device creation");
 864                 return -EACCES;
 865         }
 866
 867         if (data[IFLA_NETKIT_SCRUB]) {
 868                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_SCRUB],
 869                                     "netkit scrubbing cannot be changed after device creation");
 870                 return -EACCES;
 871         }
 872
 873         if (data[IFLA_NETKIT_PEER_SCRUB]) {
 874                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_SCRUB],
 875                                     "netkit scrubbing cannot be changed after device creation");
 876                 return -EACCES;
 877         }
 878
 879         if (data[IFLA_NETKIT_PEER_INFO]) {
 880                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_INFO],
 881                                     "netkit peer info cannot be changed after device creation");
 882                 return -EINVAL;
 883         }
 884
 885         if (data[IFLA_NETKIT_POLICY]) {
 886                 attr = data[IFLA_NETKIT_POLICY];
 887                 policy = nla_get_u32(attr);
 888                 err = netkit_check_policy(policy, attr, extack);
 889                 if (err)
 890                         return err;
 891                 WRITE_ONCE(nk->policy, policy);
 892         }
 893
 894         if (data[IFLA_NETKIT_PEER_POLICY]) {
 895                 err = -EOPNOTSUPP;
 896                 attr = data[IFLA_NETKIT_PEER_POLICY];
 897                 policy = nla_get_u32(attr);
 898                 if (peer)
 899                         err = netkit_check_policy(policy, attr, extack);
 900                 if (err)
 901                         return err;
 902                 nk = netkit_priv(peer);
 903                 WRITE_ONCE(nk->policy, policy);
 904         }
 905
 906         return 0;
 907 }
 908
 909 static size_t netkit_get_size(const struct net_device *dev)
 910 {
 911         return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */
 912                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */
 913                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */
 914                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */
 915                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */
 916                nla_total_size(sizeof(u8))  + /* IFLA_NETKIT_PRIMARY */
 917                0;
 918 }
 919
 920 static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
 921 {
 922         struct netkit *nk = netkit_priv(dev);
 923         struct net_device *peer = rtnl_dereference(nk->peer);
 924
 925         if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary))
 926                 return -EMSGSIZE;
 927         if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy))
 928                 return -EMSGSIZE;
 929         if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode))
 930                 return -EMSGSIZE;
 931         if (nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub))
 932                 return -EMSGSIZE;
 933
 934         if (peer) {
 935                 nk = netkit_priv(peer);
 936                 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy))
 937                         return -EMSGSIZE;
 938                 if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub))
 939                         return -EMSGSIZE;
 940         }
 941
 942         return 0;
 943 }
 944
 945 static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = {
 946         [IFLA_NETKIT_PEER_INFO]         = { .len = sizeof(struct ifinfomsg) },
 947         [IFLA_NETKIT_MODE]              = NLA_POLICY_MAX(NLA_U32, NETKIT_L3),
 948         [IFLA_NETKIT_POLICY]            = { .type = NLA_U32 },
 949         [IFLA_NETKIT_PEER_POLICY]       = { .type = NLA_U32 },
 950         [IFLA_NETKIT_SCRUB]             = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
 951         [IFLA_NETKIT_PEER_SCRUB]        = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
 952         [IFLA_NETKIT_PRIMARY]           = { .type = NLA_REJECT,
 953                                             .reject_message = "Primary attribute is read-only" },
 954 };
 955
 956 static struct rtnl_link_ops netkit_link_ops = {
 957         .kind           = DRV_NAME,
 958         .priv_size      = sizeof(struct netkit),
 959         .setup          = netkit_setup,
 960         .newlink        = netkit_new_link,
 961         .dellink        = netkit_del_link,
 962         .changelink     = netkit_change_link,
 963         .get_link_net   = netkit_get_link_net,
 964         .get_size       = netkit_get_size,
 965         .fill_info      = netkit_fill_info,
 966         .policy         = netkit_policy,
 967         .validate       = netkit_validate,
 968         .peer_type      = IFLA_NETKIT_PEER_INFO,
 969         .maxtype        = IFLA_NETKIT_MAX,
 970 };
 971
 972 static __init int netkit_init(void)
 973 {
 974         BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
 975                      (int)NETKIT_PASS != (int)TCX_PASS ||
 976                      (int)NETKIT_DROP != (int)TCX_DROP ||
 977                      (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
 978
 979         return rtnl_link_register(&netkit_link_ops);
 980 }
 981
 982 static __exit void netkit_exit(void)
 983 {
 984         rtnl_link_unregister(&netkit_link_ops);
 985 }
 986
 987 module_init(netkit_init);
 988 module_exit(netkit_exit);
 989
 990 MODULE_DESCRIPTION("BPF-programmable network device");
 991 MODULE_AUTHOR("Daniel Borkmann <[email protected]>");
 992 MODULE_AUTHOR("Nikolay Aleksandrov <[email protected]>");
 993 MODULE_LICENSE("GPL");
 994 MODULE_ALIAS_RTNL_LINK(DRV_NAME);