drivers/net/netkit.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2023 Isovalent */
   3
   4 #include <linux/netdevice.h>
   5 #include <linux/ethtool.h>
   6 #include <linux/etherdevice.h>
   7 #include <linux/filter.h>
   8 #include <linux/netfilter_netdev.h>
   9 #include <linux/bpf_mprog.h>
  10 #include <linux/indirect_call_wrapper.h>
  11
  12 #include <net/netkit.h>
  13 #include <net/dst.h>
  14 #include <net/tcx.h>
  15
  16 #define DRV_NAME "netkit"
  17
  18 struct netkit {
  19         /* Needed in fast-path */
  20         struct net_device __rcu *peer;
  21         struct bpf_mprog_entry __rcu *active;
  22         enum netkit_action policy;
  23         struct bpf_mprog_bundle bundle;
  24
  25         /* Needed in slow-path */
  26         enum netkit_mode mode;
  27         bool primary;
  28         u32 headroom;
  29 };
  30
  31 struct netkit_link {
  32         struct bpf_link link;
  33         struct net_device *dev;
  34         u32 location;
  35 };
  36
  37 static __always_inline int
  38 netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
  39            enum netkit_action ret)
  40 {
  41         const struct bpf_mprog_fp *fp;
  42         const struct bpf_prog *prog;
  43
  44         bpf_mprog_foreach_prog(entry, fp, prog) {
  45                 bpf_compute_data_pointers(skb);
  46                 ret = bpf_prog_run(prog, skb);
  47                 if (ret != NETKIT_NEXT)
  48                         break;
  49         }
  50         return ret;
  51 }
  52
  53 static void netkit_prep_forward(struct sk_buff *skb, bool xnet)
  54 {
  55         skb_scrub_packet(skb, xnet);
  56         skb->priority = 0;
  57         nf_skip_egress(skb, true);
  58         skb_reset_mac_header(skb);
  59 }
  60
  61 static struct netkit *netkit_priv(const struct net_device *dev)
  62 {
  63         return netdev_priv(dev);
  64 }
  65
  66 static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
  67 {
  68         struct netkit *nk = netkit_priv(dev);
  69         enum netkit_action ret = READ_ONCE(nk->policy);
  70         netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
  71         const struct bpf_mprog_entry *entry;
  72         struct net_device *peer;
  73         int len = skb->len;
  74
  75         rcu_read_lock();
  76         peer = rcu_dereference(nk->peer);
  77         if (unlikely(!peer || !(peer->flags & IFF_UP) ||
  78                      !pskb_may_pull(skb, ETH_HLEN) ||
  79                      skb_orphan_frags(skb, GFP_ATOMIC)))
  80                 goto drop;
  81         netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)));
  82         eth_skb_pkt_type(skb, peer);
  83         skb->dev = peer;
  84         entry = rcu_dereference(nk->active);
  85         if (entry)
  86                 ret = netkit_run(entry, skb, ret);
  87         switch (ret) {
  88         case NETKIT_NEXT:
  89         case NETKIT_PASS:
  90                 eth_skb_pull_mac(skb);
  91                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
  92                 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
  93                         dev_sw_netstats_tx_add(dev, 1, len);
  94                         dev_sw_netstats_rx_add(peer, len);
  95                 } else {
  96                         goto drop_stats;
  97                 }
  98                 break;
  99         case NETKIT_REDIRECT:
 100                 dev_sw_netstats_tx_add(dev, 1, len);
 101                 skb_do_redirect(skb);
 102                 break;
 103         case NETKIT_DROP:
 104         default:
 105 drop:
 106                 kfree_skb(skb);
 107 drop_stats:
 108                 dev_core_stats_tx_dropped_inc(dev);
 109                 ret_dev = NET_XMIT_DROP;
 110                 break;
 111         }
 112         rcu_read_unlock();
 113         return ret_dev;
 114 }
 115
 116 static int netkit_open(struct net_device *dev)
 117 {
 118         struct netkit *nk = netkit_priv(dev);
 119         struct net_device *peer = rtnl_dereference(nk->peer);
 120
 121         if (!peer)
 122                 return -ENOTCONN;
 123         if (peer->flags & IFF_UP) {
 124                 netif_carrier_on(dev);
 125                 netif_carrier_on(peer);
 126         }
 127         return 0;
 128 }
 129
 130 static int netkit_close(struct net_device *dev)
 131 {
 132         struct netkit *nk = netkit_priv(dev);
 133         struct net_device *peer = rtnl_dereference(nk->peer);
 134
 135         netif_carrier_off(dev);
 136         if (peer)
 137                 netif_carrier_off(peer);
 138         return 0;
 139 }
 140
 141 static int netkit_get_iflink(const struct net_device *dev)
 142 {
 143         struct netkit *nk = netkit_priv(dev);
 144         struct net_device *peer;
 145         int iflink = 0;
 146
 147         rcu_read_lock();
 148         peer = rcu_dereference(nk->peer);
 149         if (peer)
 150                 iflink = READ_ONCE(peer->ifindex);
 151         rcu_read_unlock();
 152         return iflink;
 153 }
 154
 155 static void netkit_set_multicast(struct net_device *dev)
 156 {
 157         /* Nothing to do, we receive whatever gets pushed to us! */
 158 }
 159
 160 static int netkit_set_macaddr(struct net_device *dev, void *sa)
 161 {
 162         struct netkit *nk = netkit_priv(dev);
 163
 164         if (nk->mode != NETKIT_L2)
 165                 return -EOPNOTSUPP;
 166
 167         return eth_mac_addr(dev, sa);
 168 }
 169
 170 static void netkit_set_headroom(struct net_device *dev, int headroom)
 171 {
 172         struct netkit *nk = netkit_priv(dev), *nk2;
 173         struct net_device *peer;
 174
 175         if (headroom < 0)
 176                 headroom = NET_SKB_PAD;
 177
 178         rcu_read_lock();
 179         peer = rcu_dereference(nk->peer);
 180         if (unlikely(!peer))
 181                 goto out;
 182
 183         nk2 = netkit_priv(peer);
 184         nk->headroom = headroom;
 185         headroom = max(nk->headroom, nk2->headroom);
 186
 187         peer->needed_headroom = headroom;
 188         dev->needed_headroom = headroom;
 189 out:
 190         rcu_read_unlock();
 191 }
 192
 193 INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
 194 {
 195         return rcu_dereference(netkit_priv(dev)->peer);
 196 }
 197
 198 static void netkit_get_stats(struct net_device *dev,
 199                              struct rtnl_link_stats64 *stats)
 200 {
 201         dev_fetch_sw_netstats(stats, dev->tstats);
 202         stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
 203 }
 204
 205 static void netkit_uninit(struct net_device *dev);
 206
 207 static const struct net_device_ops netkit_netdev_ops = {
 208         .ndo_open               = netkit_open,
 209         .ndo_stop               = netkit_close,
 210         .ndo_start_xmit         = netkit_xmit,
 211         .ndo_set_rx_mode        = netkit_set_multicast,
 212         .ndo_set_rx_headroom    = netkit_set_headroom,
 213         .ndo_set_mac_address    = netkit_set_macaddr,
 214         .ndo_get_iflink         = netkit_get_iflink,
 215         .ndo_get_peer_dev       = netkit_peer_dev,
 216         .ndo_get_stats64        = netkit_get_stats,
 217         .ndo_uninit             = netkit_uninit,
 218         .ndo_features_check     = passthru_features_check,
 219 };
 220
 221 static void netkit_get_drvinfo(struct net_device *dev,
 222                                struct ethtool_drvinfo *info)
 223 {
 224         strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 225 }
 226
 227 static const struct ethtool_ops netkit_ethtool_ops = {
 228         .get_drvinfo            = netkit_get_drvinfo,
 229 };
 230
 231 static void netkit_setup(struct net_device *dev)
 232 {
 233         static const netdev_features_t netkit_features_hw_vlan =
 234                 NETIF_F_HW_VLAN_CTAG_TX |
 235                 NETIF_F_HW_VLAN_CTAG_RX |
 236                 NETIF_F_HW_VLAN_STAG_TX |
 237                 NETIF_F_HW_VLAN_STAG_RX;
 238         static const netdev_features_t netkit_features =
 239                 netkit_features_hw_vlan |
 240                 NETIF_F_SG |
 241                 NETIF_F_FRAGLIST |
 242                 NETIF_F_HW_CSUM |
 243                 NETIF_F_RXCSUM |
 244                 NETIF_F_SCTP_CRC |
 245                 NETIF_F_HIGHDMA |
 246                 NETIF_F_GSO_SOFTWARE |
 247                 NETIF_F_GSO_ENCAP_ALL;
 248
 249         ether_setup(dev);
 250         dev->max_mtu = ETH_MAX_MTU;
 251         dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
 252
 253         dev->flags |= IFF_NOARP;
 254         dev->priv_flags &= ~IFF_TX_SKB_SHARING;
 255         dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
 256         dev->priv_flags |= IFF_PHONY_HEADROOM;
 257         dev->priv_flags |= IFF_NO_QUEUE;
 258
 259         dev->ethtool_ops = &netkit_ethtool_ops;
 260         dev->netdev_ops  = &netkit_netdev_ops;
 261
 262         dev->features |= netkit_features | NETIF_F_LLTX;
 263         dev->hw_features = netkit_features;
 264         dev->hw_enc_features = netkit_features;
 265         dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
 266         dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
 267
 268         dev->needs_free_netdev = true;
 269
 270         netif_set_tso_max_size(dev, GSO_MAX_SIZE);
 271 }
 272
 273 static struct net *netkit_get_link_net(const struct net_device *dev)
 274 {
 275         struct netkit *nk = netkit_priv(dev);
 276         struct net_device *peer = rtnl_dereference(nk->peer);
 277
 278         return peer ? dev_net(peer) : dev_net(dev);
 279 }
 280
 281 static int netkit_check_policy(int policy, struct nlattr *tb,
 282                                struct netlink_ext_ack *extack)
 283 {
 284         switch (policy) {
 285         case NETKIT_PASS:
 286         case NETKIT_DROP:
 287                 return 0;
 288         default:
 289                 NL_SET_ERR_MSG_ATTR(extack, tb,
 290                                     "Provided default xmit policy not supported");
 291                 return -EINVAL;
 292         }
 293 }
 294
 295 static int netkit_check_mode(int mode, struct nlattr *tb,
 296                              struct netlink_ext_ack *extack)
 297 {
 298         switch (mode) {
 299         case NETKIT_L2:
 300         case NETKIT_L3:
 301                 return 0;
 302         default:
 303                 NL_SET_ERR_MSG_ATTR(extack, tb,
 304                                     "Provided device mode can only be L2 or L3");
 305                 return -EINVAL;
 306         }
 307 }
 308
 309 static int netkit_validate(struct nlattr *tb[], struct nlattr *data[],
 310                            struct netlink_ext_ack *extack)
 311 {
 312         struct nlattr *attr = tb[IFLA_ADDRESS];
 313
 314         if (!attr)
 315                 return 0;
 316         if (nla_len(attr) != ETH_ALEN)
 317                 return -EINVAL;
 318         if (!is_valid_ether_addr(nla_data(attr)))
 319                 return -EADDRNOTAVAIL;
 320         return 0;
 321 }
 322
 323 static struct rtnl_link_ops netkit_link_ops;
 324
 325 static int netkit_new_link(struct net *src_net, struct net_device *dev,
 326                            struct nlattr *tb[], struct nlattr *data[],
 327                            struct netlink_ext_ack *extack)
 328 {
 329         struct nlattr *peer_tb[IFLA_MAX + 1], **tbp = tb, *attr;
 330         enum netkit_action default_prim = NETKIT_PASS;
 331         enum netkit_action default_peer = NETKIT_PASS;
 332         enum netkit_mode mode = NETKIT_L3;
 333         unsigned char ifname_assign_type;
 334         struct ifinfomsg *ifmp = NULL;
 335         struct net_device *peer;
 336         char ifname[IFNAMSIZ];
 337         struct netkit *nk;
 338         struct net *net;
 339         int err;
 340
 341         if (data) {
 342                 if (data[IFLA_NETKIT_MODE]) {
 343                         attr = data[IFLA_NETKIT_MODE];
 344                         mode = nla_get_u32(attr);
 345                         err = netkit_check_mode(mode, attr, extack);
 346                         if (err < 0)
 347                                 return err;
 348                 }
 349                 if (data[IFLA_NETKIT_PEER_INFO]) {
 350                         attr = data[IFLA_NETKIT_PEER_INFO];
 351                         ifmp = nla_data(attr);
 352                         err = rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack);
 353                         if (err < 0)
 354                                 return err;
 355                         err = netkit_validate(peer_tb, NULL, extack);
 356                         if (err < 0)
 357                                 return err;
 358                         tbp = peer_tb;
 359                 }
 360                 if (data[IFLA_NETKIT_POLICY]) {
 361                         attr = data[IFLA_NETKIT_POLICY];
 362                         default_prim = nla_get_u32(attr);
 363                         err = netkit_check_policy(default_prim, attr, extack);
 364                         if (err < 0)
 365                                 return err;
 366                 }
 367                 if (data[IFLA_NETKIT_PEER_POLICY]) {
 368                         attr = data[IFLA_NETKIT_PEER_POLICY];
 369                         default_peer = nla_get_u32(attr);
 370                         err = netkit_check_policy(default_peer, attr, extack);
 371                         if (err < 0)
 372                                 return err;
 373                 }
 374         }
 375
 376         if (ifmp && tbp[IFLA_IFNAME]) {
 377                 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
 378                 ifname_assign_type = NET_NAME_USER;
 379         } else {
 380                 strscpy(ifname, "nk%d", IFNAMSIZ);
 381                 ifname_assign_type = NET_NAME_ENUM;
 382         }
 383         if (mode != NETKIT_L2 &&
 384             (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS]))
 385                 return -EOPNOTSUPP;
 386
 387         net = rtnl_link_get_net(src_net, tbp);
 388         if (IS_ERR(net))
 389                 return PTR_ERR(net);
 390
 391         peer = rtnl_create_link(net, ifname, ifname_assign_type,
 392                                 &netkit_link_ops, tbp, extack);
 393         if (IS_ERR(peer)) {
 394                 put_net(net);
 395                 return PTR_ERR(peer);
 396         }
 397
 398         netif_inherit_tso_max(peer, dev);
 399
 400         if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS]))
 401                 eth_hw_addr_random(peer);
 402         if (ifmp && dev->ifindex)
 403                 peer->ifindex = ifmp->ifi_index;
 404
 405         nk = netkit_priv(peer);
 406         nk->primary = false;
 407         nk->policy = default_peer;
 408         nk->mode = mode;
 409         bpf_mprog_bundle_init(&nk->bundle);
 410
 411         err = register_netdevice(peer);
 412         put_net(net);
 413         if (err < 0)
 414                 goto err_register_peer;
 415         netif_carrier_off(peer);
 416         if (mode == NETKIT_L2)
 417                 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);
 418
 419         err = rtnl_configure_link(peer, NULL, 0, NULL);
 420         if (err < 0)
 421                 goto err_configure_peer;
 422
 423         if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS])
 424                 eth_hw_addr_random(dev);
 425         if (tb[IFLA_IFNAME])
 426                 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
 427         else
 428                 strscpy(dev->name, "nk%d", IFNAMSIZ);
 429
 430         nk = netkit_priv(dev);
 431         nk->primary = true;
 432         nk->policy = default_prim;
 433         nk->mode = mode;
 434         bpf_mprog_bundle_init(&nk->bundle);
 435
 436         err = register_netdevice(dev);
 437         if (err < 0)
 438                 goto err_configure_peer;
 439         netif_carrier_off(dev);
 440         if (mode == NETKIT_L2)
 441                 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);
 442
 443         rcu_assign_pointer(netkit_priv(dev)->peer, peer);
 444         rcu_assign_pointer(netkit_priv(peer)->peer, dev);
 445         return 0;
 446 err_configure_peer:
 447         unregister_netdevice(peer);
 448         return err;
 449 err_register_peer:
 450         free_netdev(peer);
 451         return err;
 452 }
 453
 454 static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev,
 455                                                   bool bundle_fallback)
 456 {
 457         struct netkit *nk = netkit_priv(dev);
 458         struct bpf_mprog_entry *entry;
 459
 460         ASSERT_RTNL();
 461         entry = rcu_dereference_rtnl(nk->active);
 462         if (entry)
 463                 return entry;
 464         if (bundle_fallback)
 465                 return &nk->bundle.a;
 466         return NULL;
 467 }
 468
 469 static void netkit_entry_update(struct net_device *dev,
 470                                 struct bpf_mprog_entry *entry)
 471 {
 472         struct netkit *nk = netkit_priv(dev);
 473
 474         ASSERT_RTNL();
 475         rcu_assign_pointer(nk->active, entry);
 476 }
 477
 478 static void netkit_entry_sync(void)
 479 {
 480         synchronize_rcu();
 481 }
 482
 483 static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which)
 484 {
 485         struct net_device *dev;
 486         struct netkit *nk;
 487
 488         ASSERT_RTNL();
 489
 490         switch (which) {
 491         case BPF_NETKIT_PRIMARY:
 492         case BPF_NETKIT_PEER:
 493                 break;
 494         default:
 495                 return ERR_PTR(-EINVAL);
 496         }
 497
 498         dev = __dev_get_by_index(net, ifindex);
 499         if (!dev)
 500                 return ERR_PTR(-ENODEV);
 501         if (dev->netdev_ops != &netkit_netdev_ops)
 502                 return ERR_PTR(-ENXIO);
 503
 504         nk = netkit_priv(dev);
 505         if (!nk->primary)
 506                 return ERR_PTR(-EACCES);
 507         if (which == BPF_NETKIT_PEER) {
 508                 dev = rcu_dereference_rtnl(nk->peer);
 509                 if (!dev)
 510                         return ERR_PTR(-ENODEV);
 511         }
 512         return dev;
 513 }
 514
 515 int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 516 {
 517         struct bpf_mprog_entry *entry, *entry_new;
 518         struct bpf_prog *replace_prog = NULL;
 519         struct net_device *dev;
 520         int ret;
 521
 522         rtnl_lock();
 523         dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
 524                                attr->attach_type);
 525         if (IS_ERR(dev)) {
 526                 ret = PTR_ERR(dev);
 527                 goto out;
 528         }
 529         entry = netkit_entry_fetch(dev, true);
 530         if (attr->attach_flags & BPF_F_REPLACE) {
 531                 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
 532                                                  prog->type);
 533                 if (IS_ERR(replace_prog)) {
 534                         ret = PTR_ERR(replace_prog);
 535                         replace_prog = NULL;
 536                         goto out;
 537                 }
 538         }
 539         ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
 540                                attr->attach_flags, attr->relative_fd,
 541                                attr->expected_revision);
 542         if (!ret) {
 543                 if (entry != entry_new) {
 544                         netkit_entry_update(dev, entry_new);
 545                         netkit_entry_sync();
 546                 }
 547                 bpf_mprog_commit(entry);
 548         }
 549 out:
 550         if (replace_prog)
 551                 bpf_prog_put(replace_prog);
 552         rtnl_unlock();
 553         return ret;
 554 }
 555
 556 int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
 557 {
 558         struct bpf_mprog_entry *entry, *entry_new;
 559         struct net_device *dev;
 560         int ret;
 561
 562         rtnl_lock();
 563         dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
 564                                attr->attach_type);
 565         if (IS_ERR(dev)) {
 566                 ret = PTR_ERR(dev);
 567                 goto out;
 568         }
 569         entry = netkit_entry_fetch(dev, false);
 570         if (!entry) {
 571                 ret = -ENOENT;
 572                 goto out;
 573         }
 574         ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
 575                                attr->relative_fd, attr->expected_revision);
 576         if (!ret) {
 577                 if (!bpf_mprog_total(entry_new))
 578                         entry_new = NULL;
 579                 netkit_entry_update(dev, entry_new);
 580                 netkit_entry_sync();
 581                 bpf_mprog_commit(entry);
 582         }
 583 out:
 584         rtnl_unlock();
 585         return ret;
 586 }
 587
 588 int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
 589 {
 590         struct net_device *dev;
 591         int ret;
 592
 593         rtnl_lock();
 594         dev = netkit_dev_fetch(current->nsproxy->net_ns,
 595                                attr->query.target_ifindex,
 596                                attr->query.attach_type);
 597         if (IS_ERR(dev)) {
 598                 ret = PTR_ERR(dev);
 599                 goto out;
 600         }
 601         ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));
 602 out:
 603         rtnl_unlock();
 604         return ret;
 605 }
 606
 607 static struct netkit_link *netkit_link(const struct bpf_link *link)
 608 {
 609         return container_of(link, struct netkit_link, link);
 610 }
 611
 612 static int netkit_link_prog_attach(struct bpf_link *link, u32 flags,
 613                                    u32 id_or_fd, u64 revision)
 614 {
 615         struct netkit_link *nkl = netkit_link(link);
 616         struct bpf_mprog_entry *entry, *entry_new;
 617         struct net_device *dev = nkl->dev;
 618         int ret;
 619
 620         ASSERT_RTNL();
 621         entry = netkit_entry_fetch(dev, true);
 622         ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
 623                                id_or_fd, revision);
 624         if (!ret) {
 625                 if (entry != entry_new) {
 626                         netkit_entry_update(dev, entry_new);
 627                         netkit_entry_sync();
 628                 }
 629                 bpf_mprog_commit(entry);
 630         }
 631         return ret;
 632 }
 633
 634 static void netkit_link_release(struct bpf_link *link)
 635 {
 636         struct netkit_link *nkl = netkit_link(link);
 637         struct bpf_mprog_entry *entry, *entry_new;
 638         struct net_device *dev;
 639         int ret = 0;
 640
 641         rtnl_lock();
 642         dev = nkl->dev;
 643         if (!dev)
 644                 goto out;
 645         entry = netkit_entry_fetch(dev, false);
 646         if (!entry) {
 647                 ret = -ENOENT;
 648                 goto out;
 649         }
 650         ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
 651         if (!ret) {
 652                 if (!bpf_mprog_total(entry_new))
 653                         entry_new = NULL;
 654                 netkit_entry_update(dev, entry_new);
 655                 netkit_entry_sync();
 656                 bpf_mprog_commit(entry);
 657                 nkl->dev = NULL;
 658         }
 659 out:
 660         WARN_ON_ONCE(ret);
 661         rtnl_unlock();
 662 }
 663
 664 static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog,
 665                               struct bpf_prog *oprog)
 666 {
 667         struct netkit_link *nkl = netkit_link(link);
 668         struct bpf_mprog_entry *entry, *entry_new;
 669         struct net_device *dev;
 670         int ret = 0;
 671
 672         rtnl_lock();
 673         dev = nkl->dev;
 674         if (!dev) {
 675                 ret = -ENOLINK;
 676                 goto out;
 677         }
 678         if (oprog && link->prog != oprog) {
 679                 ret = -EPERM;
 680                 goto out;
 681         }
 682         oprog = link->prog;
 683         if (oprog == nprog) {
 684                 bpf_prog_put(nprog);
 685                 goto out;
 686         }
 687         entry = netkit_entry_fetch(dev, false);
 688         if (!entry) {
 689                 ret = -ENOENT;
 690                 goto out;
 691         }
 692         ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
 693                                BPF_F_REPLACE | BPF_F_ID,
 694                                link->prog->aux->id, 0);
 695         if (!ret) {
 696                 WARN_ON_ONCE(entry != entry_new);
 697                 oprog = xchg(&link->prog, nprog);
 698                 bpf_prog_put(oprog);
 699                 bpf_mprog_commit(entry);
 700         }
 701 out:
 702         rtnl_unlock();
 703         return ret;
 704 }
 705
 706 static void netkit_link_dealloc(struct bpf_link *link)
 707 {
 708         kfree(netkit_link(link));
 709 }
 710
 711 static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
 712 {
 713         const struct netkit_link *nkl = netkit_link(link);
 714         u32 ifindex = 0;
 715
 716         rtnl_lock();
 717         if (nkl->dev)
 718                 ifindex = nkl->dev->ifindex;
 719         rtnl_unlock();
 720
 721         seq_printf(seq, "ifindex:\t%u\n", ifindex);
 722         seq_printf(seq, "attach_type:\t%u (%s)\n",
 723                    nkl->location,
 724                    nkl->location == BPF_NETKIT_PRIMARY ? "primary" : "peer");
 725 }
 726
 727 static int netkit_link_fill_info(const struct bpf_link *link,
 728                                  struct bpf_link_info *info)
 729 {
 730         const struct netkit_link *nkl = netkit_link(link);
 731         u32 ifindex = 0;
 732
 733         rtnl_lock();
 734         if (nkl->dev)
 735                 ifindex = nkl->dev->ifindex;
 736         rtnl_unlock();
 737
 738         info->netkit.ifindex = ifindex;
 739         info->netkit.attach_type = nkl->location;
 740         return 0;
 741 }
 742
 743 static int netkit_link_detach(struct bpf_link *link)
 744 {
 745         netkit_link_release(link);
 746         return 0;
 747 }
 748
 749 static const struct bpf_link_ops netkit_link_lops = {
 750         .release        = netkit_link_release,
 751         .detach         = netkit_link_detach,
 752         .dealloc        = netkit_link_dealloc,
 753         .update_prog    = netkit_link_update,
 754         .show_fdinfo    = netkit_link_fdinfo,
 755         .fill_link_info = netkit_link_fill_info,
 756 };
 757
 758 static int netkit_link_init(struct netkit_link *nkl,
 759                             struct bpf_link_primer *link_primer,
 760                             const union bpf_attr *attr,
 761                             struct net_device *dev,
 762                             struct bpf_prog *prog)
 763 {
 764         bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
 765                       &netkit_link_lops, prog);
 766         nkl->location = attr->link_create.attach_type;
 767         nkl->dev = dev;
 768         return bpf_link_prime(&nkl->link, link_primer);
 769 }
 770
 771 int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 772 {
 773         struct bpf_link_primer link_primer;
 774         struct netkit_link *nkl;
 775         struct net_device *dev;
 776         int ret;
 777
 778         rtnl_lock();
 779         dev = netkit_dev_fetch(current->nsproxy->net_ns,
 780                                attr->link_create.target_ifindex,
 781                                attr->link_create.attach_type);
 782         if (IS_ERR(dev)) {
 783                 ret = PTR_ERR(dev);
 784                 goto out;
 785         }
 786         nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT);
 787         if (!nkl) {
 788                 ret = -ENOMEM;
 789                 goto out;
 790         }
 791         ret = netkit_link_init(nkl, &link_primer, attr, dev, prog);
 792         if (ret) {
 793                 kfree(nkl);
 794                 goto out;
 795         }
 796         ret = netkit_link_prog_attach(&nkl->link,
 797                                       attr->link_create.flags,
 798                                       attr->link_create.netkit.relative_fd,
 799                                       attr->link_create.netkit.expected_revision);
 800         if (ret) {
 801                 nkl->dev = NULL;
 802                 bpf_link_cleanup(&link_primer);
 803                 goto out;
 804         }
 805         ret = bpf_link_settle(&link_primer);
 806 out:
 807         rtnl_unlock();
 808         return ret;
 809 }
 810
 811 static void netkit_release_all(struct net_device *dev)
 812 {
 813         struct bpf_mprog_entry *entry;
 814         struct bpf_tuple tuple = {};
 815         struct bpf_mprog_fp *fp;
 816         struct bpf_mprog_cp *cp;
 817
 818         entry = netkit_entry_fetch(dev, false);
 819         if (!entry)
 820                 return;
 821         netkit_entry_update(dev, NULL);
 822         netkit_entry_sync();
 823         bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
 824                 if (tuple.link)
 825                         netkit_link(tuple.link)->dev = NULL;
 826                 else
 827                         bpf_prog_put(tuple.prog);
 828         }
 829 }
 830
 831 static void netkit_uninit(struct net_device *dev)
 832 {
 833         netkit_release_all(dev);
 834 }
 835
 836 static void netkit_del_link(struct net_device *dev, struct list_head *head)
 837 {
 838         struct netkit *nk = netkit_priv(dev);
 839         struct net_device *peer = rtnl_dereference(nk->peer);
 840
 841         RCU_INIT_POINTER(nk->peer, NULL);
 842         unregister_netdevice_queue(dev, head);
 843         if (peer) {
 844                 nk = netkit_priv(peer);
 845                 RCU_INIT_POINTER(nk->peer, NULL);
 846                 unregister_netdevice_queue(peer, head);
 847         }
 848 }
 849
 850 static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
 851                               struct nlattr *data[],
 852                               struct netlink_ext_ack *extack)
 853 {
 854         struct netkit *nk = netkit_priv(dev);
 855         struct net_device *peer = rtnl_dereference(nk->peer);
 856         enum netkit_action policy;
 857         struct nlattr *attr;
 858         int err;
 859
 860         if (!nk->primary) {
 861                 NL_SET_ERR_MSG(extack,
 862                                "netkit link settings can be changed only through the primary device");
 863                 return -EACCES;
 864         }
 865
 866         if (data[IFLA_NETKIT_MODE]) {
 867                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_MODE],
 868                                     "netkit link operating mode cannot be changed after device creation");
 869                 return -EACCES;
 870         }
 871
 872         if (data[IFLA_NETKIT_PEER_INFO]) {
 873                 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_INFO],
 874                                     "netkit peer info cannot be changed after device creation");
 875                 return -EINVAL;
 876         }
 877
 878         if (data[IFLA_NETKIT_POLICY]) {
 879                 attr = data[IFLA_NETKIT_POLICY];
 880                 policy = nla_get_u32(attr);
 881                 err = netkit_check_policy(policy, attr, extack);
 882                 if (err)
 883                         return err;
 884                 WRITE_ONCE(nk->policy, policy);
 885         }
 886
 887         if (data[IFLA_NETKIT_PEER_POLICY]) {
 888                 err = -EOPNOTSUPP;
 889                 attr = data[IFLA_NETKIT_PEER_POLICY];
 890                 policy = nla_get_u32(attr);
 891                 if (peer)
 892                         err = netkit_check_policy(policy, attr, extack);
 893                 if (err)
 894                         return err;
 895                 nk = netkit_priv(peer);
 896                 WRITE_ONCE(nk->policy, policy);
 897         }
 898
 899         return 0;
 900 }
 901
 902 static size_t netkit_get_size(const struct net_device *dev)
 903 {
 904         return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */
 905                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */
 906                nla_total_size(sizeof(u8))  + /* IFLA_NETKIT_PRIMARY */
 907                nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */
 908                0;
 909 }
 910
 911 static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
 912 {
 913         struct netkit *nk = netkit_priv(dev);
 914         struct net_device *peer = rtnl_dereference(nk->peer);
 915
 916         if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary))
 917                 return -EMSGSIZE;
 918         if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy))
 919                 return -EMSGSIZE;
 920         if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode))
 921                 return -EMSGSIZE;
 922
 923         if (peer) {
 924                 nk = netkit_priv(peer);
 925                 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy))
 926                         return -EMSGSIZE;
 927         }
 928
 929         return 0;
 930 }
 931
 932 static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = {
 933         [IFLA_NETKIT_PEER_INFO]         = { .len = sizeof(struct ifinfomsg) },
 934         [IFLA_NETKIT_POLICY]            = { .type = NLA_U32 },
 935         [IFLA_NETKIT_MODE]              = { .type = NLA_U32 },
 936         [IFLA_NETKIT_PEER_POLICY]       = { .type = NLA_U32 },
 937         [IFLA_NETKIT_PRIMARY]           = { .type = NLA_REJECT,
 938                                             .reject_message = "Primary attribute is read-only" },
 939 };
 940
 941 static struct rtnl_link_ops netkit_link_ops = {
 942         .kind           = DRV_NAME,
 943         .priv_size      = sizeof(struct netkit),
 944         .setup          = netkit_setup,
 945         .newlink        = netkit_new_link,
 946         .dellink        = netkit_del_link,
 947         .changelink     = netkit_change_link,
 948         .get_link_net   = netkit_get_link_net,
 949         .get_size       = netkit_get_size,
 950         .fill_info      = netkit_fill_info,
 951         .policy         = netkit_policy,
 952         .validate       = netkit_validate,
 953         .maxtype        = IFLA_NETKIT_MAX,
 954 };
 955
 956 static __init int netkit_init(void)
 957 {
 958         BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
 959                      (int)NETKIT_PASS != (int)TCX_PASS ||
 960                      (int)NETKIT_DROP != (int)TCX_DROP ||
 961                      (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
 962
 963         return rtnl_link_register(&netkit_link_ops);
 964 }
 965
 966 static __exit void netkit_exit(void)
 967 {
 968         rtnl_link_unregister(&netkit_link_ops);
 969 }
 970
 971 module_init(netkit_init);
 972 module_exit(netkit_exit);
 973
 974 MODULE_DESCRIPTION("BPF-programmable network device");
 975 MODULE_AUTHOR("Daniel Borkmann <[email protected]>");
 976 MODULE_AUTHOR("Nikolay Aleksandrov <[email protected]>");
 977 MODULE_LICENSE("GPL");
 978 MODULE_ALIAS_RTNL_LINK(DRV_NAME);