net/ipv4/nexthop.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Generic nexthop implementation
   3  *
   4  * Copyright (c) 2017-19 Cumulus Networks
   5  * Copyright (c) 2017-19 David Ahern <[email protected]>
   6  */
   7
   8 #include <linux/nexthop.h>
   9 #include <linux/rtnetlink.h>
  10 #include <linux/slab.h>
  11 #include <linux/vmalloc.h>
  12 #include <net/arp.h>
  13 #include <net/ipv6_stubs.h>
  14 #include <net/lwtunnel.h>
  15 #include <net/ndisc.h>
  16 #include <net/nexthop.h>
  17 #include <net/route.h>
  18 #include <net/sock.h>
  19
  20 #define NH_RES_DEFAULT_IDLE_TIMER       (120 * HZ)
  21 #define NH_RES_DEFAULT_UNBALANCED_TIMER 0       /* No forced rebalancing. */
  22
  23 static void remove_nexthop(struct net *net, struct nexthop *nh,
  24                            struct nl_info *nlinfo);
  25
  26 #define NH_DEV_HASHBITS  8
  27 #define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)
  28
  29 #define NHA_OP_FLAGS_DUMP_ALL (NHA_OP_FLAG_DUMP_STATS |         \
  30                                NHA_OP_FLAG_DUMP_HW_STATS)
  31
  32 static const struct nla_policy rtm_nh_policy_new[] = {
  33         [NHA_ID]                = { .type = NLA_U32 },
  34         [NHA_GROUP]             = { .type = NLA_BINARY },
  35         [NHA_GROUP_TYPE]        = { .type = NLA_U16 },
  36         [NHA_BLACKHOLE]         = { .type = NLA_FLAG },
  37         [NHA_OIF]               = { .type = NLA_U32 },
  38         [NHA_GATEWAY]           = { .type = NLA_BINARY },
  39         [NHA_ENCAP_TYPE]        = { .type = NLA_U16 },
  40         [NHA_ENCAP]             = { .type = NLA_NESTED },
  41         [NHA_FDB]               = { .type = NLA_FLAG },
  42         [NHA_RES_GROUP]         = { .type = NLA_NESTED },
  43         [NHA_HW_STATS_ENABLE]   = NLA_POLICY_MAX(NLA_U32, true),
  44 };
  45
  46 static const struct nla_policy rtm_nh_policy_get[] = {
  47         [NHA_ID]                = { .type = NLA_U32 },
  48         [NHA_OP_FLAGS]          = NLA_POLICY_MASK(NLA_U32,
  49                                                   NHA_OP_FLAGS_DUMP_ALL),
  50 };
  51
  52 static const struct nla_policy rtm_nh_policy_del[] = {
  53         [NHA_ID]                = { .type = NLA_U32 },
  54 };
  55
  56 static const struct nla_policy rtm_nh_policy_dump[] = {
  57         [NHA_OIF]               = { .type = NLA_U32 },
  58         [NHA_GROUPS]            = { .type = NLA_FLAG },
  59         [NHA_MASTER]            = { .type = NLA_U32 },
  60         [NHA_FDB]               = { .type = NLA_FLAG },
  61         [NHA_OP_FLAGS]          = NLA_POLICY_MASK(NLA_U32,
  62                                                   NHA_OP_FLAGS_DUMP_ALL),
  63 };
  64
  65 static const struct nla_policy rtm_nh_res_policy_new[] = {
  66         [NHA_RES_GROUP_BUCKETS]                 = { .type = NLA_U16 },
  67         [NHA_RES_GROUP_IDLE_TIMER]              = { .type = NLA_U32 },
  68         [NHA_RES_GROUP_UNBALANCED_TIMER]        = { .type = NLA_U32 },
  69 };
  70
  71 static const struct nla_policy rtm_nh_policy_dump_bucket[] = {
  72         [NHA_ID]                = { .type = NLA_U32 },
  73         [NHA_OIF]               = { .type = NLA_U32 },
  74         [NHA_MASTER]            = { .type = NLA_U32 },
  75         [NHA_RES_BUCKET]        = { .type = NLA_NESTED },
  76 };
  77
  78 static const struct nla_policy rtm_nh_res_bucket_policy_dump[] = {
  79         [NHA_RES_BUCKET_NH_ID]  = { .type = NLA_U32 },
  80 };
  81
  82 static const struct nla_policy rtm_nh_policy_get_bucket[] = {
  83         [NHA_ID]                = { .type = NLA_U32 },
  84         [NHA_RES_BUCKET]        = { .type = NLA_NESTED },
  85 };
  86
  87 static const struct nla_policy rtm_nh_res_bucket_policy_get[] = {
  88         [NHA_RES_BUCKET_INDEX]  = { .type = NLA_U16 },
  89 };
  90
  91 static bool nexthop_notifiers_is_empty(struct net *net)
  92 {
  93         return !net->nexthop.notifier_chain.head;
  94 }
  95
  96 static void
  97 __nh_notifier_single_info_init(struct nh_notifier_single_info *nh_info,
  98                                const struct nh_info *nhi)
  99 {
 100         nh_info->dev = nhi->fib_nhc.nhc_dev;
 101         nh_info->gw_family = nhi->fib_nhc.nhc_gw_family;
 102         if (nh_info->gw_family == AF_INET)
 103                 nh_info->ipv4 = nhi->fib_nhc.nhc_gw.ipv4;
 104         else if (nh_info->gw_family == AF_INET6)
 105                 nh_info->ipv6 = nhi->fib_nhc.nhc_gw.ipv6;
 106
 107         nh_info->id = nhi->nh_parent->id;
 108         nh_info->is_reject = nhi->reject_nh;
 109         nh_info->is_fdb = nhi->fdb_nh;
 110         nh_info->has_encap = !!nhi->fib_nhc.nhc_lwtstate;
 111 }
 112
 113 static int nh_notifier_single_info_init(struct nh_notifier_info *info,
 114                                         const struct nexthop *nh)
 115 {
 116         struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 117
 118         info->type = NH_NOTIFIER_INFO_TYPE_SINGLE;
 119         info->nh = kzalloc(sizeof(*info->nh), GFP_KERNEL);
 120         if (!info->nh)
 121                 return -ENOMEM;
 122
 123         __nh_notifier_single_info_init(info->nh, nhi);
 124
 125         return 0;
 126 }
 127
 128 static void nh_notifier_single_info_fini(struct nh_notifier_info *info)
 129 {
 130         kfree(info->nh);
 131 }
 132
 133 static int nh_notifier_mpath_info_init(struct nh_notifier_info *info,
 134                                        struct nh_group *nhg)
 135 {
 136         u16 num_nh = nhg->num_nh;
 137         int i;
 138
 139         info->type = NH_NOTIFIER_INFO_TYPE_GRP;
 140         info->nh_grp = kzalloc(struct_size(info->nh_grp, nh_entries, num_nh),
 141                                GFP_KERNEL);
 142         if (!info->nh_grp)
 143                 return -ENOMEM;
 144
 145         info->nh_grp->num_nh = num_nh;
 146         info->nh_grp->is_fdb = nhg->fdb_nh;
 147         info->nh_grp->hw_stats = nhg->hw_stats;
 148
 149         for (i = 0; i < num_nh; i++) {
 150                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 151                 struct nh_info *nhi;
 152
 153                 nhi = rtnl_dereference(nhge->nh->nh_info);
 154                 info->nh_grp->nh_entries[i].weight = nhge->weight;
 155                 __nh_notifier_single_info_init(&info->nh_grp->nh_entries[i].nh,
 156                                                nhi);
 157         }
 158
 159         return 0;
 160 }
 161
 162 static int nh_notifier_res_table_info_init(struct nh_notifier_info *info,
 163                                            struct nh_group *nhg)
 164 {
 165         struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
 166         u16 num_nh_buckets = res_table->num_nh_buckets;
 167         unsigned long size;
 168         u16 i;
 169
 170         info->type = NH_NOTIFIER_INFO_TYPE_RES_TABLE;
 171         size = struct_size(info->nh_res_table, nhs, num_nh_buckets);
 172         info->nh_res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO |
 173                                        __GFP_NOWARN);
 174         if (!info->nh_res_table)
 175                 return -ENOMEM;
 176
 177         info->nh_res_table->num_nh_buckets = num_nh_buckets;
 178         info->nh_res_table->hw_stats = nhg->hw_stats;
 179
 180         for (i = 0; i < num_nh_buckets; i++) {
 181                 struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
 182                 struct nh_grp_entry *nhge;
 183                 struct nh_info *nhi;
 184
 185                 nhge = rtnl_dereference(bucket->nh_entry);
 186                 nhi = rtnl_dereference(nhge->nh->nh_info);
 187                 __nh_notifier_single_info_init(&info->nh_res_table->nhs[i],
 188                                                nhi);
 189         }
 190
 191         return 0;
 192 }
 193
 194 static int nh_notifier_grp_info_init(struct nh_notifier_info *info,
 195                                      const struct nexthop *nh)
 196 {
 197         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 198
 199         if (nhg->hash_threshold)
 200                 return nh_notifier_mpath_info_init(info, nhg);
 201         else if (nhg->resilient)
 202                 return nh_notifier_res_table_info_init(info, nhg);
 203         return -EINVAL;
 204 }
 205
 206 static void nh_notifier_grp_info_fini(struct nh_notifier_info *info,
 207                                       const struct nexthop *nh)
 208 {
 209         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 210
 211         if (nhg->hash_threshold)
 212                 kfree(info->nh_grp);
 213         else if (nhg->resilient)
 214                 vfree(info->nh_res_table);
 215 }
 216
 217 static int nh_notifier_info_init(struct nh_notifier_info *info,
 218                                  const struct nexthop *nh)
 219 {
 220         info->id = nh->id;
 221
 222         if (nh->is_group)
 223                 return nh_notifier_grp_info_init(info, nh);
 224         else
 225                 return nh_notifier_single_info_init(info, nh);
 226 }
 227
 228 static void nh_notifier_info_fini(struct nh_notifier_info *info,
 229                                   const struct nexthop *nh)
 230 {
 231         if (nh->is_group)
 232                 nh_notifier_grp_info_fini(info, nh);
 233         else
 234                 nh_notifier_single_info_fini(info);
 235 }
 236
 237 static int call_nexthop_notifiers(struct net *net,
 238                                   enum nexthop_event_type event_type,
 239                                   struct nexthop *nh,
 240                                   struct netlink_ext_ack *extack)
 241 {
 242         struct nh_notifier_info info = {
 243                 .net = net,
 244                 .extack = extack,
 245         };
 246         int err;
 247
 248         ASSERT_RTNL();
 249
 250         if (nexthop_notifiers_is_empty(net))
 251                 return 0;
 252
 253         err = nh_notifier_info_init(&info, nh);
 254         if (err) {
 255                 NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
 256                 return err;
 257         }
 258
 259         err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 260                                            event_type, &info);
 261         nh_notifier_info_fini(&info, nh);
 262
 263         return notifier_to_errno(err);
 264 }
 265
 266 static int
 267 nh_notifier_res_bucket_idle_timer_get(const struct nh_notifier_info *info,
 268                                       bool force, unsigned int *p_idle_timer_ms)
 269 {
 270         struct nh_res_table *res_table;
 271         struct nh_group *nhg;
 272         struct nexthop *nh;
 273         int err = 0;
 274
 275         /* When 'force' is false, nexthop bucket replacement is performed
 276          * because the bucket was deemed to be idle. In this case, capable
 277          * listeners can choose to perform an atomic replacement: The bucket is
 278          * only replaced if it is inactive. However, if the idle timer interval
 279          * is smaller than the interval in which a listener is querying
 280          * buckets' activity from the device, then atomic replacement should
 281          * not be tried. Pass the idle timer value to listeners, so that they
 282          * could determine which type of replacement to perform.
 283          */
 284         if (force) {
 285                 *p_idle_timer_ms = 0;
 286                 return 0;
 287         }
 288
 289         rcu_read_lock();
 290
 291         nh = nexthop_find_by_id(info->net, info->id);
 292         if (!nh) {
 293                 err = -EINVAL;
 294                 goto out;
 295         }
 296
 297         nhg = rcu_dereference(nh->nh_grp);
 298         res_table = rcu_dereference(nhg->res_table);
 299         *p_idle_timer_ms = jiffies_to_msecs(res_table->idle_timer);
 300
 301 out:
 302         rcu_read_unlock();
 303
 304         return err;
 305 }
 306
 307 static int nh_notifier_res_bucket_info_init(struct nh_notifier_info *info,
 308                                             u16 bucket_index, bool force,
 309                                             struct nh_info *oldi,
 310                                             struct nh_info *newi)
 311 {
 312         unsigned int idle_timer_ms;
 313         int err;
 314
 315         err = nh_notifier_res_bucket_idle_timer_get(info, force,
 316                                                     &idle_timer_ms);
 317         if (err)
 318                 return err;
 319
 320         info->type = NH_NOTIFIER_INFO_TYPE_RES_BUCKET;
 321         info->nh_res_bucket = kzalloc(sizeof(*info->nh_res_bucket),
 322                                       GFP_KERNEL);
 323         if (!info->nh_res_bucket)
 324                 return -ENOMEM;
 325
 326         info->nh_res_bucket->bucket_index = bucket_index;
 327         info->nh_res_bucket->idle_timer_ms = idle_timer_ms;
 328         info->nh_res_bucket->force = force;
 329         __nh_notifier_single_info_init(&info->nh_res_bucket->old_nh, oldi);
 330         __nh_notifier_single_info_init(&info->nh_res_bucket->new_nh, newi);
 331         return 0;
 332 }
 333
 334 static void nh_notifier_res_bucket_info_fini(struct nh_notifier_info *info)
 335 {
 336         kfree(info->nh_res_bucket);
 337 }
 338
 339 static int __call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
 340                                                u16 bucket_index, bool force,
 341                                                struct nh_info *oldi,
 342                                                struct nh_info *newi,
 343                                                struct netlink_ext_ack *extack)
 344 {
 345         struct nh_notifier_info info = {
 346                 .net = net,
 347                 .extack = extack,
 348                 .id = nhg_id,
 349         };
 350         int err;
 351
 352         if (nexthop_notifiers_is_empty(net))
 353                 return 0;
 354
 355         err = nh_notifier_res_bucket_info_init(&info, bucket_index, force,
 356                                                oldi, newi);
 357         if (err)
 358                 return err;
 359
 360         err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 361                                            NEXTHOP_EVENT_BUCKET_REPLACE, &info);
 362         nh_notifier_res_bucket_info_fini(&info);
 363
 364         return notifier_to_errno(err);
 365 }
 366
 367 /* There are three users of RES_TABLE, and NHs etc. referenced from there:
 368  *
 369  * 1) a collection of callbacks for NH maintenance. This operates under
 370  *    RTNL,
 371  * 2) the delayed work that gradually balances the resilient table,
 372  * 3) and nexthop_select_path(), operating under RCU.
 373  *
 374  * Both the delayed work and the RTNL block are writers, and need to
 375  * maintain mutual exclusion. Since there are only two and well-known
 376  * writers for each table, the RTNL code can make sure it has exclusive
 377  * access thus:
 378  *
 379  * - Have the DW operate without locking;
 380  * - synchronously cancel the DW;
 381  * - do the writing;
 382  * - if the write was not actually a delete, call upkeep, which schedules
 383  *   DW again if necessary.
 384  *
 385  * The functions that are always called from the RTNL context use
 386  * rtnl_dereference(). The functions that can also be called from the DW do
 387  * a raw dereference and rely on the above mutual exclusion scheme.
 388  */
 389 #define nh_res_dereference(p) (rcu_dereference_raw(p))
 390
 391 static int call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
 392                                              u16 bucket_index, bool force,
 393                                              struct nexthop *old_nh,
 394                                              struct nexthop *new_nh,
 395                                              struct netlink_ext_ack *extack)
 396 {
 397         struct nh_info *oldi = nh_res_dereference(old_nh->nh_info);
 398         struct nh_info *newi = nh_res_dereference(new_nh->nh_info);
 399
 400         return __call_nexthop_res_bucket_notifiers(net, nhg_id, bucket_index,
 401                                                    force, oldi, newi, extack);
 402 }
 403
 404 static int call_nexthop_res_table_notifiers(struct net *net, struct nexthop *nh,
 405                                             struct netlink_ext_ack *extack)
 406 {
 407         struct nh_notifier_info info = {
 408                 .net = net,
 409                 .extack = extack,
 410                 .id = nh->id,
 411         };
 412         struct nh_group *nhg;
 413         int err;
 414
 415         ASSERT_RTNL();
 416
 417         if (nexthop_notifiers_is_empty(net))
 418                 return 0;
 419
 420         /* At this point, the nexthop buckets are still not populated. Only
 421          * emit a notification with the logical nexthops, so that a listener
 422          * could potentially veto it in case of unsupported configuration.
 423          */
 424         nhg = rtnl_dereference(nh->nh_grp);
 425         err = nh_notifier_mpath_info_init(&info, nhg);
 426         if (err) {
 427                 NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
 428                 return err;
 429         }
 430
 431         err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 432                                            NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE,
 433                                            &info);
 434         kfree(info.nh_grp);
 435
 436         return notifier_to_errno(err);
 437 }
 438
 439 static int call_nexthop_notifier(struct notifier_block *nb, struct net *net,
 440                                  enum nexthop_event_type event_type,
 441                                  struct nexthop *nh,
 442                                  struct netlink_ext_ack *extack)
 443 {
 444         struct nh_notifier_info info = {
 445                 .net = net,
 446                 .extack = extack,
 447         };
 448         int err;
 449
 450         err = nh_notifier_info_init(&info, nh);
 451         if (err)
 452                 return err;
 453
 454         err = nb->notifier_call(nb, event_type, &info);
 455         nh_notifier_info_fini(&info, nh);
 456
 457         return notifier_to_errno(err);
 458 }
 459
 460 static unsigned int nh_dev_hashfn(unsigned int val)
 461 {
 462         unsigned int mask = NH_DEV_HASHSIZE - 1;
 463
 464         return (val ^
 465                 (val >> NH_DEV_HASHBITS) ^
 466                 (val >> (NH_DEV_HASHBITS * 2))) & mask;
 467 }
 468
 469 static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
 470 {
 471         struct net_device *dev = nhi->fib_nhc.nhc_dev;
 472         struct hlist_head *head;
 473         unsigned int hash;
 474
 475         WARN_ON(!dev);
 476
 477         hash = nh_dev_hashfn(dev->ifindex);
 478         head = &net->nexthop.devhash[hash];
 479         hlist_add_head(&nhi->dev_hash, head);
 480 }
 481
 482 static void nexthop_free_group(struct nexthop *nh)
 483 {
 484         struct nh_group *nhg;
 485         int i;
 486
 487         nhg = rcu_dereference_raw(nh->nh_grp);
 488         for (i = 0; i < nhg->num_nh; ++i) {
 489                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 490
 491                 WARN_ON(!list_empty(&nhge->nh_list));
 492                 free_percpu(nhge->stats);
 493                 nexthop_put(nhge->nh);
 494         }
 495
 496         WARN_ON(nhg->spare == nhg);
 497
 498         if (nhg->resilient)
 499                 vfree(rcu_dereference_raw(nhg->res_table));
 500
 501         kfree(nhg->spare);
 502         kfree(nhg);
 503 }
 504
 505 static void nexthop_free_single(struct nexthop *nh)
 506 {
 507         struct nh_info *nhi;
 508
 509         nhi = rcu_dereference_raw(nh->nh_info);
 510         switch (nhi->family) {
 511         case AF_INET:
 512                 fib_nh_release(nh->net, &nhi->fib_nh);
 513                 break;
 514         case AF_INET6:
 515                 ipv6_stub->fib6_nh_release(&nhi->fib6_nh);
 516                 break;
 517         }
 518         kfree(nhi);
 519 }
 520
 521 void nexthop_free_rcu(struct rcu_head *head)
 522 {
 523         struct nexthop *nh = container_of(head, struct nexthop, rcu);
 524
 525         if (nh->is_group)
 526                 nexthop_free_group(nh);
 527         else
 528                 nexthop_free_single(nh);
 529
 530         kfree(nh);
 531 }
 532 EXPORT_SYMBOL_GPL(nexthop_free_rcu);
 533
 534 static struct nexthop *nexthop_alloc(void)
 535 {
 536         struct nexthop *nh;
 537
 538         nh = kzalloc(sizeof(struct nexthop), GFP_KERNEL);
 539         if (nh) {
 540                 INIT_LIST_HEAD(&nh->fi_list);
 541                 INIT_LIST_HEAD(&nh->f6i_list);
 542                 INIT_LIST_HEAD(&nh->grp_list);
 543                 INIT_LIST_HEAD(&nh->fdb_list);
 544         }
 545         return nh;
 546 }
 547
 548 static struct nh_group *nexthop_grp_alloc(u16 num_nh)
 549 {
 550         struct nh_group *nhg;
 551
 552         nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);
 553         if (nhg)
 554                 nhg->num_nh = num_nh;
 555
 556         return nhg;
 557 }
 558
 559 static void nh_res_table_upkeep_dw(struct work_struct *work);
 560
 561 static struct nh_res_table *
 562 nexthop_res_table_alloc(struct net *net, u32 nhg_id, struct nh_config *cfg)
 563 {
 564         const u16 num_nh_buckets = cfg->nh_grp_res_num_buckets;
 565         struct nh_res_table *res_table;
 566         unsigned long size;
 567
 568         size = struct_size(res_table, nh_buckets, num_nh_buckets);
 569         res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
 570         if (!res_table)
 571                 return NULL;
 572
 573         res_table->net = net;
 574         res_table->nhg_id = nhg_id;
 575         INIT_DELAYED_WORK(&res_table->upkeep_dw, &nh_res_table_upkeep_dw);
 576         INIT_LIST_HEAD(&res_table->uw_nh_entries);
 577         res_table->idle_timer = cfg->nh_grp_res_idle_timer;
 578         res_table->unbalanced_timer = cfg->nh_grp_res_unbalanced_timer;
 579         res_table->num_nh_buckets = num_nh_buckets;
 580         return res_table;
 581 }
 582
 583 static void nh_base_seq_inc(struct net *net)
 584 {
 585         while (++net->nexthop.seq == 0)
 586                 ;
 587 }
 588
 589 /* no reference taken; rcu lock or rtnl must be held */
 590 struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
 591 {
 592         struct rb_node **pp, *parent = NULL, *next;
 593
 594         pp = &net->nexthop.rb_root.rb_node;
 595         while (1) {
 596                 struct nexthop *nh;
 597
 598                 next = rcu_dereference_raw(*pp);
 599                 if (!next)
 600                         break;
 601                 parent = next;
 602
 603                 nh = rb_entry(parent, struct nexthop, rb_node);
 604                 if (id < nh->id)
 605                         pp = &next->rb_left;
 606                 else if (id > nh->id)
 607                         pp = &next->rb_right;
 608                 else
 609                         return nh;
 610         }
 611         return NULL;
 612 }
 613 EXPORT_SYMBOL_GPL(nexthop_find_by_id);
 614
 615 /* used for auto id allocation; called with rtnl held */
 616 static u32 nh_find_unused_id(struct net *net)
 617 {
 618         u32 id_start = net->nexthop.last_id_allocated;
 619
 620         while (1) {
 621                 net->nexthop.last_id_allocated++;
 622                 if (net->nexthop.last_id_allocated == id_start)
 623                         break;
 624
 625                 if (!nexthop_find_by_id(net, net->nexthop.last_id_allocated))
 626                         return net->nexthop.last_id_allocated;
 627         }
 628         return 0;
 629 }
 630
 631 static void nh_res_time_set_deadline(unsigned long next_time,
 632                                      unsigned long *deadline)
 633 {
 634         if (time_before(next_time, *deadline))
 635                 *deadline = next_time;
 636 }
 637
 638 static clock_t nh_res_table_unbalanced_time(struct nh_res_table *res_table)
 639 {
 640         if (list_empty(&res_table->uw_nh_entries))
 641                 return 0;
 642         return jiffies_delta_to_clock_t(jiffies - res_table->unbalanced_since);
 643 }
 644
 645 static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)
 646 {
 647         struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
 648         struct nlattr *nest;
 649
 650         nest = nla_nest_start(skb, NHA_RES_GROUP);
 651         if (!nest)
 652                 return -EMSGSIZE;
 653
 654         if (nla_put_u16(skb, NHA_RES_GROUP_BUCKETS,
 655                         res_table->num_nh_buckets) ||
 656             nla_put_u32(skb, NHA_RES_GROUP_IDLE_TIMER,
 657                         jiffies_to_clock_t(res_table->idle_timer)) ||
 658             nla_put_u32(skb, NHA_RES_GROUP_UNBALANCED_TIMER,
 659                         jiffies_to_clock_t(res_table->unbalanced_timer)) ||
 660             nla_put_u64_64bit(skb, NHA_RES_GROUP_UNBALANCED_TIME,
 661                               nh_res_table_unbalanced_time(res_table),
 662                               NHA_RES_GROUP_PAD))
 663                 goto nla_put_failure;
 664
 665         nla_nest_end(skb, nest);
 666         return 0;
 667
 668 nla_put_failure:
 669         nla_nest_cancel(skb, nest);
 670         return -EMSGSIZE;
 671 }
 672
 673 static void nh_grp_entry_stats_inc(struct nh_grp_entry *nhge)
 674 {
 675         struct nh_grp_entry_stats *cpu_stats;
 676
 677         cpu_stats = get_cpu_ptr(nhge->stats);
 678         u64_stats_update_begin(&cpu_stats->syncp);
 679         u64_stats_inc(&cpu_stats->packets);
 680         u64_stats_update_end(&cpu_stats->syncp);
 681         put_cpu_ptr(cpu_stats);
 682 }
 683
 684 static void nh_grp_entry_stats_read(struct nh_grp_entry *nhge,
 685                                     u64 *ret_packets)
 686 {
 687         int i;
 688
 689         *ret_packets = 0;
 690
 691         for_each_possible_cpu(i) {
 692                 struct nh_grp_entry_stats *cpu_stats;
 693                 unsigned int start;
 694                 u64 packets;
 695
 696                 cpu_stats = per_cpu_ptr(nhge->stats, i);
 697                 do {
 698                         start = u64_stats_fetch_begin(&cpu_stats->syncp);
 699                         packets = u64_stats_read(&cpu_stats->packets);
 700                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, start));
 701
 702                 *ret_packets += packets;
 703         }
 704 }
 705
 706 static int nh_notifier_grp_hw_stats_init(struct nh_notifier_info *info,
 707                                          const struct nexthop *nh)
 708 {
 709         struct nh_group *nhg;
 710         int i;
 711
 712         ASSERT_RTNL();
 713         nhg = rtnl_dereference(nh->nh_grp);
 714
 715         info->id = nh->id;
 716         info->type = NH_NOTIFIER_INFO_TYPE_GRP_HW_STATS;
 717         info->nh_grp_hw_stats = kzalloc(struct_size(info->nh_grp_hw_stats,
 718                                                     stats, nhg->num_nh),
 719                                         GFP_KERNEL);
 720         if (!info->nh_grp_hw_stats)
 721                 return -ENOMEM;
 722
 723         info->nh_grp_hw_stats->num_nh = nhg->num_nh;
 724         for (i = 0; i < nhg->num_nh; i++) {
 725                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 726
 727                 info->nh_grp_hw_stats->stats[i].id = nhge->nh->id;
 728         }
 729
 730         return 0;
 731 }
 732
 733 static void nh_notifier_grp_hw_stats_fini(struct nh_notifier_info *info)
 734 {
 735         kfree(info->nh_grp_hw_stats);
 736 }
 737
 738 void nh_grp_hw_stats_report_delta(struct nh_notifier_grp_hw_stats_info *info,
 739                                   unsigned int nh_idx,
 740                                   u64 delta_packets)
 741 {
 742         info->hw_stats_used = true;
 743         info->stats[nh_idx].packets += delta_packets;
 744 }
 745 EXPORT_SYMBOL(nh_grp_hw_stats_report_delta);
 746
 747 static void nh_grp_hw_stats_apply_update(struct nexthop *nh,
 748                                          struct nh_notifier_info *info)
 749 {
 750         struct nh_group *nhg;
 751         int i;
 752
 753         ASSERT_RTNL();
 754         nhg = rtnl_dereference(nh->nh_grp);
 755
 756         for (i = 0; i < nhg->num_nh; i++) {
 757                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 758
 759                 nhge->packets_hw += info->nh_grp_hw_stats->stats[i].packets;
 760         }
 761 }
 762
 763 static int nh_grp_hw_stats_update(struct nexthop *nh, bool *hw_stats_used)
 764 {
 765         struct nh_notifier_info info = {
 766                 .net = nh->net,
 767         };
 768         struct net *net = nh->net;
 769         int err;
 770
 771         if (nexthop_notifiers_is_empty(net)) {
 772                 *hw_stats_used = false;
 773                 return 0;
 774         }
 775
 776         err = nh_notifier_grp_hw_stats_init(&info, nh);
 777         if (err)
 778                 return err;
 779
 780         err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 781                                            NEXTHOP_EVENT_HW_STATS_REPORT_DELTA,
 782                                            &info);
 783
 784         /* Cache whatever we got, even if there was an error, otherwise the
 785          * successful stats retrievals would get lost.
 786          */
 787         nh_grp_hw_stats_apply_update(nh, &info);
 788         *hw_stats_used = info.nh_grp_hw_stats->hw_stats_used;
 789
 790         nh_notifier_grp_hw_stats_fini(&info);
 791         return notifier_to_errno(err);
 792 }
 793
 794 static int nla_put_nh_group_stats_entry(struct sk_buff *skb,
 795                                         struct nh_grp_entry *nhge,
 796                                         u32 op_flags)
 797 {
 798         struct nlattr *nest;
 799         u64 packets;
 800
 801         nh_grp_entry_stats_read(nhge, &packets);
 802
 803         nest = nla_nest_start(skb, NHA_GROUP_STATS_ENTRY);
 804         if (!nest)
 805                 return -EMSGSIZE;
 806
 807         if (nla_put_u32(skb, NHA_GROUP_STATS_ENTRY_ID, nhge->nh->id) ||
 808             nla_put_uint(skb, NHA_GROUP_STATS_ENTRY_PACKETS,
 809                          packets + nhge->packets_hw))
 810                 goto nla_put_failure;
 811
 812         if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS &&
 813             nla_put_uint(skb, NHA_GROUP_STATS_ENTRY_PACKETS_HW,
 814                          nhge->packets_hw))
 815                 goto nla_put_failure;
 816
 817         nla_nest_end(skb, nest);
 818         return 0;
 819
 820 nla_put_failure:
 821         nla_nest_cancel(skb, nest);
 822         return -EMSGSIZE;
 823 }
 824
 825 static int nla_put_nh_group_stats(struct sk_buff *skb, struct nexthop *nh,
 826                                   u32 op_flags)
 827 {
 828         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 829         struct nlattr *nest;
 830         bool hw_stats_used;
 831         int err;
 832         int i;
 833
 834         if (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats))
 835                 goto err_out;
 836
 837         if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS &&
 838             nhg->hw_stats) {
 839                 err = nh_grp_hw_stats_update(nh, &hw_stats_used);
 840                 if (err)
 841                         goto out;
 842
 843                 if (nla_put_u32(skb, NHA_HW_STATS_USED, hw_stats_used))
 844                         goto err_out;
 845         }
 846
 847         nest = nla_nest_start(skb, NHA_GROUP_STATS);
 848         if (!nest)
 849                 goto err_out;
 850
 851         for (i = 0; i < nhg->num_nh; i++)
 852                 if (nla_put_nh_group_stats_entry(skb, &nhg->nh_entries[i],
 853                                                  op_flags))
 854                         goto cancel_out;
 855
 856         nla_nest_end(skb, nest);
 857         return 0;
 858
 859 cancel_out:
 860         nla_nest_cancel(skb, nest);
 861 err_out:
 862         err = -EMSGSIZE;
 863 out:
 864         return err;
 865 }
 866
 867 static int nla_put_nh_group(struct sk_buff *skb, struct nexthop *nh,
 868                             u32 op_flags)
 869 {
 870         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 871         struct nexthop_grp *p;
 872         size_t len = nhg->num_nh * sizeof(*p);
 873         struct nlattr *nla;
 874         u16 group_type = 0;
 875         int i;
 876
 877         if (nhg->hash_threshold)
 878                 group_type = NEXTHOP_GRP_TYPE_MPATH;
 879         else if (nhg->resilient)
 880                 group_type = NEXTHOP_GRP_TYPE_RES;
 881
 882         if (nla_put_u16(skb, NHA_GROUP_TYPE, group_type))
 883                 goto nla_put_failure;
 884
 885         nla = nla_reserve(skb, NHA_GROUP, len);
 886         if (!nla)
 887                 goto nla_put_failure;
 888
 889         p = nla_data(nla);
 890         for (i = 0; i < nhg->num_nh; ++i) {
 891                 *p++ = (struct nexthop_grp) {
 892                         .id = nhg->nh_entries[i].nh->id,
 893                         .weight = nhg->nh_entries[i].weight - 1,
 894                 };
 895         }
 896
 897         if (nhg->resilient && nla_put_nh_group_res(skb, nhg))
 898                 goto nla_put_failure;
 899
 900         if (op_flags & NHA_OP_FLAG_DUMP_STATS &&
 901             (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats) ||
 902              nla_put_nh_group_stats(skb, nh, op_flags)))
 903                 goto nla_put_failure;
 904
 905         return 0;
 906
 907 nla_put_failure:
 908         return -EMSGSIZE;
 909 }
 910
 911 static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
 912                         int event, u32 portid, u32 seq, unsigned int nlflags,
 913                         u32 op_flags)
 914 {
 915         struct fib6_nh *fib6_nh;
 916         struct fib_nh *fib_nh;
 917         struct nlmsghdr *nlh;
 918         struct nh_info *nhi;
 919         struct nhmsg *nhm;
 920
 921         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
 922         if (!nlh)
 923                 return -EMSGSIZE;
 924
 925         nhm = nlmsg_data(nlh);
 926         nhm->nh_family = AF_UNSPEC;
 927         nhm->nh_flags = nh->nh_flags;
 928         nhm->nh_protocol = nh->protocol;
 929         nhm->nh_scope = 0;
 930         nhm->resvd = 0;
 931
 932         if (nla_put_u32(skb, NHA_ID, nh->id))
 933                 goto nla_put_failure;
 934
 935         if (nh->is_group) {
 936                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 937
 938                 if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))
 939                         goto nla_put_failure;
 940                 if (nla_put_nh_group(skb, nh, op_flags))
 941                         goto nla_put_failure;
 942                 goto out;
 943         }
 944
 945         nhi = rtnl_dereference(nh->nh_info);
 946         nhm->nh_family = nhi->family;
 947         if (nhi->reject_nh) {
 948                 if (nla_put_flag(skb, NHA_BLACKHOLE))
 949                         goto nla_put_failure;
 950                 goto out;
 951         } else if (nhi->fdb_nh) {
 952                 if (nla_put_flag(skb, NHA_FDB))
 953                         goto nla_put_failure;
 954         } else {
 955                 const struct net_device *dev;
 956
 957                 dev = nhi->fib_nhc.nhc_dev;
 958                 if (dev && nla_put_u32(skb, NHA_OIF, dev->ifindex))
 959                         goto nla_put_failure;
 960         }
 961
 962         nhm->nh_scope = nhi->fib_nhc.nhc_scope;
 963         switch (nhi->family) {
 964         case AF_INET:
 965                 fib_nh = &nhi->fib_nh;
 966                 if (fib_nh->fib_nh_gw_family &&
 967                     nla_put_be32(skb, NHA_GATEWAY, fib_nh->fib_nh_gw4))
 968                         goto nla_put_failure;
 969                 break;
 970
 971         case AF_INET6:
 972                 fib6_nh = &nhi->fib6_nh;
 973                 if (fib6_nh->fib_nh_gw_family &&
 974                     nla_put_in6_addr(skb, NHA_GATEWAY, &fib6_nh->fib_nh_gw6))
 975                         goto nla_put_failure;
 976                 break;
 977         }
 978
 979         if (nhi->fib_nhc.nhc_lwtstate &&
 980             lwtunnel_fill_encap(skb, nhi->fib_nhc.nhc_lwtstate,
 981                                 NHA_ENCAP, NHA_ENCAP_TYPE) < 0)
 982                 goto nla_put_failure;
 983
 984 out:
 985         nlmsg_end(skb, nlh);
 986         return 0;
 987
 988 nla_put_failure:
 989         nlmsg_cancel(skb, nlh);
 990         return -EMSGSIZE;
 991 }
 992
 993 static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)
 994 {
 995         return nla_total_size(0) +      /* NHA_RES_GROUP */
 996                 nla_total_size(2) +     /* NHA_RES_GROUP_BUCKETS */
 997                 nla_total_size(4) +     /* NHA_RES_GROUP_IDLE_TIMER */
 998                 nla_total_size(4) +     /* NHA_RES_GROUP_UNBALANCED_TIMER */
 999                 nla_total_size_64bit(8);/* NHA_RES_GROUP_UNBALANCED_TIME */
1000 }
1001
1002 static size_t nh_nlmsg_size_grp(struct nexthop *nh)
1003 {
1004         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
1005         size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;
1006         size_t tot = nla_total_size(sz) +
1007                 nla_total_size(2); /* NHA_GROUP_TYPE */
1008
1009         if (nhg->resilient)
1010                 tot += nh_nlmsg_size_grp_res(nhg);
1011
1012         return tot;
1013 }
1014
1015 static size_t nh_nlmsg_size_single(struct nexthop *nh)
1016 {
1017         struct nh_info *nhi = rtnl_dereference(nh->nh_info);
1018         size_t sz;
1019
1020         /* covers NHA_BLACKHOLE since NHA_OIF and BLACKHOLE
1021          * are mutually exclusive
1022          */
1023         sz = nla_total_size(4);  /* NHA_OIF */
1024
1025         switch (nhi->family) {
1026         case AF_INET:
1027                 if (nhi->fib_nh.fib_nh_gw_family)
1028                         sz += nla_total_size(4);  /* NHA_GATEWAY */
1029                 break;
1030
1031         case AF_INET6:
1032                 /* NHA_GATEWAY */
1033                 if (nhi->fib6_nh.fib_nh_gw_family)
1034                         sz += nla_total_size(sizeof(const struct in6_addr));
1035                 break;
1036         }
1037
1038         if (nhi->fib_nhc.nhc_lwtstate) {
1039                 sz += lwtunnel_get_encap_size(nhi->fib_nhc.nhc_lwtstate);
1040                 sz += nla_total_size(2);  /* NHA_ENCAP_TYPE */
1041         }
1042
1043         return sz;
1044 }
1045
1046 static size_t nh_nlmsg_size(struct nexthop *nh)
1047 {
1048         size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg));
1049
1050         sz += nla_total_size(4); /* NHA_ID */
1051
1052         if (nh->is_group)
1053                 sz += nh_nlmsg_size_grp(nh);
1054         else
1055                 sz += nh_nlmsg_size_single(nh);
1056
1057         return sz;
1058 }
1059
1060 static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
1061 {
1062         unsigned int nlflags = info->nlh ? info->nlh->nlmsg_flags : 0;
1063         u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
1064         struct sk_buff *skb;
1065         int err = -ENOBUFS;
1066
1067         skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any());
1068         if (!skb)
1069                 goto errout;
1070
1071         err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags, 0);
1072         if (err < 0) {
1073                 /* -EMSGSIZE implies BUG in nh_nlmsg_size() */
1074                 WARN_ON(err == -EMSGSIZE);
1075                 kfree_skb(skb);
1076                 goto errout;
1077         }
1078
1079         rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_NEXTHOP,
1080                     info->nlh, gfp_any());
1081         return;
1082 errout:
1083         if (err < 0)
1084                 rtnl_set_sk_err(info->nl_net, RTNLGRP_NEXTHOP, err);
1085 }
1086
1087 static unsigned long nh_res_bucket_used_time(const struct nh_res_bucket *bucket)
1088 {
1089         return (unsigned long)atomic_long_read(&bucket->used_time);
1090 }
1091
1092 static unsigned long
1093 nh_res_bucket_idle_point(const struct nh_res_table *res_table,
1094                          const struct nh_res_bucket *bucket,
1095                          unsigned long now)
1096 {
1097         unsigned long time = nh_res_bucket_used_time(bucket);
1098
1099         /* Bucket was not used since it was migrated. The idle time is now. */
1100         if (time == bucket->migrated_time)
1101                 return now;
1102
1103         return time + res_table->idle_timer;
1104 }
1105
1106 static unsigned long
1107 nh_res_table_unb_point(const struct nh_res_table *res_table)
1108 {
1109         return res_table->unbalanced_since + res_table->unbalanced_timer;
1110 }
1111
1112 static void nh_res_bucket_set_idle(const struct nh_res_table *res_table,
1113                                    struct nh_res_bucket *bucket)
1114 {
1115         unsigned long now = jiffies;
1116
1117         atomic_long_set(&bucket->used_time, (long)now);
1118         bucket->migrated_time = now;
1119 }
1120
1121 static void nh_res_bucket_set_busy(struct nh_res_bucket *bucket)
1122 {
1123         atomic_long_set(&bucket->used_time, (long)jiffies);
1124 }
1125
1126 static clock_t nh_res_bucket_idle_time(const struct nh_res_bucket *bucket)
1127 {
1128         unsigned long used_time = nh_res_bucket_used_time(bucket);
1129
1130         return jiffies_delta_to_clock_t(jiffies - used_time);
1131 }
1132
1133 static int nh_fill_res_bucket(struct sk_buff *skb, struct nexthop *nh,
1134                               struct nh_res_bucket *bucket, u16 bucket_index,
1135                               int event, u32 portid, u32 seq,
1136                               unsigned int nlflags,
1137                               struct netlink_ext_ack *extack)
1138 {
1139         struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
1140         struct nlmsghdr *nlh;
1141         struct nlattr *nest;
1142         struct nhmsg *nhm;
1143
1144         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
1145         if (!nlh)
1146                 return -EMSGSIZE;
1147
1148         nhm = nlmsg_data(nlh);
1149         nhm->nh_family = AF_UNSPEC;
1150         nhm->nh_flags = bucket->nh_flags;
1151         nhm->nh_protocol = nh->protocol;
1152         nhm->nh_scope = 0;
1153         nhm->resvd = 0;
1154
1155         if (nla_put_u32(skb, NHA_ID, nh->id))
1156                 goto nla_put_failure;
1157
1158         nest = nla_nest_start(skb, NHA_RES_BUCKET);
1159         if (!nest)
1160                 goto nla_put_failure;
1161
1162         if (nla_put_u16(skb, NHA_RES_BUCKET_INDEX, bucket_index) ||
1163             nla_put_u32(skb, NHA_RES_BUCKET_NH_ID, nhge->nh->id) ||
1164             nla_put_u64_64bit(skb, NHA_RES_BUCKET_IDLE_TIME,
1165                               nh_res_bucket_idle_time(bucket),
1166                               NHA_RES_BUCKET_PAD))
1167                 goto nla_put_failure_nest;
1168
1169         nla_nest_end(skb, nest);
1170         nlmsg_end(skb, nlh);
1171         return 0;
1172
1173 nla_put_failure_nest:
1174         nla_nest_cancel(skb, nest);
1175 nla_put_failure:
1176         nlmsg_cancel(skb, nlh);
1177         return -EMSGSIZE;
1178 }
1179
1180 static void nexthop_bucket_notify(struct nh_res_table *res_table,
1181                                   u16 bucket_index)
1182 {
1183         struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
1184         struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
1185         struct nexthop *nh = nhge->nh_parent;
1186         struct sk_buff *skb;
1187         int err = -ENOBUFS;
1188
1189         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1190         if (!skb)
1191                 goto errout;
1192
1193         err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
1194                                  RTM_NEWNEXTHOPBUCKET, 0, 0, NLM_F_REPLACE,
1195                                  NULL);
1196         if (err < 0) {
1197                 kfree_skb(skb);
1198                 goto errout;
1199         }
1200
1201         rtnl_notify(skb, nh->net, 0, RTNLGRP_NEXTHOP, NULL, GFP_KERNEL);
1202         return;
1203 errout:
1204         if (err < 0)
1205                 rtnl_set_sk_err(nh->net, RTNLGRP_NEXTHOP, err);
1206 }
1207
1208 static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
1209                            bool *is_fdb, struct netlink_ext_ack *extack)
1210 {
1211         if (nh->is_group) {
1212                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
1213
1214                 /* Nesting groups within groups is not supported. */
1215                 if (nhg->hash_threshold) {
1216                         NL_SET_ERR_MSG(extack,
1217                                        "Hash-threshold group can not be a nexthop within a group");
1218                         return false;
1219                 }
1220                 if (nhg->resilient) {
1221                         NL_SET_ERR_MSG(extack,
1222                                        "Resilient group can not be a nexthop within a group");
1223                         return false;
1224                 }
1225                 *is_fdb = nhg->fdb_nh;
1226         } else {
1227                 struct nh_info *nhi = rtnl_dereference(nh->nh_info);
1228
1229                 if (nhi->reject_nh && npaths > 1) {
1230                         NL_SET_ERR_MSG(extack,
1231                                        "Blackhole nexthop can not be used in a group with more than 1 path");
1232                         return false;
1233                 }
1234                 *is_fdb = nhi->fdb_nh;
1235         }
1236
1237         return true;
1238 }
1239
1240 static int nh_check_attr_fdb_group(struct nexthop *nh, u8 *nh_family,
1241                                    struct netlink_ext_ack *extack)
1242 {
1243         struct nh_info *nhi;
1244
1245         nhi = rtnl_dereference(nh->nh_info);
1246
1247         if (!nhi->fdb_nh) {
1248                 NL_SET_ERR_MSG(extack, "FDB nexthop group can only have fdb nexthops");
1249                 return -EINVAL;
1250         }
1251
1252         if (*nh_family == AF_UNSPEC) {
1253                 *nh_family = nhi->family;
1254         } else if (*nh_family != nhi->family) {
1255                 NL_SET_ERR_MSG(extack, "FDB nexthop group cannot have mixed family nexthops");
1256                 return -EINVAL;
1257         }
1258
1259         return 0;
1260 }
1261
1262 static int nh_check_attr_group(struct net *net,
1263                                struct nlattr *tb[], size_t tb_size,
1264                                u16 nh_grp_type, struct netlink_ext_ack *extack)
1265 {
1266         unsigned int len = nla_len(tb[NHA_GROUP]);
1267         u8 nh_family = AF_UNSPEC;
1268         struct nexthop_grp *nhg;
1269         unsigned int i, j;
1270         u8 nhg_fdb = 0;
1271
1272         if (!len || len & (sizeof(struct nexthop_grp) - 1)) {
1273                 NL_SET_ERR_MSG(extack,
1274                                "Invalid length for nexthop group attribute");
1275                 return -EINVAL;
1276         }
1277
1278         /* convert len to number of nexthop ids */
1279         len /= sizeof(*nhg);
1280
1281         nhg = nla_data(tb[NHA_GROUP]);
1282         for (i = 0; i < len; ++i) {
1283                 if (nhg[i].resvd1 || nhg[i].resvd2) {
1284                         NL_SET_ERR_MSG(extack, "Reserved fields in nexthop_grp must be 0");
1285                         return -EINVAL;
1286                 }
1287                 if (nhg[i].weight > 254) {
1288                         NL_SET_ERR_MSG(extack, "Invalid value for weight");
1289                         return -EINVAL;
1290                 }
1291                 for (j = i + 1; j < len; ++j) {
1292                         if (nhg[i].id == nhg[j].id) {
1293                                 NL_SET_ERR_MSG(extack, "Nexthop id can not be used twice in a group");
1294                                 return -EINVAL;
1295                         }
1296                 }
1297         }
1298
1299         if (tb[NHA_FDB])
1300                 nhg_fdb = 1;
1301         nhg = nla_data(tb[NHA_GROUP]);
1302         for (i = 0; i < len; ++i) {
1303                 struct nexthop *nh;
1304                 bool is_fdb_nh;
1305
1306                 nh = nexthop_find_by_id(net, nhg[i].id);
1307                 if (!nh) {
1308                         NL_SET_ERR_MSG(extack, "Invalid nexthop id");
1309                         return -EINVAL;
1310                 }
1311                 if (!valid_group_nh(nh, len, &is_fdb_nh, extack))
1312                         return -EINVAL;
1313
1314                 if (nhg_fdb && nh_check_attr_fdb_group(nh, &nh_family, extack))
1315                         return -EINVAL;
1316
1317                 if (!nhg_fdb && is_fdb_nh) {
1318                         NL_SET_ERR_MSG(extack, "Non FDB nexthop group cannot have fdb nexthops");
1319                         return -EINVAL;
1320                 }
1321         }
1322         for (i = NHA_GROUP_TYPE + 1; i < tb_size; ++i) {
1323                 if (!tb[i])
1324                         continue;
1325                 switch (i) {
1326                 case NHA_HW_STATS_ENABLE:
1327                 case NHA_FDB:
1328                         continue;
1329                 case NHA_RES_GROUP:
1330                         if (nh_grp_type == NEXTHOP_GRP_TYPE_RES)
1331                                 continue;
1332                         break;
1333                 }
1334                 NL_SET_ERR_MSG(extack,
1335                                "No other attributes can be set in nexthop groups");
1336                 return -EINVAL;
1337         }
1338
1339         return 0;
1340 }
1341
1342 static bool ipv6_good_nh(const struct fib6_nh *nh)
1343 {
1344         int state = NUD_REACHABLE;
1345         struct neighbour *n;
1346
1347         rcu_read_lock();
1348
1349         n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);
1350         if (n)
1351                 state = READ_ONCE(n->nud_state);
1352
1353         rcu_read_unlock();
1354
1355         return !!(state & NUD_VALID);
1356 }
1357
1358 static bool ipv4_good_nh(const struct fib_nh *nh)
1359 {
1360         int state = NUD_REACHABLE;
1361         struct neighbour *n;
1362
1363         rcu_read_lock();
1364
1365         n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
1366                                       (__force u32)nh->fib_nh_gw4);
1367         if (n)
1368                 state = READ_ONCE(n->nud_state);
1369
1370         rcu_read_unlock();
1371
1372         return !!(state & NUD_VALID);
1373 }
1374
1375 static bool nexthop_is_good_nh(const struct nexthop *nh)
1376 {
1377         struct nh_info *nhi = rcu_dereference(nh->nh_info);
1378
1379         switch (nhi->family) {
1380         case AF_INET:
1381                 return ipv4_good_nh(&nhi->fib_nh);
1382         case AF_INET6:
1383                 return ipv6_good_nh(&nhi->fib6_nh);
1384         }
1385
1386         return false;
1387 }
1388
1389 static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash)
1390 {
1391         int i;
1392
1393         for (i = 0; i < nhg->num_nh; i++) {
1394                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1395
1396                 if (hash > atomic_read(&nhge->hthr.upper_bound))
1397                         continue;
1398
1399                 nh_grp_entry_stats_inc(nhge);
1400                 return nhge->nh;
1401         }
1402
1403         WARN_ON_ONCE(1);
1404         return NULL;
1405 }
1406
1407 static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)
1408 {
1409         struct nh_grp_entry *nhge0 = NULL;
1410         int i;
1411
1412         if (nhg->fdb_nh)
1413                 return nexthop_select_path_fdb(nhg, hash);
1414
1415         for (i = 0; i < nhg->num_nh; ++i) {
1416                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1417
1418                 /* nexthops always check if it is good and does
1419                  * not rely on a sysctl for this behavior
1420                  */
1421                 if (!nexthop_is_good_nh(nhge->nh))
1422                         continue;
1423
1424                 if (!nhge0)
1425                         nhge0 = nhge;
1426
1427                 if (hash > atomic_read(&nhge->hthr.upper_bound))
1428                         continue;
1429
1430                 nh_grp_entry_stats_inc(nhge);
1431                 return nhge->nh;
1432         }
1433
1434         if (!nhge0)
1435                 nhge0 = &nhg->nh_entries[0];
1436         nh_grp_entry_stats_inc(nhge0);
1437         return nhge0->nh;
1438 }
1439
1440 static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)
1441 {
1442         struct nh_res_table *res_table = rcu_dereference(nhg->res_table);
1443         u16 bucket_index = hash % res_table->num_nh_buckets;
1444         struct nh_res_bucket *bucket;
1445         struct nh_grp_entry *nhge;
1446
1447         /* nexthop_select_path() is expected to return a non-NULL value, so
1448          * skip protocol validation and just hand out whatever there is.
1449          */
1450         bucket = &res_table->nh_buckets[bucket_index];
1451         nh_res_bucket_set_busy(bucket);
1452         nhge = rcu_dereference(bucket->nh_entry);
1453         nh_grp_entry_stats_inc(nhge);
1454         return nhge->nh;
1455 }
1456
1457 struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
1458 {
1459         struct nh_group *nhg;
1460
1461         if (!nh->is_group)
1462                 return nh;
1463
1464         nhg = rcu_dereference(nh->nh_grp);
1465         if (nhg->hash_threshold)
1466                 return nexthop_select_path_hthr(nhg, hash);
1467         else if (nhg->resilient)
1468                 return nexthop_select_path_res(nhg, hash);
1469
1470         /* Unreachable. */
1471         return NULL;
1472 }
1473 EXPORT_SYMBOL_GPL(nexthop_select_path);
1474
1475 int nexthop_for_each_fib6_nh(struct nexthop *nh,
1476                              int (*cb)(struct fib6_nh *nh, void *arg),
1477                              void *arg)
1478 {
1479         struct nh_info *nhi;
1480         int err;
1481
1482         if (nh->is_group) {
1483                 struct nh_group *nhg;
1484                 int i;
1485
1486                 nhg = rcu_dereference_rtnl(nh->nh_grp);
1487                 for (i = 0; i < nhg->num_nh; i++) {
1488                         struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1489
1490                         nhi = rcu_dereference_rtnl(nhge->nh->nh_info);
1491                         err = cb(&nhi->fib6_nh, arg);
1492                         if (err)
1493                                 return err;
1494                 }
1495         } else {
1496                 nhi = rcu_dereference_rtnl(nh->nh_info);
1497                 err = cb(&nhi->fib6_nh, arg);
1498                 if (err)
1499                         return err;
1500         }
1501
1502         return 0;
1503 }
1504 EXPORT_SYMBOL_GPL(nexthop_for_each_fib6_nh);
1505
1506 static int check_src_addr(const struct in6_addr *saddr,
1507                           struct netlink_ext_ack *extack)
1508 {
1509         if (!ipv6_addr_any(saddr)) {
1510                 NL_SET_ERR_MSG(extack, "IPv6 routes using source address can not use nexthop objects");
1511                 return -EINVAL;
1512         }
1513         return 0;
1514 }
1515
1516 int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
1517                        struct netlink_ext_ack *extack)
1518 {
1519         struct nh_info *nhi;
1520         bool is_fdb_nh;
1521
1522         /* fib6_src is unique to a fib6_info and limits the ability to cache
1523          * routes in fib6_nh within a nexthop that is potentially shared
1524          * across multiple fib entries. If the config wants to use source
1525          * routing it can not use nexthop objects. mlxsw also does not allow
1526          * fib6_src on routes.
1527          */
1528         if (cfg && check_src_addr(&cfg->fc_src, extack) < 0)
1529                 return -EINVAL;
1530
1531         if (nh->is_group) {
1532                 struct nh_group *nhg;
1533
1534                 nhg = rtnl_dereference(nh->nh_grp);
1535                 if (nhg->has_v4)
1536                         goto no_v4_nh;
1537                 is_fdb_nh = nhg->fdb_nh;
1538         } else {
1539                 nhi = rtnl_dereference(nh->nh_info);
1540                 if (nhi->family == AF_INET)
1541                         goto no_v4_nh;
1542                 is_fdb_nh = nhi->fdb_nh;
1543         }
1544
1545         if (is_fdb_nh) {
1546                 NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
1547                 return -EINVAL;
1548         }
1549
1550         return 0;
1551 no_v4_nh:
1552         NL_SET_ERR_MSG(extack, "IPv6 routes can not use an IPv4 nexthop");
1553         return -EINVAL;
1554 }
1555 EXPORT_SYMBOL_GPL(fib6_check_nexthop);
1556
1557 /* if existing nexthop has ipv6 routes linked to it, need
1558  * to verify this new spec works with ipv6
1559  */
1560 static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new,
1561                               struct netlink_ext_ack *extack)
1562 {
1563         struct fib6_info *f6i;
1564
1565         if (list_empty(&old->f6i_list))
1566                 return 0;
1567
1568         list_for_each_entry(f6i, &old->f6i_list, nh_list) {
1569                 if (check_src_addr(&f6i->fib6_src.addr, extack) < 0)
1570                         return -EINVAL;
1571         }
1572
1573         return fib6_check_nexthop(new, NULL, extack);
1574 }
1575
1576 static int nexthop_check_scope(struct nh_info *nhi, u8 scope,
1577                                struct netlink_ext_ack *extack)
1578 {
1579         if (scope == RT_SCOPE_HOST && nhi->fib_nhc.nhc_gw_family) {
1580                 NL_SET_ERR_MSG(extack,
1581                                "Route with host scope can not have a gateway");
1582                 return -EINVAL;
1583         }
1584
1585         if (nhi->fib_nhc.nhc_flags & RTNH_F_ONLINK && scope >= RT_SCOPE_LINK) {
1586                 NL_SET_ERR_MSG(extack, "Scope mismatch with nexthop");
1587                 return -EINVAL;
1588         }
1589
1590         return 0;
1591 }
1592
1593 /* Invoked by fib add code to verify nexthop by id is ok with
1594  * config for prefix; parts of fib_check_nh not done when nexthop
1595  * object is used.
1596  */
1597 int fib_check_nexthop(struct nexthop *nh, u8 scope,
1598                       struct netlink_ext_ack *extack)
1599 {
1600         struct nh_info *nhi;
1601         int err = 0;
1602
1603         if (nh->is_group) {
1604                 struct nh_group *nhg;
1605
1606                 nhg = rtnl_dereference(nh->nh_grp);
1607                 if (nhg->fdb_nh) {
1608                         NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
1609                         err = -EINVAL;
1610                         goto out;
1611                 }
1612
1613                 if (scope == RT_SCOPE_HOST) {
1614                         NL_SET_ERR_MSG(extack, "Route with host scope can not have multiple nexthops");
1615                         err = -EINVAL;
1616                         goto out;
1617                 }
1618
1619                 /* all nexthops in a group have the same scope */
1620                 nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);
1621                 err = nexthop_check_scope(nhi, scope, extack);
1622         } else {
1623                 nhi = rtnl_dereference(nh->nh_info);
1624                 if (nhi->fdb_nh) {
1625                         NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
1626                         err = -EINVAL;
1627                         goto out;
1628                 }
1629                 err = nexthop_check_scope(nhi, scope, extack);
1630         }
1631
1632 out:
1633         return err;
1634 }
1635
1636 static int fib_check_nh_list(struct nexthop *old, struct nexthop *new,
1637                              struct netlink_ext_ack *extack)
1638 {
1639         struct fib_info *fi;
1640
1641         list_for_each_entry(fi, &old->fi_list, nh_list) {
1642                 int err;
1643
1644                 err = fib_check_nexthop(new, fi->fib_scope, extack);
1645                 if (err)
1646                         return err;
1647         }
1648         return 0;
1649 }
1650
1651 static bool nh_res_nhge_is_balanced(const struct nh_grp_entry *nhge)
1652 {
1653         return nhge->res.count_buckets == nhge->res.wants_buckets;
1654 }
1655
1656 static bool nh_res_nhge_is_ow(const struct nh_grp_entry *nhge)
1657 {
1658         return nhge->res.count_buckets > nhge->res.wants_buckets;
1659 }
1660
1661 static bool nh_res_nhge_is_uw(const struct nh_grp_entry *nhge)
1662 {
1663         return nhge->res.count_buckets < nhge->res.wants_buckets;
1664 }
1665
1666 static bool nh_res_table_is_balanced(const struct nh_res_table *res_table)
1667 {
1668         return list_empty(&res_table->uw_nh_entries);
1669 }
1670
1671 static void nh_res_bucket_unset_nh(struct nh_res_bucket *bucket)
1672 {
1673         struct nh_grp_entry *nhge;
1674
1675         if (bucket->occupied) {
1676                 nhge = nh_res_dereference(bucket->nh_entry);
1677                 nhge->res.count_buckets--;
1678                 bucket->occupied = false;
1679         }
1680 }
1681
1682 static void nh_res_bucket_set_nh(struct nh_res_bucket *bucket,
1683                                  struct nh_grp_entry *nhge)
1684 {
1685         nh_res_bucket_unset_nh(bucket);
1686
1687         bucket->occupied = true;
1688         rcu_assign_pointer(bucket->nh_entry, nhge);
1689         nhge->res.count_buckets++;
1690 }
1691
1692 static bool nh_res_bucket_should_migrate(struct nh_res_table *res_table,
1693                                          struct nh_res_bucket *bucket,
1694                                          unsigned long *deadline, bool *force)
1695 {
1696         unsigned long now = jiffies;
1697         struct nh_grp_entry *nhge;
1698         unsigned long idle_point;
1699
1700         if (!bucket->occupied) {
1701                 /* The bucket is not occupied, its NHGE pointer is either
1702                  * NULL or obsolete. We _have to_ migrate: set force.
1703                  */
1704                 *force = true;
1705                 return true;
1706         }
1707
1708         nhge = nh_res_dereference(bucket->nh_entry);
1709
1710         /* If the bucket is populated by an underweight or balanced
1711          * nexthop, do not migrate.
1712          */
1713         if (!nh_res_nhge_is_ow(nhge))
1714                 return false;
1715
1716         /* At this point we know that the bucket is populated with an
1717          * overweight nexthop. It needs to be migrated to a new nexthop if
1718          * the idle timer of unbalanced timer expired.
1719          */
1720
1721         idle_point = nh_res_bucket_idle_point(res_table, bucket, now);
1722         if (time_after_eq(now, idle_point)) {
1723                 /* The bucket is idle. We _can_ migrate: unset force. */
1724                 *force = false;
1725                 return true;
1726         }
1727
1728         /* Unbalanced timer of 0 means "never force". */
1729         if (res_table->unbalanced_timer) {
1730                 unsigned long unb_point;
1731
1732                 unb_point = nh_res_table_unb_point(res_table);
1733                 if (time_after(now, unb_point)) {
1734                         /* The bucket is not idle, but the unbalanced timer
1735                          * expired. We _can_ migrate, but set force anyway,
1736                          * so that drivers know to ignore activity reports
1737                          * from the HW.
1738                          */
1739                         *force = true;
1740                         return true;
1741                 }
1742
1743                 nh_res_time_set_deadline(unb_point, deadline);
1744         }
1745
1746         nh_res_time_set_deadline(idle_point, deadline);
1747         return false;
1748 }
1749
1750 static bool nh_res_bucket_migrate(struct nh_res_table *res_table,
1751                                   u16 bucket_index, bool notify,
1752                                   bool notify_nl, bool force)
1753 {
1754         struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
1755         struct nh_grp_entry *new_nhge;
1756         struct netlink_ext_ack extack;
1757         int err;
1758
1759         new_nhge = list_first_entry_or_null(&res_table->uw_nh_entries,
1760                                             struct nh_grp_entry,
1761                                             res.uw_nh_entry);
1762         if (WARN_ON_ONCE(!new_nhge))
1763                 /* If this function is called, "bucket" is either not
1764                  * occupied, or it belongs to a next hop that is
1765                  * overweight. In either case, there ought to be a
1766                  * corresponding underweight next hop.
1767                  */
1768                 return false;
1769
1770         if (notify) {
1771                 struct nh_grp_entry *old_nhge;
1772
1773                 old_nhge = nh_res_dereference(bucket->nh_entry);
1774                 err = call_nexthop_res_bucket_notifiers(res_table->net,
1775                                                         res_table->nhg_id,
1776                                                         bucket_index, force,
1777                                                         old_nhge->nh,
1778                                                         new_nhge->nh, &extack);
1779                 if (err) {
1780                         pr_err_ratelimited("%s\n", extack._msg);
1781                         if (!force)
1782                                 return false;
1783                         /* It is not possible to veto a forced replacement, so
1784                          * just clear the hardware flags from the nexthop
1785                          * bucket to indicate to user space that this bucket is
1786                          * not correctly populated in hardware.
1787                          */
1788                         bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
1789                 }
1790         }
1791
1792         nh_res_bucket_set_nh(bucket, new_nhge);
1793         nh_res_bucket_set_idle(res_table, bucket);
1794
1795         if (notify_nl)
1796                 nexthop_bucket_notify(res_table, bucket_index);
1797
1798         if (nh_res_nhge_is_balanced(new_nhge))
1799                 list_del(&new_nhge->res.uw_nh_entry);
1800         return true;
1801 }
1802
1803 #define NH_RES_UPKEEP_DW_MINIMUM_INTERVAL (HZ / 2)
1804
1805 static void nh_res_table_upkeep(struct nh_res_table *res_table,
1806                                 bool notify, bool notify_nl)
1807 {
1808         unsigned long now = jiffies;
1809         unsigned long deadline;
1810         u16 i;
1811
1812         /* Deadline is the next time that upkeep should be run. It is the
1813          * earliest time at which one of the buckets might be migrated.
1814          * Start at the most pessimistic estimate: either unbalanced_timer
1815          * from now, or if there is none, idle_timer from now. For each
1816          * encountered time point, call nh_res_time_set_deadline() to
1817          * refine the estimate.
1818          */
1819         if (res_table->unbalanced_timer)
1820                 deadline = now + res_table->unbalanced_timer;
1821         else
1822                 deadline = now + res_table->idle_timer;
1823
1824         for (i = 0; i < res_table->num_nh_buckets; i++) {
1825                 struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
1826                 bool force;
1827
1828                 if (nh_res_bucket_should_migrate(res_table, bucket,
1829                                                  &deadline, &force)) {
1830                         if (!nh_res_bucket_migrate(res_table, i, notify,
1831                                                    notify_nl, force)) {
1832                                 unsigned long idle_point;
1833
1834                                 /* A driver can override the migration
1835                                  * decision if the HW reports that the
1836                                  * bucket is actually not idle. Therefore
1837                                  * remark the bucket as busy again and
1838                                  * update the deadline.
1839                                  */
1840                                 nh_res_bucket_set_busy(bucket);
1841                                 idle_point = nh_res_bucket_idle_point(res_table,
1842                                                                       bucket,
1843                                                                       now);
1844                                 nh_res_time_set_deadline(idle_point, &deadline);
1845                         }
1846                 }
1847         }
1848
1849         /* If the group is still unbalanced, schedule the next upkeep to
1850          * either the deadline computed above, or the minimum deadline,
1851          * whichever comes later.
1852          */
1853         if (!nh_res_table_is_balanced(res_table)) {
1854                 unsigned long now = jiffies;
1855                 unsigned long min_deadline;
1856
1857                 min_deadline = now + NH_RES_UPKEEP_DW_MINIMUM_INTERVAL;
1858                 if (time_before(deadline, min_deadline))
1859                         deadline = min_deadline;
1860
1861                 queue_delayed_work(system_power_efficient_wq,
1862                                    &res_table->upkeep_dw, deadline - now);
1863         }
1864 }
1865
1866 static void nh_res_table_upkeep_dw(struct work_struct *work)
1867 {
1868         struct delayed_work *dw = to_delayed_work(work);
1869         struct nh_res_table *res_table;
1870
1871         res_table = container_of(dw, struct nh_res_table, upkeep_dw);
1872         nh_res_table_upkeep(res_table, true, true);
1873 }
1874
1875 static void nh_res_table_cancel_upkeep(struct nh_res_table *res_table)
1876 {
1877         cancel_delayed_work_sync(&res_table->upkeep_dw);
1878 }
1879
1880 static void nh_res_group_rebalance(struct nh_group *nhg,
1881                                    struct nh_res_table *res_table)
1882 {
1883         int prev_upper_bound = 0;
1884         int total = 0;
1885         int w = 0;
1886         int i;
1887
1888         INIT_LIST_HEAD(&res_table->uw_nh_entries);
1889
1890         for (i = 0; i < nhg->num_nh; ++i)
1891                 total += nhg->nh_entries[i].weight;
1892
1893         for (i = 0; i < nhg->num_nh; ++i) {
1894                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1895                 int upper_bound;
1896
1897                 w += nhge->weight;
1898                 upper_bound = DIV_ROUND_CLOSEST(res_table->num_nh_buckets * w,
1899                                                 total);
1900                 nhge->res.wants_buckets = upper_bound - prev_upper_bound;
1901                 prev_upper_bound = upper_bound;
1902
1903                 if (nh_res_nhge_is_uw(nhge)) {
1904                         if (list_empty(&res_table->uw_nh_entries))
1905                                 res_table->unbalanced_since = jiffies;
1906                         list_add(&nhge->res.uw_nh_entry,
1907                                  &res_table->uw_nh_entries);
1908                 }
1909         }
1910 }
1911
1912 /* Migrate buckets in res_table so that they reference NHGE's from NHG with
1913  * the right NH ID. Set those buckets that do not have a corresponding NHGE
1914  * entry in NHG as not occupied.
1915  */
1916 static void nh_res_table_migrate_buckets(struct nh_res_table *res_table,
1917                                          struct nh_group *nhg)
1918 {
1919         u16 i;
1920
1921         for (i = 0; i < res_table->num_nh_buckets; i++) {
1922                 struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
1923                 u32 id = rtnl_dereference(bucket->nh_entry)->nh->id;
1924                 bool found = false;
1925                 int j;
1926
1927                 for (j = 0; j < nhg->num_nh; j++) {
1928                         struct nh_grp_entry *nhge = &nhg->nh_entries[j];
1929
1930                         if (nhge->nh->id == id) {
1931                                 nh_res_bucket_set_nh(bucket, nhge);
1932                                 found = true;
1933                                 break;
1934                         }
1935                 }
1936
1937                 if (!found)
1938                         nh_res_bucket_unset_nh(bucket);
1939         }
1940 }
1941
1942 static void replace_nexthop_grp_res(struct nh_group *oldg,
1943                                     struct nh_group *newg)
1944 {
1945         /* For NH group replacement, the new NHG might only have a stub
1946          * hash table with 0 buckets, because the number of buckets was not
1947          * specified. For NH removal, oldg and newg both reference the same
1948          * res_table. So in any case, in the following, we want to work
1949          * with oldg->res_table.
1950          */
1951         struct nh_res_table *old_res_table = rtnl_dereference(oldg->res_table);
1952         unsigned long prev_unbalanced_since = old_res_table->unbalanced_since;
1953         bool prev_has_uw = !list_empty(&old_res_table->uw_nh_entries);
1954
1955         nh_res_table_cancel_upkeep(old_res_table);
1956         nh_res_table_migrate_buckets(old_res_table, newg);
1957         nh_res_group_rebalance(newg, old_res_table);
1958         if (prev_has_uw && !list_empty(&old_res_table->uw_nh_entries))
1959                 old_res_table->unbalanced_since = prev_unbalanced_since;
1960         nh_res_table_upkeep(old_res_table, true, false);
1961 }
1962
1963 static void nh_hthr_group_rebalance(struct nh_group *nhg)
1964 {
1965         int total = 0;
1966         int w = 0;
1967         int i;
1968
1969         for (i = 0; i < nhg->num_nh; ++i)
1970                 total += nhg->nh_entries[i].weight;
1971
1972         for (i = 0; i < nhg->num_nh; ++i) {
1973                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1974                 int upper_bound;
1975
1976                 w += nhge->weight;
1977                 upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, total) - 1;
1978                 atomic_set(&nhge->hthr.upper_bound, upper_bound);
1979         }
1980 }
1981
1982 static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
1983                                 struct nl_info *nlinfo)
1984 {
1985         struct nh_grp_entry *nhges, *new_nhges;
1986         struct nexthop *nhp = nhge->nh_parent;
1987         struct netlink_ext_ack extack;
1988         struct nexthop *nh = nhge->nh;
1989         struct nh_group *nhg, *newg;
1990         int i, j, err;
1991
1992         WARN_ON(!nh);
1993
1994         nhg = rtnl_dereference(nhp->nh_grp);
1995         newg = nhg->spare;
1996
1997         /* last entry, keep it visible and remove the parent */
1998         if (nhg->num_nh == 1) {
1999                 remove_nexthop(net, nhp, nlinfo);
2000                 return;
2001         }
2002
2003         newg->has_v4 = false;
2004         newg->is_multipath = nhg->is_multipath;
2005         newg->hash_threshold = nhg->hash_threshold;
2006         newg->resilient = nhg->resilient;
2007         newg->fdb_nh = nhg->fdb_nh;
2008         newg->num_nh = nhg->num_nh;
2009
2010         /* copy old entries to new except the one getting removed */
2011         nhges = nhg->nh_entries;
2012         new_nhges = newg->nh_entries;
2013         for (i = 0, j = 0; i < nhg->num_nh; ++i) {
2014                 struct nh_info *nhi;
2015
2016                 /* current nexthop getting removed */
2017                 if (nhg->nh_entries[i].nh == nh) {
2018                         newg->num_nh--;
2019                         continue;
2020                 }
2021
2022                 nhi = rtnl_dereference(nhges[i].nh->nh_info);
2023                 if (nhi->family == AF_INET)
2024                         newg->has_v4 = true;
2025
2026                 list_del(&nhges[i].nh_list);
2027                 new_nhges[j].stats = nhges[i].stats;
2028                 new_nhges[j].nh_parent = nhges[i].nh_parent;
2029                 new_nhges[j].nh = nhges[i].nh;
2030                 new_nhges[j].weight = nhges[i].weight;
2031                 list_add(&new_nhges[j].nh_list, &new_nhges[j].nh->grp_list);
2032                 j++;
2033         }
2034
2035         if (newg->hash_threshold)
2036                 nh_hthr_group_rebalance(newg);
2037         else if (newg->resilient)
2038                 replace_nexthop_grp_res(nhg, newg);
2039
2040         rcu_assign_pointer(nhp->nh_grp, newg);
2041
2042         list_del(&nhge->nh_list);
2043         free_percpu(nhge->stats);
2044         nexthop_put(nhge->nh);
2045
2046         /* Removal of a NH from a resilient group is notified through
2047          * bucket notifications.
2048          */
2049         if (newg->hash_threshold) {
2050                 err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp,
2051                                              &extack);
2052                 if (err)
2053                         pr_err("%s\n", extack._msg);
2054         }
2055
2056         if (nlinfo)
2057                 nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo);
2058 }
2059
2060 static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
2061                                        struct nl_info *nlinfo)
2062 {
2063         struct nh_grp_entry *nhge, *tmp;
2064
2065         list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list)
2066                 remove_nh_grp_entry(net, nhge, nlinfo);
2067
2068         /* make sure all see the newly published array before releasing rtnl */
2069         synchronize_net();
2070 }
2071
2072 static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
2073 {
2074         struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);
2075         struct nh_res_table *res_table;
2076         int i, num_nh = nhg->num_nh;
2077
2078         for (i = 0; i < num_nh; ++i) {
2079                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
2080
2081                 if (WARN_ON(!nhge->nh))
2082                         continue;
2083
2084                 list_del_init(&nhge->nh_list);
2085         }
2086
2087         if (nhg->resilient) {
2088                 res_table = rtnl_dereference(nhg->res_table);
2089                 nh_res_table_cancel_upkeep(res_table);
2090         }
2091 }
2092
2093 /* not called for nexthop replace */
2094 static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
2095 {
2096         struct fib6_info *f6i, *tmp;
2097         bool do_flush = false;
2098         struct fib_info *fi;
2099
2100         list_for_each_entry(fi, &nh->fi_list, nh_list) {
2101                 fi->fib_flags |= RTNH_F_DEAD;
2102                 do_flush = true;
2103         }
2104         if (do_flush)
2105                 fib_flush(net);
2106
2107         /* ip6_del_rt removes the entry from this list hence the _safe */
2108         list_for_each_entry_safe(f6i, tmp, &nh->f6i_list, nh_list) {
2109                 /* __ip6_del_rt does a release, so do a hold here */
2110                 fib6_info_hold(f6i);
2111                 ipv6_stub->ip6_del_rt(net, f6i,
2112                                       !READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode));
2113         }
2114 }
2115
2116 static void __remove_nexthop(struct net *net, struct nexthop *nh,
2117                              struct nl_info *nlinfo)
2118 {
2119         __remove_nexthop_fib(net, nh);
2120
2121         if (nh->is_group) {
2122                 remove_nexthop_group(nh, nlinfo);
2123         } else {
2124                 struct nh_info *nhi;
2125
2126                 nhi = rtnl_dereference(nh->nh_info);
2127                 if (nhi->fib_nhc.nhc_dev)
2128                         hlist_del(&nhi->dev_hash);
2129
2130                 remove_nexthop_from_groups(net, nh, nlinfo);
2131         }
2132 }
2133
2134 static void remove_nexthop(struct net *net, struct nexthop *nh,
2135                            struct nl_info *nlinfo)
2136 {
2137         call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL);
2138
2139         /* remove from the tree */
2140         rb_erase(&nh->rb_node, &net->nexthop.rb_root);
2141
2142         if (nlinfo)
2143                 nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);
2144
2145         __remove_nexthop(net, nh, nlinfo);
2146         nh_base_seq_inc(net);
2147
2148         nexthop_put(nh);
2149 }
2150
2151 /* if any FIB entries reference this nexthop, any dst entries
2152  * need to be regenerated
2153  */
2154 static void nh_rt_cache_flush(struct net *net, struct nexthop *nh,
2155                               struct nexthop *replaced_nh)
2156 {
2157         struct fib6_info *f6i;
2158         struct nh_group *nhg;
2159         int i;
2160
2161         if (!list_empty(&nh->fi_list))
2162                 rt_cache_flush(net);
2163
2164         list_for_each_entry(f6i, &nh->f6i_list, nh_list)
2165                 ipv6_stub->fib6_update_sernum(net, f6i);
2166
2167         /* if an IPv6 group was replaced, we have to release all old
2168          * dsts to make sure all refcounts are released
2169          */
2170         if (!replaced_nh->is_group)
2171                 return;
2172
2173         nhg = rtnl_dereference(replaced_nh->nh_grp);
2174         for (i = 0; i < nhg->num_nh; i++) {
2175                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
2176                 struct nh_info *nhi = rtnl_dereference(nhge->nh->nh_info);
2177
2178                 if (nhi->family == AF_INET6)
2179                         ipv6_stub->fib6_nh_release_dsts(&nhi->fib6_nh);
2180         }
2181 }
2182
2183 static int replace_nexthop_grp(struct net *net, struct nexthop *old,
2184                                struct nexthop *new, const struct nh_config *cfg,
2185                                struct netlink_ext_ack *extack)
2186 {
2187         struct nh_res_table *tmp_table = NULL;
2188         struct nh_res_table *new_res_table;
2189         struct nh_res_table *old_res_table;
2190         struct nh_group *oldg, *newg;
2191         int i, err;
2192
2193         if (!new->is_group) {
2194                 NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with a nexthop.");
2195                 return -EINVAL;
2196         }
2197
2198         oldg = rtnl_dereference(old->nh_grp);
2199         newg = rtnl_dereference(new->nh_grp);
2200
2201         if (newg->hash_threshold != oldg->hash_threshold) {
2202                 NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with one of a different type.");
2203                 return -EINVAL;
2204         }
2205
2206         if (newg->hash_threshold) {
2207                 err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new,
2208                                              extack);
2209                 if (err)
2210                         return err;
2211         } else if (newg->resilient) {
2212                 new_res_table = rtnl_dereference(newg->res_table);
2213                 old_res_table = rtnl_dereference(oldg->res_table);
2214
2215                 /* Accept if num_nh_buckets was not given, but if it was
2216                  * given, demand that the value be correct.
2217                  */
2218                 if (cfg->nh_grp_res_has_num_buckets &&
2219                     cfg->nh_grp_res_num_buckets !=
2220                     old_res_table->num_nh_buckets) {
2221                         NL_SET_ERR_MSG(extack, "Can not change number of buckets of a resilient nexthop group.");
2222                         return -EINVAL;
2223                 }
2224
2225                 /* Emit a pre-replace notification so that listeners could veto
2226                  * a potentially unsupported configuration. Otherwise,
2227                  * individual bucket replacement notifications would need to be
2228                  * vetoed, which is something that should only happen if the
2229                  * bucket is currently active.
2230                  */
2231                 err = call_nexthop_res_table_notifiers(net, new, extack);
2232                 if (err)
2233                         return err;
2234
2235                 if (cfg->nh_grp_res_has_idle_timer)
2236                         old_res_table->idle_timer = cfg->nh_grp_res_idle_timer;
2237                 if (cfg->nh_grp_res_has_unbalanced_timer)
2238                         old_res_table->unbalanced_timer =
2239                                 cfg->nh_grp_res_unbalanced_timer;
2240
2241                 replace_nexthop_grp_res(oldg, newg);
2242
2243                 tmp_table = new_res_table;
2244                 rcu_assign_pointer(newg->res_table, old_res_table);
2245                 rcu_assign_pointer(newg->spare->res_table, old_res_table);
2246         }
2247
2248         /* update parents - used by nexthop code for cleanup */
2249         for (i = 0; i < newg->num_nh; i++)
2250                 newg->nh_entries[i].nh_parent = old;
2251
2252         rcu_assign_pointer(old->nh_grp, newg);
2253
2254         /* Make sure concurrent readers are not using 'oldg' anymore. */
2255         synchronize_net();
2256
2257         if (newg->resilient) {
2258                 rcu_assign_pointer(oldg->res_table, tmp_table);
2259                 rcu_assign_pointer(oldg->spare->res_table, tmp_table);
2260         }
2261
2262         for (i = 0; i < oldg->num_nh; i++)
2263                 oldg->nh_entries[i].nh_parent = new;
2264
2265         rcu_assign_pointer(new->nh_grp, oldg);
2266
2267         return 0;
2268 }
2269
2270 static void nh_group_v4_update(struct nh_group *nhg)
2271 {
2272         struct nh_grp_entry *nhges;
2273         bool has_v4 = false;
2274         int i;
2275
2276         nhges = nhg->nh_entries;
2277         for (i = 0; i < nhg->num_nh; i++) {
2278                 struct nh_info *nhi;
2279
2280                 nhi = rtnl_dereference(nhges[i].nh->nh_info);
2281                 if (nhi->family == AF_INET)
2282                         has_v4 = true;
2283         }
2284         nhg->has_v4 = has_v4;
2285 }
2286
2287 static int replace_nexthop_single_notify_res(struct net *net,
2288                                              struct nh_res_table *res_table,
2289                                              struct nexthop *old,
2290                                              struct nh_info *oldi,
2291                                              struct nh_info *newi,
2292                                              struct netlink_ext_ack *extack)
2293 {
2294         u32 nhg_id = res_table->nhg_id;
2295         int err;
2296         u16 i;
2297
2298         for (i = 0; i < res_table->num_nh_buckets; i++) {
2299                 struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
2300                 struct nh_grp_entry *nhge;
2301
2302                 nhge = rtnl_dereference(bucket->nh_entry);
2303                 if (nhge->nh == old) {
2304                         err = __call_nexthop_res_bucket_notifiers(net, nhg_id,
2305                                                                   i, true,
2306                                                                   oldi, newi,
2307                                                                   extack);
2308                         if (err)
2309                                 goto err_notify;
2310                 }
2311         }
2312
2313         return 0;
2314
2315 err_notify:
2316         while (i-- > 0) {
2317                 struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
2318                 struct nh_grp_entry *nhge;
2319
2320                 nhge = rtnl_dereference(bucket->nh_entry);
2321                 if (nhge->nh == old)
2322                         __call_nexthop_res_bucket_notifiers(net, nhg_id, i,
2323                                                             true, newi, oldi,
2324                                                             extack);
2325         }
2326         return err;
2327 }
2328
2329 static int replace_nexthop_single_notify(struct net *net,
2330                                          struct nexthop *group_nh,
2331                                          struct nexthop *old,
2332                                          struct nh_info *oldi,
2333                                          struct nh_info *newi,
2334                                          struct netlink_ext_ack *extack)
2335 {
2336         struct nh_group *nhg = rtnl_dereference(group_nh->nh_grp);
2337         struct nh_res_table *res_table;
2338
2339         if (nhg->hash_threshold) {
2340                 return call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE,
2341                                               group_nh, extack);
2342         } else if (nhg->resilient) {
2343                 res_table = rtnl_dereference(nhg->res_table);
2344                 return replace_nexthop_single_notify_res(net, res_table,
2345                                                          old, oldi, newi,
2346                                                          extack);
2347         }
2348
2349         return -EINVAL;
2350 }
2351
2352 static int replace_nexthop_single(struct net *net, struct nexthop *old,
2353                                   struct nexthop *new,
2354                                   struct netlink_ext_ack *extack)
2355 {
2356         u8 old_protocol, old_nh_flags;
2357         struct nh_info *oldi, *newi;
2358         struct nh_grp_entry *nhge;
2359         int err;
2360
2361         if (new->is_group) {
2362                 NL_SET_ERR_MSG(extack, "Can not replace a nexthop with a nexthop group.");
2363                 return -EINVAL;
2364         }
2365
2366         err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
2367         if (err)
2368                 return err;
2369
2370         /* Hardware flags were set on 'old' as 'new' is not in the red-black
2371          * tree. Therefore, inherit the flags from 'old' to 'new'.
2372          */
2373         new->nh_flags |= old->nh_flags & (RTNH_F_OFFLOAD | RTNH_F_TRAP);
2374
2375         oldi = rtnl_dereference(old->nh_info);
2376         newi = rtnl_dereference(new->nh_info);
2377
2378         newi->nh_parent = old;
2379         oldi->nh_parent = new;
2380
2381         old_protocol = old->protocol;
2382         old_nh_flags = old->nh_flags;
2383
2384         old->protocol = new->protocol;
2385         old->nh_flags = new->nh_flags;
2386
2387         rcu_assign_pointer(old->nh_info, newi);
2388         rcu_assign_pointer(new->nh_info, oldi);
2389
2390         /* Send a replace notification for all the groups using the nexthop. */
2391         list_for_each_entry(nhge, &old->grp_list, nh_list) {
2392                 struct nexthop *nhp = nhge->nh_parent;
2393
2394                 err = replace_nexthop_single_notify(net, nhp, old, oldi, newi,
2395                                                     extack);
2396                 if (err)
2397                         goto err_notify;
2398         }
2399
2400         /* When replacing an IPv4 nexthop with an IPv6 nexthop, potentially
2401          * update IPv4 indication in all the groups using the nexthop.
2402          */
2403         if (oldi->family == AF_INET && newi->family == AF_INET6) {
2404                 list_for_each_entry(nhge, &old->grp_list, nh_list) {
2405                         struct nexthop *nhp = nhge->nh_parent;
2406                         struct nh_group *nhg;
2407
2408                         nhg = rtnl_dereference(nhp->nh_grp);
2409                         nh_group_v4_update(nhg);
2410                 }
2411         }
2412
2413         return 0;
2414
2415 err_notify:
2416         rcu_assign_pointer(new->nh_info, newi);
2417         rcu_assign_pointer(old->nh_info, oldi);
2418         old->nh_flags = old_nh_flags;
2419         old->protocol = old_protocol;
2420         oldi->nh_parent = old;
2421         newi->nh_parent = new;
2422         list_for_each_entry_continue_reverse(nhge, &old->grp_list, nh_list) {
2423                 struct nexthop *nhp = nhge->nh_parent;
2424
2425                 replace_nexthop_single_notify(net, nhp, old, newi, oldi, NULL);
2426         }
2427         call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, old, extack);
2428         return err;
2429 }
2430
2431 static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,
2432                                      struct nl_info *info)
2433 {
2434         struct fib6_info *f6i;
2435
2436         if (!list_empty(&nh->fi_list)) {
2437                 struct fib_info *fi;
2438
2439                 /* expectation is a few fib_info per nexthop and then
2440                  * a lot of routes per fib_info. So mark the fib_info
2441                  * and then walk the fib tables once
2442                  */
2443                 list_for_each_entry(fi, &nh->fi_list, nh_list)
2444                         fi->nh_updated = true;
2445
2446                 fib_info_notify_update(net, info);
2447
2448                 list_for_each_entry(fi, &nh->fi_list, nh_list)
2449                         fi->nh_updated = false;
2450         }
2451
2452         list_for_each_entry(f6i, &nh->f6i_list, nh_list)
2453                 ipv6_stub->fib6_rt_update(net, f6i, info);
2454 }
2455
2456 /* send RTM_NEWROUTE with REPLACE flag set for all FIB entries
2457  * linked to this nexthop and for all groups that the nexthop
2458  * is a member of
2459  */
2460 static void nexthop_replace_notify(struct net *net, struct nexthop *nh,
2461                                    struct nl_info *info)
2462 {
2463         struct nh_grp_entry *nhge;
2464
2465         __nexthop_replace_notify(net, nh, info);
2466
2467         list_for_each_entry(nhge, &nh->grp_list, nh_list)
2468                 __nexthop_replace_notify(net, nhge->nh_parent, info);
2469 }
2470
2471 static int replace_nexthop(struct net *net, struct nexthop *old,
2472                            struct nexthop *new, const struct nh_config *cfg,
2473                            struct netlink_ext_ack *extack)
2474 {
2475         bool new_is_reject = false;
2476         struct nh_grp_entry *nhge;
2477         int err;
2478
2479         /* check that existing FIB entries are ok with the
2480          * new nexthop definition
2481          */
2482         err = fib_check_nh_list(old, new, extack);
2483         if (err)
2484                 return err;
2485
2486         err = fib6_check_nh_list(old, new, extack);
2487         if (err)
2488                 return err;
2489
2490         if (!new->is_group) {
2491                 struct nh_info *nhi = rtnl_dereference(new->nh_info);
2492
2493                 new_is_reject = nhi->reject_nh;
2494         }
2495
2496         list_for_each_entry(nhge, &old->grp_list, nh_list) {
2497                 /* if new nexthop is a blackhole, any groups using this
2498                  * nexthop cannot have more than 1 path
2499                  */
2500                 if (new_is_reject &&
2501                     nexthop_num_path(nhge->nh_parent) > 1) {
2502                         NL_SET_ERR_MSG(extack, "Blackhole nexthop can not be a member of a group with more than one path");
2503                         return -EINVAL;
2504                 }
2505
2506                 err = fib_check_nh_list(nhge->nh_parent, new, extack);
2507                 if (err)
2508                         return err;
2509
2510                 err = fib6_check_nh_list(nhge->nh_parent, new, extack);
2511                 if (err)
2512                         return err;
2513         }
2514
2515         if (old->is_group)
2516                 err = replace_nexthop_grp(net, old, new, cfg, extack);
2517         else
2518                 err = replace_nexthop_single(net, old, new, extack);
2519
2520         if (!err) {
2521                 nh_rt_cache_flush(net, old, new);
2522
2523                 __remove_nexthop(net, new, NULL);
2524                 nexthop_put(new);
2525         }
2526
2527         return err;
2528 }
2529
2530 /* called with rtnl_lock held */
2531 static int insert_nexthop(struct net *net, struct nexthop *new_nh,
2532                           struct nh_config *cfg, struct netlink_ext_ack *extack)
2533 {
2534         struct rb_node **pp, *parent = NULL, *next;
2535         struct rb_root *root = &net->nexthop.rb_root;
2536         bool replace = !!(cfg->nlflags & NLM_F_REPLACE);
2537         bool create = !!(cfg->nlflags & NLM_F_CREATE);
2538         u32 new_id = new_nh->id;
2539         int replace_notify = 0;
2540         int rc = -EEXIST;
2541
2542         pp = &root->rb_node;
2543         while (1) {
2544                 struct nexthop *nh;
2545
2546                 next = *pp;
2547                 if (!next)
2548                         break;
2549
2550                 parent = next;
2551
2552                 nh = rb_entry(parent, struct nexthop, rb_node);
2553                 if (new_id < nh->id) {
2554                         pp = &next->rb_left;
2555                 } else if (new_id > nh->id) {
2556                         pp = &next->rb_right;
2557                 } else if (replace) {
2558                         rc = replace_nexthop(net, nh, new_nh, cfg, extack);
2559                         if (!rc) {
2560                                 new_nh = nh; /* send notification with old nh */
2561                                 replace_notify = 1;
2562                         }
2563                         goto out;
2564                 } else {
2565                         /* id already exists and not a replace */
2566                         goto out;
2567                 }
2568         }
2569
2570         if (replace && !create) {
2571                 NL_SET_ERR_MSG(extack, "Replace specified without create and no entry exists");
2572                 rc = -ENOENT;
2573                 goto out;
2574         }
2575
2576         if (new_nh->is_group) {
2577                 struct nh_group *nhg = rtnl_dereference(new_nh->nh_grp);
2578                 struct nh_res_table *res_table;
2579
2580                 if (nhg->resilient) {
2581                         res_table = rtnl_dereference(nhg->res_table);
2582
2583                         /* Not passing the number of buckets is OK when
2584                          * replacing, but not when creating a new group.
2585                          */
2586                         if (!cfg->nh_grp_res_has_num_buckets) {
2587                                 NL_SET_ERR_MSG(extack, "Number of buckets not specified for nexthop group insertion");
2588                                 rc = -EINVAL;
2589                                 goto out;
2590                         }
2591
2592                         nh_res_group_rebalance(nhg, res_table);
2593
2594                         /* Do not send bucket notifications, we do full
2595                          * notification below.
2596                          */
2597                         nh_res_table_upkeep(res_table, false, false);
2598                 }
2599         }
2600
2601         rb_link_node_rcu(&new_nh->rb_node, parent, pp);
2602         rb_insert_color(&new_nh->rb_node, root);
2603
2604         /* The initial insertion is a full notification for hash-threshold as
2605          * well as resilient groups.
2606          */
2607         rc = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new_nh, extack);
2608         if (rc)
2609                 rb_erase(&new_nh->rb_node, &net->nexthop.rb_root);
2610
2611 out:
2612         if (!rc) {
2613                 nh_base_seq_inc(net);
2614                 nexthop_notify(RTM_NEWNEXTHOP, new_nh, &cfg->nlinfo);
2615                 if (replace_notify &&
2616                     READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode))
2617                         nexthop_replace_notify(net, new_nh, &cfg->nlinfo);
2618         }
2619
2620         return rc;
2621 }
2622
2623 /* rtnl */
2624 /* remove all nexthops tied to a device being deleted */
2625 static void nexthop_flush_dev(struct net_device *dev, unsigned long event)
2626 {
2627         unsigned int hash = nh_dev_hashfn(dev->ifindex);
2628         struct net *net = dev_net(dev);
2629         struct hlist_head *head = &net->nexthop.devhash[hash];
2630         struct hlist_node *n;
2631         struct nh_info *nhi;
2632
2633         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
2634                 if (nhi->fib_nhc.nhc_dev != dev)
2635                         continue;
2636
2637                 if (nhi->reject_nh &&
2638                     (event == NETDEV_DOWN || event == NETDEV_CHANGE))
2639                         continue;
2640
2641                 remove_nexthop(net, nhi->nh_parent, NULL);
2642         }
2643 }
2644
2645 /* rtnl; called when net namespace is deleted */
2646 static void flush_all_nexthops(struct net *net)
2647 {
2648         struct rb_root *root = &net->nexthop.rb_root;
2649         struct rb_node *node;
2650         struct nexthop *nh;
2651
2652         while ((node = rb_first(root))) {
2653                 nh = rb_entry(node, struct nexthop, rb_node);
2654                 remove_nexthop(net, nh, NULL);
2655                 cond_resched();
2656         }
2657 }
2658
2659 static struct nexthop *nexthop_create_group(struct net *net,
2660                                             struct nh_config *cfg)
2661 {
2662         struct nlattr *grps_attr = cfg->nh_grp;
2663         struct nexthop_grp *entry = nla_data(grps_attr);
2664         u16 num_nh = nla_len(grps_attr) / sizeof(*entry);
2665         struct nh_group *nhg;
2666         struct nexthop *nh;
2667         int err;
2668         int i;
2669
2670         if (WARN_ON(!num_nh))
2671                 return ERR_PTR(-EINVAL);
2672
2673         nh = nexthop_alloc();
2674         if (!nh)
2675                 return ERR_PTR(-ENOMEM);
2676
2677         nh->is_group = 1;
2678
2679         nhg = nexthop_grp_alloc(num_nh);
2680         if (!nhg) {
2681                 kfree(nh);
2682                 return ERR_PTR(-ENOMEM);
2683         }
2684
2685         /* spare group used for removals */
2686         nhg->spare = nexthop_grp_alloc(num_nh);
2687         if (!nhg->spare) {
2688                 kfree(nhg);
2689                 kfree(nh);
2690                 return ERR_PTR(-ENOMEM);
2691         }
2692         nhg->spare->spare = nhg;
2693
2694         for (i = 0; i < nhg->num_nh; ++i) {
2695                 struct nexthop *nhe;
2696                 struct nh_info *nhi;
2697
2698                 nhe = nexthop_find_by_id(net, entry[i].id);
2699                 if (!nexthop_get(nhe)) {
2700                         err = -ENOENT;
2701                         goto out_no_nh;
2702                 }
2703
2704                 nhi = rtnl_dereference(nhe->nh_info);
2705                 if (nhi->family == AF_INET)
2706                         nhg->has_v4 = true;
2707
2708                 nhg->nh_entries[i].stats =
2709                         netdev_alloc_pcpu_stats(struct nh_grp_entry_stats);
2710                 if (!nhg->nh_entries[i].stats) {
2711                         err = -ENOMEM;
2712                         nexthop_put(nhe);
2713                         goto out_no_nh;
2714                 }
2715                 nhg->nh_entries[i].nh = nhe;
2716                 nhg->nh_entries[i].weight = entry[i].weight + 1;
2717                 list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);
2718                 nhg->nh_entries[i].nh_parent = nh;
2719         }
2720
2721         if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_MPATH) {
2722                 nhg->hash_threshold = 1;
2723                 nhg->is_multipath = true;
2724         } else if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES) {
2725                 struct nh_res_table *res_table;
2726
2727                 res_table = nexthop_res_table_alloc(net, cfg->nh_id, cfg);
2728                 if (!res_table) {
2729                         err = -ENOMEM;
2730                         goto out_no_nh;
2731                 }
2732
2733                 rcu_assign_pointer(nhg->spare->res_table, res_table);
2734                 rcu_assign_pointer(nhg->res_table, res_table);
2735                 nhg->resilient = true;
2736                 nhg->is_multipath = true;
2737         }
2738
2739         WARN_ON_ONCE(nhg->hash_threshold + nhg->resilient != 1);
2740
2741         if (nhg->hash_threshold)
2742                 nh_hthr_group_rebalance(nhg);
2743
2744         if (cfg->nh_fdb)
2745                 nhg->fdb_nh = 1;
2746
2747         if (cfg->nh_hw_stats)
2748                 nhg->hw_stats = true;
2749
2750         rcu_assign_pointer(nh->nh_grp, nhg);
2751
2752         return nh;
2753
2754 out_no_nh:
2755         for (i--; i >= 0; --i) {
2756                 list_del(&nhg->nh_entries[i].nh_list);
2757                 free_percpu(nhg->nh_entries[i].stats);
2758                 nexthop_put(nhg->nh_entries[i].nh);
2759         }
2760
2761         kfree(nhg->spare);
2762         kfree(nhg);
2763         kfree(nh);
2764
2765         return ERR_PTR(err);
2766 }
2767
2768 static int nh_create_ipv4(struct net *net, struct nexthop *nh,
2769                           struct nh_info *nhi, struct nh_config *cfg,
2770                           struct netlink_ext_ack *extack)
2771 {
2772         struct fib_nh *fib_nh = &nhi->fib_nh;
2773         struct fib_config fib_cfg = {
2774                 .fc_oif   = cfg->nh_ifindex,
2775                 .fc_gw4   = cfg->gw.ipv4,
2776                 .fc_gw_family = cfg->gw.ipv4 ? AF_INET : 0,
2777                 .fc_flags = cfg->nh_flags,
2778                 .fc_nlinfo = cfg->nlinfo,
2779                 .fc_encap = cfg->nh_encap,
2780                 .fc_encap_type = cfg->nh_encap_type,
2781         };
2782         u32 tb_id = (cfg->dev ? l3mdev_fib_table(cfg->dev) : RT_TABLE_MAIN);
2783         int err;
2784
2785         err = fib_nh_init(net, fib_nh, &fib_cfg, 1, extack);
2786         if (err) {
2787                 fib_nh_release(net, fib_nh);
2788                 goto out;
2789         }
2790
2791         if (nhi->fdb_nh)
2792                 goto out;
2793
2794         /* sets nh_dev if successful */
2795         err = fib_check_nh(net, fib_nh, tb_id, 0, extack);
2796         if (!err) {
2797                 nh->nh_flags = fib_nh->fib_nh_flags;
2798                 fib_info_update_nhc_saddr(net, &fib_nh->nh_common,
2799                                           !fib_nh->fib_nh_scope ? 0 : fib_nh->fib_nh_scope - 1);
2800         } else {
2801                 fib_nh_release(net, fib_nh);
2802         }
2803 out:
2804         return err;
2805 }
2806
2807 static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
2808                           struct nh_info *nhi, struct nh_config *cfg,
2809                           struct netlink_ext_ack *extack)
2810 {
2811         struct fib6_nh *fib6_nh = &nhi->fib6_nh;
2812         struct fib6_config fib6_cfg = {
2813                 .fc_table = l3mdev_fib_table(cfg->dev),
2814                 .fc_ifindex = cfg->nh_ifindex,
2815                 .fc_gateway = cfg->gw.ipv6,
2816                 .fc_flags = cfg->nh_flags,
2817                 .fc_nlinfo = cfg->nlinfo,
2818                 .fc_encap = cfg->nh_encap,
2819                 .fc_encap_type = cfg->nh_encap_type,
2820                 .fc_is_fdb = cfg->nh_fdb,
2821         };
2822         int err;
2823
2824         if (!ipv6_addr_any(&cfg->gw.ipv6))
2825                 fib6_cfg.fc_flags |= RTF_GATEWAY;
2826
2827         /* sets nh_dev if successful */
2828         err = ipv6_stub->fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL,
2829                                       extack);
2830         if (err) {
2831                 /* IPv6 is not enabled, don't call fib6_nh_release */
2832                 if (err == -EAFNOSUPPORT)
2833                         goto out;
2834                 ipv6_stub->fib6_nh_release(fib6_nh);
2835         } else {
2836                 nh->nh_flags = fib6_nh->fib_nh_flags;
2837         }
2838 out:
2839         return err;
2840 }
2841
2842 static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
2843                                       struct netlink_ext_ack *extack)
2844 {
2845         struct nh_info *nhi;
2846         struct nexthop *nh;
2847         int err = 0;
2848
2849         nh = nexthop_alloc();
2850         if (!nh)
2851                 return ERR_PTR(-ENOMEM);
2852
2853         nhi = kzalloc(sizeof(*nhi), GFP_KERNEL);
2854         if (!nhi) {
2855                 kfree(nh);
2856                 return ERR_PTR(-ENOMEM);
2857         }
2858
2859         nh->nh_flags = cfg->nh_flags;
2860         nh->net = net;
2861
2862         nhi->nh_parent = nh;
2863         nhi->family = cfg->nh_family;
2864         nhi->fib_nhc.nhc_scope = RT_SCOPE_LINK;
2865
2866         if (cfg->nh_fdb)
2867                 nhi->fdb_nh = 1;
2868
2869         if (cfg->nh_blackhole) {
2870                 nhi->reject_nh = 1;
2871                 cfg->nh_ifindex = net->loopback_dev->ifindex;
2872         }
2873
2874         switch (cfg->nh_family) {
2875         case AF_INET:
2876                 err = nh_create_ipv4(net, nh, nhi, cfg, extack);
2877                 break;
2878         case AF_INET6:
2879                 err = nh_create_ipv6(net, nh, nhi, cfg, extack);
2880                 break;
2881         }
2882
2883         if (err) {
2884                 kfree(nhi);
2885                 kfree(nh);
2886                 return ERR_PTR(err);
2887         }
2888
2889         /* add the entry to the device based hash */
2890         if (!nhi->fdb_nh)
2891                 nexthop_devhash_add(net, nhi);
2892
2893         rcu_assign_pointer(nh->nh_info, nhi);
2894
2895         return nh;
2896 }
2897
2898 /* called with rtnl lock held */
2899 static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
2900                                    struct netlink_ext_ack *extack)
2901 {
2902         struct nexthop *nh;
2903         int err;
2904
2905         if (cfg->nlflags & NLM_F_REPLACE && !cfg->nh_id) {
2906                 NL_SET_ERR_MSG(extack, "Replace requires nexthop id");
2907                 return ERR_PTR(-EINVAL);
2908         }
2909
2910         if (!cfg->nh_id) {
2911                 cfg->nh_id = nh_find_unused_id(net);
2912                 if (!cfg->nh_id) {
2913                         NL_SET_ERR_MSG(extack, "No unused id");
2914                         return ERR_PTR(-EINVAL);
2915                 }
2916         }
2917
2918         if (cfg->nh_grp)
2919                 nh = nexthop_create_group(net, cfg);
2920         else
2921                 nh = nexthop_create(net, cfg, extack);
2922
2923         if (IS_ERR(nh))
2924                 return nh;
2925
2926         refcount_set(&nh->refcnt, 1);
2927         nh->id = cfg->nh_id;
2928         nh->protocol = cfg->nh_protocol;
2929         nh->net = net;
2930
2931         err = insert_nexthop(net, nh, cfg, extack);
2932         if (err) {
2933                 __remove_nexthop(net, nh, NULL);
2934                 nexthop_put(nh);
2935                 nh = ERR_PTR(err);
2936         }
2937
2938         return nh;
2939 }
2940
2941 static int rtm_nh_get_timer(struct nlattr *attr, unsigned long fallback,
2942                             unsigned long *timer_p, bool *has_p,
2943                             struct netlink_ext_ack *extack)
2944 {
2945         unsigned long timer;
2946         u32 value;
2947
2948         if (!attr) {
2949                 *timer_p = fallback;
2950                 *has_p = false;
2951                 return 0;
2952         }
2953
2954         value = nla_get_u32(attr);
2955         timer = clock_t_to_jiffies(value);
2956         if (timer == ~0UL) {
2957                 NL_SET_ERR_MSG(extack, "Timer value too large");
2958                 return -EINVAL;
2959         }
2960
2961         *timer_p = timer;
2962         *has_p = true;
2963         return 0;
2964 }
2965
2966 static int rtm_to_nh_config_grp_res(struct nlattr *res, struct nh_config *cfg,
2967                                     struct netlink_ext_ack *extack)
2968 {
2969         struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_policy_new)] = {};
2970         int err;
2971
2972         if (res) {
2973                 err = nla_parse_nested(tb,
2974                                        ARRAY_SIZE(rtm_nh_res_policy_new) - 1,
2975                                        res, rtm_nh_res_policy_new, extack);
2976                 if (err < 0)
2977                         return err;
2978         }
2979
2980         if (tb[NHA_RES_GROUP_BUCKETS]) {
2981                 cfg->nh_grp_res_num_buckets =
2982                         nla_get_u16(tb[NHA_RES_GROUP_BUCKETS]);
2983                 cfg->nh_grp_res_has_num_buckets = true;
2984                 if (!cfg->nh_grp_res_num_buckets) {
2985                         NL_SET_ERR_MSG(extack, "Number of buckets needs to be non-0");
2986                         return -EINVAL;
2987                 }
2988         }
2989
2990         err = rtm_nh_get_timer(tb[NHA_RES_GROUP_IDLE_TIMER],
2991                                NH_RES_DEFAULT_IDLE_TIMER,
2992                                &cfg->nh_grp_res_idle_timer,
2993                                &cfg->nh_grp_res_has_idle_timer,
2994                                extack);
2995         if (err)
2996                 return err;
2997
2998         return rtm_nh_get_timer(tb[NHA_RES_GROUP_UNBALANCED_TIMER],
2999                                 NH_RES_DEFAULT_UNBALANCED_TIMER,
3000                                 &cfg->nh_grp_res_unbalanced_timer,
3001                                 &cfg->nh_grp_res_has_unbalanced_timer,
3002                                 extack);
3003 }
3004
3005 static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
3006                             struct nlmsghdr *nlh, struct nh_config *cfg,
3007                             struct netlink_ext_ack *extack)
3008 {
3009         struct nhmsg *nhm = nlmsg_data(nlh);
3010         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_new)];
3011         int err;
3012
3013         err = nlmsg_parse(nlh, sizeof(*nhm), tb,
3014                           ARRAY_SIZE(rtm_nh_policy_new) - 1,
3015                           rtm_nh_policy_new, extack);
3016         if (err < 0)
3017                 return err;
3018
3019         err = -EINVAL;
3020         if (nhm->resvd || nhm->nh_scope) {
3021                 NL_SET_ERR_MSG(extack, "Invalid values in ancillary header");
3022                 goto out;
3023         }
3024         if (nhm->nh_flags & ~NEXTHOP_VALID_USER_FLAGS) {
3025                 NL_SET_ERR_MSG(extack, "Invalid nexthop flags in ancillary header");
3026                 goto out;
3027         }
3028
3029         switch (nhm->nh_family) {
3030         case AF_INET:
3031         case AF_INET6:
3032                 break;
3033         case AF_UNSPEC:
3034                 if (tb[NHA_GROUP])
3035                         break;
3036                 fallthrough;
3037         default:
3038                 NL_SET_ERR_MSG(extack, "Invalid address family");
3039                 goto out;
3040         }
3041
3042         memset(cfg, 0, sizeof(*cfg));
3043         cfg->nlflags = nlh->nlmsg_flags;
3044         cfg->nlinfo.portid = NETLINK_CB(skb).portid;
3045         cfg->nlinfo.nlh = nlh;
3046         cfg->nlinfo.nl_net = net;
3047
3048         cfg->nh_family = nhm->nh_family;
3049         cfg->nh_protocol = nhm->nh_protocol;
3050         cfg->nh_flags = nhm->nh_flags;
3051
3052         if (tb[NHA_ID])
3053                 cfg->nh_id = nla_get_u32(tb[NHA_ID]);
3054
3055         if (tb[NHA_FDB]) {
3056                 if (tb[NHA_OIF] || tb[NHA_BLACKHOLE] ||
3057                     tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE]) {
3058                         NL_SET_ERR_MSG(extack, "Fdb attribute can not be used with encap, oif or blackhole");
3059                         goto out;
3060                 }
3061                 if (nhm->nh_flags) {
3062                         NL_SET_ERR_MSG(extack, "Unsupported nexthop flags in ancillary header");
3063                         goto out;
3064                 }
3065                 cfg->nh_fdb = nla_get_flag(tb[NHA_FDB]);
3066         }
3067
3068         if (tb[NHA_GROUP]) {
3069                 if (nhm->nh_family != AF_UNSPEC) {
3070                         NL_SET_ERR_MSG(extack, "Invalid family for group");
3071                         goto out;
3072                 }
3073                 cfg->nh_grp = tb[NHA_GROUP];
3074
3075                 cfg->nh_grp_type = NEXTHOP_GRP_TYPE_MPATH;
3076                 if (tb[NHA_GROUP_TYPE])
3077                         cfg->nh_grp_type = nla_get_u16(tb[NHA_GROUP_TYPE]);
3078
3079                 if (cfg->nh_grp_type > NEXTHOP_GRP_TYPE_MAX) {
3080                         NL_SET_ERR_MSG(extack, "Invalid group type");
3081                         goto out;
3082                 }
3083                 err = nh_check_attr_group(net, tb, ARRAY_SIZE(tb),
3084                                           cfg->nh_grp_type, extack);
3085                 if (err)
3086                         goto out;
3087
3088                 if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES)
3089                         err = rtm_to_nh_config_grp_res(tb[NHA_RES_GROUP],
3090                                                        cfg, extack);
3091
3092                 if (tb[NHA_HW_STATS_ENABLE])
3093                         cfg->nh_hw_stats = nla_get_u32(tb[NHA_HW_STATS_ENABLE]);
3094
3095                 /* no other attributes should be set */
3096                 goto out;
3097         }
3098
3099         if (tb[NHA_BLACKHOLE]) {
3100                 if (tb[NHA_GATEWAY] || tb[NHA_OIF] ||
3101                     tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE] || tb[NHA_FDB]) {
3102                         NL_SET_ERR_MSG(extack, "Blackhole attribute can not be used with gateway, oif, encap or fdb");
3103                         goto out;
3104                 }
3105
3106                 cfg->nh_blackhole = 1;
3107                 err = 0;
3108                 goto out;
3109         }
3110
3111         if (!cfg->nh_fdb && !tb[NHA_OIF]) {
3112                 NL_SET_ERR_MSG(extack, "Device attribute required for non-blackhole and non-fdb nexthops");
3113                 goto out;
3114         }
3115
3116         if (!cfg->nh_fdb && tb[NHA_OIF]) {
3117                 cfg->nh_ifindex = nla_get_u32(tb[NHA_OIF]);
3118                 if (cfg->nh_ifindex)
3119                         cfg->dev = __dev_get_by_index(net, cfg->nh_ifindex);
3120
3121                 if (!cfg->dev) {
3122                         NL_SET_ERR_MSG(extack, "Invalid device index");
3123                         goto out;
3124                 } else if (!(cfg->dev->flags & IFF_UP)) {
3125                         NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3126                         err = -ENETDOWN;
3127                         goto out;
3128                 } else if (!netif_carrier_ok(cfg->dev)) {
3129                         NL_SET_ERR_MSG(extack, "Carrier for nexthop device is down");
3130                         err = -ENETDOWN;
3131                         goto out;
3132                 }
3133         }
3134
3135         err = -EINVAL;
3136         if (tb[NHA_GATEWAY]) {
3137                 struct nlattr *gwa = tb[NHA_GATEWAY];
3138
3139                 switch (cfg->nh_family) {
3140                 case AF_INET:
3141                         if (nla_len(gwa) != sizeof(u32)) {
3142                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
3143                                 goto out;
3144                         }
3145                         cfg->gw.ipv4 = nla_get_be32(gwa);
3146                         break;
3147                 case AF_INET6:
3148                         if (nla_len(gwa) != sizeof(struct in6_addr)) {
3149                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
3150                                 goto out;
3151                         }
3152                         cfg->gw.ipv6 = nla_get_in6_addr(gwa);
3153                         break;
3154                 default:
3155                         NL_SET_ERR_MSG(extack,
3156                                        "Unknown address family for gateway");
3157                         goto out;
3158                 }
3159         } else {
3160                 /* device only nexthop (no gateway) */
3161                 if (cfg->nh_flags & RTNH_F_ONLINK) {
3162                         NL_SET_ERR_MSG(extack,
3163                                        "ONLINK flag can not be set for nexthop without a gateway");
3164                         goto out;
3165                 }
3166         }
3167
3168         if (tb[NHA_ENCAP]) {
3169                 cfg->nh_encap = tb[NHA_ENCAP];
3170
3171                 if (!tb[NHA_ENCAP_TYPE]) {
3172                         NL_SET_ERR_MSG(extack, "LWT encapsulation type is missing");
3173                         goto out;
3174                 }
3175
3176                 cfg->nh_encap_type = nla_get_u16(tb[NHA_ENCAP_TYPE]);
3177                 err = lwtunnel_valid_encap_type(cfg->nh_encap_type, extack);
3178                 if (err < 0)
3179                         goto out;
3180
3181         } else if (tb[NHA_ENCAP_TYPE]) {
3182                 NL_SET_ERR_MSG(extack, "LWT encapsulation attribute is missing");
3183                 goto out;
3184         }
3185
3186         if (tb[NHA_HW_STATS_ENABLE]) {
3187                 NL_SET_ERR_MSG(extack, "Cannot enable nexthop hardware statistics for non-group nexthops");
3188                 goto out;
3189         }
3190
3191         err = 0;
3192 out:
3193         return err;
3194 }
3195
3196 /* rtnl */
3197 static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
3198                            struct netlink_ext_ack *extack)
3199 {
3200         struct net *net = sock_net(skb->sk);
3201         struct nh_config cfg;
3202         struct nexthop *nh;
3203         int err;
3204
3205         err = rtm_to_nh_config(net, skb, nlh, &cfg, extack);
3206         if (!err) {
3207                 nh = nexthop_add(net, &cfg, extack);
3208                 if (IS_ERR(nh))
3209                         err = PTR_ERR(nh);
3210         }
3211
3212         return err;
3213 }
3214
3215 static int nh_valid_get_del_req(const struct nlmsghdr *nlh,
3216                                 struct nlattr **tb, u32 *id, u32 *op_flags,
3217                                 struct netlink_ext_ack *extack)
3218 {
3219         struct nhmsg *nhm = nlmsg_data(nlh);
3220
3221         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
3222                 NL_SET_ERR_MSG(extack, "Invalid values in header");
3223                 return -EINVAL;
3224         }
3225
3226         if (!tb[NHA_ID]) {
3227                 NL_SET_ERR_MSG(extack, "Nexthop id is missing");
3228                 return -EINVAL;
3229         }
3230
3231         *id = nla_get_u32(tb[NHA_ID]);
3232         if (!(*id)) {
3233                 NL_SET_ERR_MSG(extack, "Invalid nexthop id");
3234                 return -EINVAL;
3235         }
3236
3237         if (op_flags) {
3238                 if (tb[NHA_OP_FLAGS])
3239                         *op_flags = nla_get_u32(tb[NHA_OP_FLAGS]);
3240                 else
3241                         *op_flags = 0;
3242         }
3243
3244         return 0;
3245 }
3246
3247 /* rtnl */
3248 static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
3249                            struct netlink_ext_ack *extack)
3250 {
3251         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_del)];
3252         struct net *net = sock_net(skb->sk);
3253         struct nl_info nlinfo = {
3254                 .nlh = nlh,
3255                 .nl_net = net,
3256                 .portid = NETLINK_CB(skb).portid,
3257         };
3258         struct nexthop *nh;
3259         int err;
3260         u32 id;
3261
3262         err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3263                           ARRAY_SIZE(rtm_nh_policy_del) - 1, rtm_nh_policy_del,
3264                           extack);
3265         if (err < 0)
3266                 return err;
3267
3268         err = nh_valid_get_del_req(nlh, tb, &id, NULL, extack);
3269         if (err)
3270                 return err;
3271
3272         nh = nexthop_find_by_id(net, id);
3273         if (!nh)
3274                 return -ENOENT;
3275
3276         remove_nexthop(net, nh, &nlinfo);
3277
3278         return 0;
3279 }
3280
3281 /* rtnl */
3282 static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
3283                            struct netlink_ext_ack *extack)
3284 {
3285         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get)];
3286         struct net *net = sock_net(in_skb->sk);
3287         struct sk_buff *skb = NULL;
3288         struct nexthop *nh;
3289         u32 op_flags;
3290         int err;
3291         u32 id;
3292
3293         err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3294                           ARRAY_SIZE(rtm_nh_policy_get) - 1, rtm_nh_policy_get,
3295                           extack);
3296         if (err < 0)
3297                 return err;
3298
3299         err = nh_valid_get_del_req(nlh, tb, &id, &op_flags, extack);
3300         if (err)
3301                 return err;
3302
3303         err = -ENOBUFS;
3304         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
3305         if (!skb)
3306                 goto out;
3307
3308         err = -ENOENT;
3309         nh = nexthop_find_by_id(net, id);
3310         if (!nh)
3311                 goto errout_free;
3312
3313         err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,
3314                            nlh->nlmsg_seq, 0, op_flags);
3315         if (err < 0) {
3316                 WARN_ON(err == -EMSGSIZE);
3317                 goto errout_free;
3318         }
3319
3320         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
3321 out:
3322         return err;
3323 errout_free:
3324         kfree_skb(skb);
3325         goto out;
3326 }
3327
3328 struct nh_dump_filter {
3329         u32 nh_id;
3330         int dev_idx;
3331         int master_idx;
3332         bool group_filter;
3333         bool fdb_filter;
3334         u32 res_bucket_nh_id;
3335         u32 op_flags;
3336 };
3337
3338 static bool nh_dump_filtered(struct nexthop *nh,
3339                              struct nh_dump_filter *filter, u8 family)
3340 {
3341         const struct net_device *dev;
3342         const struct nh_info *nhi;
3343
3344         if (filter->group_filter && !nh->is_group)
3345                 return true;
3346
3347         if (!filter->dev_idx && !filter->master_idx && !family)
3348                 return false;
3349
3350         if (nh->is_group)
3351                 return true;
3352
3353         nhi = rtnl_dereference(nh->nh_info);
3354         if (family && nhi->family != family)
3355                 return true;
3356
3357         dev = nhi->fib_nhc.nhc_dev;
3358         if (filter->dev_idx && (!dev || dev->ifindex != filter->dev_idx))
3359                 return true;
3360
3361         if (filter->master_idx) {
3362                 struct net_device *master;
3363
3364                 if (!dev)
3365                         return true;
3366
3367                 master = netdev_master_upper_dev_get((struct net_device *)dev);
3368                 if (!master || master->ifindex != filter->master_idx)
3369                         return true;
3370         }
3371
3372         return false;
3373 }
3374
3375 static int __nh_valid_dump_req(const struct nlmsghdr *nlh, struct nlattr **tb,
3376                                struct nh_dump_filter *filter,
3377                                struct netlink_ext_ack *extack)
3378 {
3379         struct nhmsg *nhm;
3380         u32 idx;
3381
3382         if (tb[NHA_OIF]) {
3383                 idx = nla_get_u32(tb[NHA_OIF]);
3384                 if (idx > INT_MAX) {
3385                         NL_SET_ERR_MSG(extack, "Invalid device index");
3386                         return -EINVAL;
3387                 }
3388                 filter->dev_idx = idx;
3389         }
3390         if (tb[NHA_MASTER]) {
3391                 idx = nla_get_u32(tb[NHA_MASTER]);
3392                 if (idx > INT_MAX) {
3393                         NL_SET_ERR_MSG(extack, "Invalid master device index");
3394                         return -EINVAL;
3395                 }
3396                 filter->master_idx = idx;
3397         }
3398         filter->group_filter = nla_get_flag(tb[NHA_GROUPS]);
3399         filter->fdb_filter = nla_get_flag(tb[NHA_FDB]);
3400
3401         nhm = nlmsg_data(nlh);
3402         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
3403                 NL_SET_ERR_MSG(extack, "Invalid values in header for nexthop dump request");
3404                 return -EINVAL;
3405         }
3406
3407         return 0;
3408 }
3409
3410 static int nh_valid_dump_req(const struct nlmsghdr *nlh,
3411                              struct nh_dump_filter *filter,
3412                              struct netlink_callback *cb)
3413 {
3414         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump)];
3415         int err;
3416
3417         err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3418                           ARRAY_SIZE(rtm_nh_policy_dump) - 1,
3419                           rtm_nh_policy_dump, cb->extack);
3420         if (err < 0)
3421                 return err;
3422
3423         if (tb[NHA_OP_FLAGS])
3424                 filter->op_flags = nla_get_u32(tb[NHA_OP_FLAGS]);
3425         else
3426                 filter->op_flags = 0;
3427
3428         return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
3429 }
3430
3431 struct rtm_dump_nh_ctx {
3432         u32 idx;
3433 };
3434
3435 static struct rtm_dump_nh_ctx *
3436 rtm_dump_nh_ctx(struct netlink_callback *cb)
3437 {
3438         struct rtm_dump_nh_ctx *ctx = (void *)cb->ctx;
3439
3440         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3441         return ctx;
3442 }
3443
3444 static int rtm_dump_walk_nexthops(struct sk_buff *skb,
3445                                   struct netlink_callback *cb,
3446                                   struct rb_root *root,
3447                                   struct rtm_dump_nh_ctx *ctx,
3448                                   int (*nh_cb)(struct sk_buff *skb,
3449                                                struct netlink_callback *cb,
3450                                                struct nexthop *nh, void *data),
3451                                   void *data)
3452 {
3453         struct rb_node *node;
3454         int s_idx;
3455         int err;
3456
3457         s_idx = ctx->idx;
3458         for (node = rb_first(root); node; node = rb_next(node)) {
3459                 struct nexthop *nh;
3460
3461                 nh = rb_entry(node, struct nexthop, rb_node);
3462                 if (nh->id < s_idx)
3463                         continue;
3464
3465                 ctx->idx = nh->id;
3466                 err = nh_cb(skb, cb, nh, data);
3467                 if (err)
3468                         return err;
3469         }
3470
3471         return 0;
3472 }
3473
3474 static int rtm_dump_nexthop_cb(struct sk_buff *skb, struct netlink_callback *cb,
3475                                struct nexthop *nh, void *data)
3476 {
3477         struct nhmsg *nhm = nlmsg_data(cb->nlh);
3478         struct nh_dump_filter *filter = data;
3479
3480         if (nh_dump_filtered(nh, filter, nhm->nh_family))
3481                 return 0;
3482
3483         return nh_fill_node(skb, nh, RTM_NEWNEXTHOP,
3484                             NETLINK_CB(cb->skb).portid,
3485                             cb->nlh->nlmsg_seq, NLM_F_MULTI, filter->op_flags);
3486 }
3487
3488 /* rtnl */
3489 static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
3490 {
3491         struct rtm_dump_nh_ctx *ctx = rtm_dump_nh_ctx(cb);
3492         struct net *net = sock_net(skb->sk);
3493         struct rb_root *root = &net->nexthop.rb_root;
3494         struct nh_dump_filter filter = {};
3495         int err;
3496
3497         err = nh_valid_dump_req(cb->nlh, &filter, cb);
3498         if (err < 0)
3499                 return err;
3500
3501         err = rtm_dump_walk_nexthops(skb, cb, root, ctx,
3502                                      &rtm_dump_nexthop_cb, &filter);
3503
3504         cb->seq = net->nexthop.seq;
3505         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3506         return err;
3507 }
3508
3509 static struct nexthop *
3510 nexthop_find_group_resilient(struct net *net, u32 id,
3511                              struct netlink_ext_ack *extack)
3512 {
3513         struct nh_group *nhg;
3514         struct nexthop *nh;
3515
3516         nh = nexthop_find_by_id(net, id);
3517         if (!nh)
3518                 return ERR_PTR(-ENOENT);
3519
3520         if (!nh->is_group) {
3521                 NL_SET_ERR_MSG(extack, "Not a nexthop group");
3522                 return ERR_PTR(-EINVAL);
3523         }
3524
3525         nhg = rtnl_dereference(nh->nh_grp);
3526         if (!nhg->resilient) {
3527                 NL_SET_ERR_MSG(extack, "Nexthop group not of type resilient");
3528                 return ERR_PTR(-EINVAL);
3529         }
3530
3531         return nh;
3532 }
3533
3534 static int nh_valid_dump_nhid(struct nlattr *attr, u32 *nh_id_p,
3535                               struct netlink_ext_ack *extack)
3536 {
3537         u32 idx;
3538
3539         if (attr) {
3540                 idx = nla_get_u32(attr);
3541                 if (!idx) {
3542                         NL_SET_ERR_MSG(extack, "Invalid nexthop id");
3543                         return -EINVAL;
3544                 }
3545                 *nh_id_p = idx;
3546         } else {
3547                 *nh_id_p = 0;
3548         }
3549
3550         return 0;
3551 }
3552
3553 static int nh_valid_dump_bucket_req(const struct nlmsghdr *nlh,
3554                                     struct nh_dump_filter *filter,
3555                                     struct netlink_callback *cb)
3556 {
3557         struct nlattr *res_tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_dump)];
3558         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump_bucket)];
3559         int err;
3560
3561         err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3562                           ARRAY_SIZE(rtm_nh_policy_dump_bucket) - 1,
3563                           rtm_nh_policy_dump_bucket, NULL);
3564         if (err < 0)
3565                 return err;
3566
3567         err = nh_valid_dump_nhid(tb[NHA_ID], &filter->nh_id, cb->extack);
3568         if (err)
3569                 return err;
3570
3571         if (tb[NHA_RES_BUCKET]) {
3572                 size_t max = ARRAY_SIZE(rtm_nh_res_bucket_policy_dump) - 1;
3573
3574                 err = nla_parse_nested(res_tb, max,
3575                                        tb[NHA_RES_BUCKET],
3576                                        rtm_nh_res_bucket_policy_dump,
3577                                        cb->extack);
3578                 if (err < 0)
3579                         return err;
3580
3581                 err = nh_valid_dump_nhid(res_tb[NHA_RES_BUCKET_NH_ID],
3582                                          &filter->res_bucket_nh_id,
3583                                          cb->extack);
3584                 if (err)
3585                         return err;
3586         }
3587
3588         return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
3589 }
3590
3591 struct rtm_dump_res_bucket_ctx {
3592         struct rtm_dump_nh_ctx nh;
3593         u16 bucket_index;
3594 };
3595
3596 static struct rtm_dump_res_bucket_ctx *
3597 rtm_dump_res_bucket_ctx(struct netlink_callback *cb)
3598 {
3599         struct rtm_dump_res_bucket_ctx *ctx = (void *)cb->ctx;
3600
3601         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3602         return ctx;
3603 }
3604
3605 struct rtm_dump_nexthop_bucket_data {
3606         struct rtm_dump_res_bucket_ctx *ctx;
3607         struct nh_dump_filter filter;
3608 };
3609
3610 static int rtm_dump_nexthop_bucket_nh(struct sk_buff *skb,
3611                                       struct netlink_callback *cb,
3612                                       struct nexthop *nh,
3613                                       struct rtm_dump_nexthop_bucket_data *dd)
3614 {
3615         u32 portid = NETLINK_CB(cb->skb).portid;
3616         struct nhmsg *nhm = nlmsg_data(cb->nlh);
3617         struct nh_res_table *res_table;
3618         struct nh_group *nhg;
3619         u16 bucket_index;
3620         int err;
3621
3622         nhg = rtnl_dereference(nh->nh_grp);
3623         res_table = rtnl_dereference(nhg->res_table);
3624         for (bucket_index = dd->ctx->bucket_index;
3625              bucket_index < res_table->num_nh_buckets;
3626              bucket_index++) {
3627                 struct nh_res_bucket *bucket;
3628                 struct nh_grp_entry *nhge;
3629
3630                 bucket = &res_table->nh_buckets[bucket_index];
3631                 nhge = rtnl_dereference(bucket->nh_entry);
3632                 if (nh_dump_filtered(nhge->nh, &dd->filter, nhm->nh_family))
3633                         continue;
3634
3635                 if (dd->filter.res_bucket_nh_id &&
3636                     dd->filter.res_bucket_nh_id != nhge->nh->id)
3637                         continue;
3638
3639                 dd->ctx->bucket_index = bucket_index;
3640                 err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
3641                                          RTM_NEWNEXTHOPBUCKET, portid,
3642                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
3643                                          cb->extack);
3644                 if (err)
3645                         return err;
3646         }
3647
3648         dd->ctx->bucket_index = 0;
3649
3650         return 0;
3651 }
3652
3653 static int rtm_dump_nexthop_bucket_cb(struct sk_buff *skb,
3654                                       struct netlink_callback *cb,
3655                                       struct nexthop *nh, void *data)
3656 {
3657         struct rtm_dump_nexthop_bucket_data *dd = data;
3658         struct nh_group *nhg;
3659
3660         if (!nh->is_group)
3661                 return 0;
3662
3663         nhg = rtnl_dereference(nh->nh_grp);
3664         if (!nhg->resilient)
3665                 return 0;
3666
3667         return rtm_dump_nexthop_bucket_nh(skb, cb, nh, dd);
3668 }
3669
3670 /* rtnl */
3671 static int rtm_dump_nexthop_bucket(struct sk_buff *skb,
3672                                    struct netlink_callback *cb)
3673 {
3674         struct rtm_dump_res_bucket_ctx *ctx = rtm_dump_res_bucket_ctx(cb);
3675         struct rtm_dump_nexthop_bucket_data dd = { .ctx = ctx };
3676         struct net *net = sock_net(skb->sk);
3677         struct nexthop *nh;
3678         int err;
3679
3680         err = nh_valid_dump_bucket_req(cb->nlh, &dd.filter, cb);
3681         if (err)
3682                 return err;
3683
3684         if (dd.filter.nh_id) {
3685                 nh = nexthop_find_group_resilient(net, dd.filter.nh_id,
3686                                                   cb->extack);
3687                 if (IS_ERR(nh))
3688                         return PTR_ERR(nh);
3689                 err = rtm_dump_nexthop_bucket_nh(skb, cb, nh, &dd);
3690         } else {
3691                 struct rb_root *root = &net->nexthop.rb_root;
3692
3693                 err = rtm_dump_walk_nexthops(skb, cb, root, &ctx->nh,
3694                                              &rtm_dump_nexthop_bucket_cb, &dd);
3695         }
3696
3697         cb->seq = net->nexthop.seq;
3698         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3699         return err;
3700 }
3701
3702 static int nh_valid_get_bucket_req_res_bucket(struct nlattr *res,
3703                                               u16 *bucket_index,
3704                                               struct netlink_ext_ack *extack)
3705 {
3706         struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_get)];
3707         int err;
3708
3709         err = nla_parse_nested(tb, ARRAY_SIZE(rtm_nh_res_bucket_policy_get) - 1,
3710                                res, rtm_nh_res_bucket_policy_get, extack);
3711         if (err < 0)
3712                 return err;
3713
3714         if (!tb[NHA_RES_BUCKET_INDEX]) {
3715                 NL_SET_ERR_MSG(extack, "Bucket index is missing");
3716                 return -EINVAL;
3717         }
3718
3719         *bucket_index = nla_get_u16(tb[NHA_RES_BUCKET_INDEX]);
3720         return 0;
3721 }
3722
3723 static int nh_valid_get_bucket_req(const struct nlmsghdr *nlh,
3724                                    u32 *id, u16 *bucket_index,
3725                                    struct netlink_ext_ack *extack)
3726 {
3727         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get_bucket)];
3728         int err;
3729
3730         err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3731                           ARRAY_SIZE(rtm_nh_policy_get_bucket) - 1,
3732                           rtm_nh_policy_get_bucket, extack);
3733         if (err < 0)
3734                 return err;
3735
3736         err = nh_valid_get_del_req(nlh, tb, id, NULL, extack);
3737         if (err)
3738                 return err;
3739
3740         if (!tb[NHA_RES_BUCKET]) {
3741                 NL_SET_ERR_MSG(extack, "Bucket information is missing");
3742                 return -EINVAL;
3743         }
3744
3745         err = nh_valid_get_bucket_req_res_bucket(tb[NHA_RES_BUCKET],
3746                                                  bucket_index, extack);
3747         if (err)
3748                 return err;
3749
3750         return 0;
3751 }
3752
3753 /* rtnl */
3754 static int rtm_get_nexthop_bucket(struct sk_buff *in_skb, struct nlmsghdr *nlh,
3755                                   struct netlink_ext_ack *extack)
3756 {
3757         struct net *net = sock_net(in_skb->sk);
3758         struct nh_res_table *res_table;
3759         struct sk_buff *skb = NULL;
3760         struct nh_group *nhg;
3761         struct nexthop *nh;
3762         u16 bucket_index;
3763         int err;
3764         u32 id;
3765
3766         err = nh_valid_get_bucket_req(nlh, &id, &bucket_index, extack);
3767         if (err)
3768                 return err;
3769
3770         nh = nexthop_find_group_resilient(net, id, extack);
3771         if (IS_ERR(nh))
3772                 return PTR_ERR(nh);
3773
3774         nhg = rtnl_dereference(nh->nh_grp);
3775         res_table = rtnl_dereference(nhg->res_table);
3776         if (bucket_index >= res_table->num_nh_buckets) {
3777                 NL_SET_ERR_MSG(extack, "Bucket index out of bounds");
3778                 return -ENOENT;
3779         }
3780
3781         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
3782         if (!skb)
3783                 return -ENOBUFS;
3784
3785         err = nh_fill_res_bucket(skb, nh, &res_table->nh_buckets[bucket_index],
3786                                  bucket_index, RTM_NEWNEXTHOPBUCKET,
3787                                  NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
3788                                  0, extack);
3789         if (err < 0) {
3790                 WARN_ON(err == -EMSGSIZE);
3791                 goto errout_free;
3792         }
3793
3794         return rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
3795
3796 errout_free:
3797         kfree_skb(skb);
3798         return err;
3799 }
3800
3801 static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
3802 {
3803         unsigned int hash = nh_dev_hashfn(dev->ifindex);
3804         struct net *net = dev_net(dev);
3805         struct hlist_head *head = &net->nexthop.devhash[hash];
3806         struct hlist_node *n;
3807         struct nh_info *nhi;
3808
3809         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
3810                 if (nhi->fib_nhc.nhc_dev == dev) {
3811                         if (nhi->family == AF_INET)
3812                                 fib_nhc_update_mtu(&nhi->fib_nhc, dev->mtu,
3813                                                    orig_mtu);
3814                 }
3815         }
3816 }
3817
3818 /* rtnl */
3819 static int nh_netdev_event(struct notifier_block *this,
3820                            unsigned long event, void *ptr)
3821 {
3822         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3823         struct netdev_notifier_info_ext *info_ext;
3824
3825         switch (event) {
3826         case NETDEV_DOWN:
3827         case NETDEV_UNREGISTER:
3828                 nexthop_flush_dev(dev, event);
3829                 break;
3830         case NETDEV_CHANGE:
3831                 if (!(dev_get_flags(dev) & (IFF_RUNNING | IFF_LOWER_UP)))
3832                         nexthop_flush_dev(dev, event);
3833                 break;
3834         case NETDEV_CHANGEMTU:
3835                 info_ext = ptr;
3836                 nexthop_sync_mtu(dev, info_ext->ext.mtu);
3837                 rt_cache_flush(dev_net(dev));
3838                 break;
3839         }
3840         return NOTIFY_DONE;
3841 }
3842
3843 static struct notifier_block nh_netdev_notifier = {
3844         .notifier_call = nh_netdev_event,
3845 };
3846
3847 static int nexthops_dump(struct net *net, struct notifier_block *nb,
3848                          enum nexthop_event_type event_type,
3849                          struct netlink_ext_ack *extack)
3850 {
3851         struct rb_root *root = &net->nexthop.rb_root;
3852         struct rb_node *node;
3853         int err = 0;
3854
3855         for (node = rb_first(root); node; node = rb_next(node)) {
3856                 struct nexthop *nh;
3857
3858                 nh = rb_entry(node, struct nexthop, rb_node);
3859                 err = call_nexthop_notifier(nb, net, event_type, nh, extack);
3860                 if (err)
3861                         break;
3862         }
3863
3864         return err;
3865 }
3866
3867 int register_nexthop_notifier(struct net *net, struct notifier_block *nb,
3868                               struct netlink_ext_ack *extack)
3869 {
3870         int err;
3871
3872         rtnl_lock();
3873         err = nexthops_dump(net, nb, NEXTHOP_EVENT_REPLACE, extack);
3874         if (err)
3875                 goto unlock;
3876         err = blocking_notifier_chain_register(&net->nexthop.notifier_chain,
3877                                                nb);
3878 unlock:
3879         rtnl_unlock();
3880         return err;
3881 }
3882 EXPORT_SYMBOL(register_nexthop_notifier);
3883
3884 int __unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
3885 {
3886         int err;
3887
3888         err = blocking_notifier_chain_unregister(&net->nexthop.notifier_chain,
3889                                                  nb);
3890         if (!err)
3891                 nexthops_dump(net, nb, NEXTHOP_EVENT_DEL, NULL);
3892         return err;
3893 }
3894 EXPORT_SYMBOL(__unregister_nexthop_notifier);
3895
3896 int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
3897 {
3898         int err;
3899
3900         rtnl_lock();
3901         err = __unregister_nexthop_notifier(net, nb);
3902         rtnl_unlock();
3903         return err;
3904 }
3905 EXPORT_SYMBOL(unregister_nexthop_notifier);
3906
3907 void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap)
3908 {
3909         struct nexthop *nexthop;
3910
3911         rcu_read_lock();
3912
3913         nexthop = nexthop_find_by_id(net, id);
3914         if (!nexthop)
3915                 goto out;
3916
3917         nexthop->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
3918         if (offload)
3919                 nexthop->nh_flags |= RTNH_F_OFFLOAD;
3920         if (trap)
3921                 nexthop->nh_flags |= RTNH_F_TRAP;
3922
3923 out:
3924         rcu_read_unlock();
3925 }
3926 EXPORT_SYMBOL(nexthop_set_hw_flags);
3927
3928 void nexthop_bucket_set_hw_flags(struct net *net, u32 id, u16 bucket_index,
3929                                  bool offload, bool trap)
3930 {
3931         struct nh_res_table *res_table;
3932         struct nh_res_bucket *bucket;
3933         struct nexthop *nexthop;
3934         struct nh_group *nhg;
3935
3936         rcu_read_lock();
3937
3938         nexthop = nexthop_find_by_id(net, id);
3939         if (!nexthop || !nexthop->is_group)
3940                 goto out;
3941
3942         nhg = rcu_dereference(nexthop->nh_grp);
3943         if (!nhg->resilient)
3944                 goto out;
3945
3946         if (bucket_index >= nhg->res_table->num_nh_buckets)
3947                 goto out;
3948
3949         res_table = rcu_dereference(nhg->res_table);
3950         bucket = &res_table->nh_buckets[bucket_index];
3951         bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
3952         if (offload)
3953                 bucket->nh_flags |= RTNH_F_OFFLOAD;
3954         if (trap)
3955                 bucket->nh_flags |= RTNH_F_TRAP;
3956
3957 out:
3958         rcu_read_unlock();
3959 }
3960 EXPORT_SYMBOL(nexthop_bucket_set_hw_flags);
3961
3962 void nexthop_res_grp_activity_update(struct net *net, u32 id, u16 num_buckets,
3963                                      unsigned long *activity)
3964 {
3965         struct nh_res_table *res_table;
3966         struct nexthop *nexthop;
3967         struct nh_group *nhg;
3968         u16 i;
3969
3970         rcu_read_lock();
3971
3972         nexthop = nexthop_find_by_id(net, id);
3973         if (!nexthop || !nexthop->is_group)
3974                 goto out;
3975
3976         nhg = rcu_dereference(nexthop->nh_grp);
3977         if (!nhg->resilient)
3978                 goto out;
3979
3980         /* Instead of silently ignoring some buckets, demand that the sizes
3981          * be the same.
3982          */
3983         res_table = rcu_dereference(nhg->res_table);
3984         if (num_buckets != res_table->num_nh_buckets)
3985                 goto out;
3986
3987         for (i = 0; i < num_buckets; i++) {
3988                 if (test_bit(i, activity))
3989                         nh_res_bucket_set_busy(&res_table->nh_buckets[i]);
3990         }
3991
3992 out:
3993         rcu_read_unlock();
3994 }
3995 EXPORT_SYMBOL(nexthop_res_grp_activity_update);
3996
3997 static void __net_exit nexthop_net_exit_batch_rtnl(struct list_head *net_list,
3998                                                    struct list_head *dev_to_kill)
3999 {
4000         struct net *net;
4001
4002         ASSERT_RTNL();
4003         list_for_each_entry(net, net_list, exit_list)
4004                 flush_all_nexthops(net);
4005 }
4006
4007 static void __net_exit nexthop_net_exit(struct net *net)
4008 {
4009         kfree(net->nexthop.devhash);
4010         net->nexthop.devhash = NULL;
4011 }
4012
4013 static int __net_init nexthop_net_init(struct net *net)
4014 {
4015         size_t sz = sizeof(struct hlist_head) * NH_DEV_HASHSIZE;
4016
4017         net->nexthop.rb_root = RB_ROOT;
4018         net->nexthop.devhash = kzalloc(sz, GFP_KERNEL);
4019         if (!net->nexthop.devhash)
4020                 return -ENOMEM;
4021         BLOCKING_INIT_NOTIFIER_HEAD(&net->nexthop.notifier_chain);
4022
4023         return 0;
4024 }
4025
4026 static struct pernet_operations nexthop_net_ops = {
4027         .init = nexthop_net_init,
4028         .exit = nexthop_net_exit,
4029         .exit_batch_rtnl = nexthop_net_exit_batch_rtnl,
4030 };
4031
4032 static int __init nexthop_init(void)
4033 {
4034         register_pernet_subsys(&nexthop_net_ops);
4035
4036         register_netdevice_notifier(&nh_netdev_notifier);
4037
4038         rtnl_register(PF_UNSPEC, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
4039         rtnl_register(PF_UNSPEC, RTM_DELNEXTHOP, rtm_del_nexthop, NULL, 0);
4040         rtnl_register(PF_UNSPEC, RTM_GETNEXTHOP, rtm_get_nexthop,
4041                       rtm_dump_nexthop, 0);
4042
4043         rtnl_register(PF_INET, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
4044         rtnl_register(PF_INET, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
4045
4046         rtnl_register(PF_INET6, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
4047         rtnl_register(PF_INET6, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
4048
4049         rtnl_register(PF_UNSPEC, RTM_GETNEXTHOPBUCKET, rtm_get_nexthop_bucket,
4050                       rtm_dump_nexthop_bucket, 0);
4051
4052         return 0;
4053 }
4054 subsys_initcall(nexthop_init);