net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <[email protected]>
  10  *              Alan Cox, <[email protected]>
  11  *              Linus Torvalds, <[email protected]>
  12  *              Alexey Kuznetsov, <[email protected]>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              ([email protected])      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <linux/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #endif
 112 #include <net/secure_seq.h>
 113 #include <net/ip_tunnels.h>
 114 #include <net/l3mdev.h>
 115
 116 #include "fib_lookup.h"
 117
 118 #define RT_FL_TOS(oldflp4) \
 119         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_redirect_number __read_mostly  = 9;
 125 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 126 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 127 static int ip_rt_error_cost __read_mostly       = HZ;
 128 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 130 static u32 ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 131 static int ip_rt_min_advmss __read_mostly       = 256;
 132
 133 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 134
 135 /*
 136  *      Interface to generic destination cache.
 137  */
 138
 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 140 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 141 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 143 static void              ipv4_link_failure(struct sk_buff *skb);
 144 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 145                                            struct sk_buff *skb, u32 mtu);
 146 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 147                                         struct sk_buff *skb);
 148 static void             ipv4_dst_destroy(struct dst_entry *dst);
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
 160
 161 static struct dst_ops ipv4_dst_ops = {
 162         .family =               AF_INET,
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174         .confirm_neigh =        ipv4_confirm_neigh,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    0, /* st->in_hit */
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    0, /* st->out_hit */
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    0, /* st->gc_total */
 310                    0, /* st->gc_ignored */
 311                    0, /* st->gc_goal_miss */
 312                    0, /* st->gc_dst_overflow */
 313                    0, /* st->in_hlist_search */
 314                    0  /* st->out_hlist_search */
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .open    = rt_cpu_seq_open,
 334         .read    = seq_read,
 335         .llseek  = seq_lseek,
 336         .release = seq_release,
 337 };
 338
 339 #ifdef CONFIG_IP_ROUTE_CLASSID
 340 static int rt_acct_proc_show(struct seq_file *m, void *v)
 341 {
 342         struct ip_rt_acct *dst, *src;
 343         unsigned int i, j;
 344
 345         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 346         if (!dst)
 347                 return -ENOMEM;
 348
 349         for_each_possible_cpu(i) {
 350                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 351                 for (j = 0; j < 256; j++) {
 352                         dst[j].o_bytes   += src[j].o_bytes;
 353                         dst[j].o_packets += src[j].o_packets;
 354                         dst[j].i_bytes   += src[j].i_bytes;
 355                         dst[j].i_packets += src[j].i_packets;
 356                 }
 357         }
 358
 359         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 360         kfree(dst);
 361         return 0;
 362 }
 363 #endif
 364
 365 static int __net_init ip_rt_do_proc_init(struct net *net)
 366 {
 367         struct proc_dir_entry *pde;
 368
 369         pde = proc_create("rt_cache", 0444, net->proc_net,
 370                           &rt_cache_seq_fops);
 371         if (!pde)
 372                 goto err1;
 373
 374         pde = proc_create("rt_cache", 0444,
 375                           net->proc_net_stat, &rt_cpu_seq_fops);
 376         if (!pde)
 377                 goto err2;
 378
 379 #ifdef CONFIG_IP_ROUTE_CLASSID
 380         pde = proc_create_single("rt_acct", 0, net->proc_net,
 381                         rt_acct_proc_show);
 382         if (!pde)
 383                 goto err3;
 384 #endif
 385         return 0;
 386
 387 #ifdef CONFIG_IP_ROUTE_CLASSID
 388 err3:
 389         remove_proc_entry("rt_cache", net->proc_net_stat);
 390 #endif
 391 err2:
 392         remove_proc_entry("rt_cache", net->proc_net);
 393 err1:
 394         return -ENOMEM;
 395 }
 396
 397 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 398 {
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400         remove_proc_entry("rt_cache", net->proc_net);
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402         remove_proc_entry("rt_acct", net->proc_net);
 403 #endif
 404 }
 405
 406 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 407         .init = ip_rt_do_proc_init,
 408         .exit = ip_rt_do_proc_exit,
 409 };
 410
 411 static int __init ip_rt_proc_init(void)
 412 {
 413         return register_pernet_subsys(&ip_rt_proc_ops);
 414 }
 415
 416 #else
 417 static inline int ip_rt_proc_init(void)
 418 {
 419         return 0;
 420 }
 421 #endif /* CONFIG_PROC_FS */
 422
 423 static inline bool rt_is_expired(const struct rtable *rth)
 424 {
 425         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 426 }
 427
 428 void rt_cache_flush(struct net *net)
 429 {
 430         rt_genid_bump_ipv4(net);
 431 }
 432
 433 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 434                                            struct sk_buff *skb,
 435                                            const void *daddr)
 436 {
 437         struct net_device *dev = dst->dev;
 438         const __be32 *pkey = daddr;
 439         const struct rtable *rt;
 440         struct neighbour *n;
 441
 442         rt = (const struct rtable *) dst;
 443         if (rt->rt_gateway)
 444                 pkey = (const __be32 *) &rt->rt_gateway;
 445         else if (skb)
 446                 pkey = &ip_hdr(skb)->daddr;
 447
 448         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 449         if (n)
 450                 return n;
 451         return neigh_create(&arp_tbl, pkey, dev);
 452 }
 453
 454 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 455 {
 456         struct net_device *dev = dst->dev;
 457         const __be32 *pkey = daddr;
 458         const struct rtable *rt;
 459
 460         rt = (const struct rtable *)dst;
 461         if (rt->rt_gateway)
 462                 pkey = (const __be32 *)&rt->rt_gateway;
 463         else if (!daddr ||
 464                  (rt->rt_flags &
 465                   (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
 466                 return;
 467
 468         __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
 469 }
 470
 471 #define IP_IDENTS_SZ 2048u
 472
 473 static atomic_t *ip_idents __read_mostly;
 474 static u32 *ip_tstamps __read_mostly;
 475
 476 /* In order to protect privacy, we add a perturbation to identifiers
 477  * if one generator is seldom used. This makes hard for an attacker
 478  * to infer how many packets were sent between two points in time.
 479  */
 480 u32 ip_idents_reserve(u32 hash, int segs)
 481 {
 482         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 483         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 484         u32 old = READ_ONCE(*p_tstamp);
 485         u32 now = (u32)jiffies;
 486         u32 new, delta = 0;
 487
 488         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 489                 delta = prandom_u32_max(now - old);
 490
 491         /* Do not use atomic_add_return() as it makes UBSAN unhappy */
 492         do {
 493                 old = (u32)atomic_read(p_id);
 494                 new = old + delta + segs;
 495         } while (atomic_cmpxchg(p_id, old, new) != old);
 496
 497         return new - segs;
 498 }
 499 EXPORT_SYMBOL(ip_idents_reserve);
 500
 501 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 502 {
 503         static u32 ip_idents_hashrnd __read_mostly;
 504         u32 hash, id;
 505
 506         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 507
 508         hash = jhash_3words((__force u32)iph->daddr,
 509                             (__force u32)iph->saddr,
 510                             iph->protocol ^ net_hash_mix(net),
 511                             ip_idents_hashrnd);
 512         id = ip_idents_reserve(hash, segs);
 513         iph->id = htons(id);
 514 }
 515 EXPORT_SYMBOL(__ip_select_ident);
 516
 517 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
 518                              const struct sock *sk,
 519                              const struct iphdr *iph,
 520                              int oif, u8 tos,
 521                              u8 prot, u32 mark, int flow_flags)
 522 {
 523         if (sk) {
 524                 const struct inet_sock *inet = inet_sk(sk);
 525
 526                 oif = sk->sk_bound_dev_if;
 527                 mark = sk->sk_mark;
 528                 tos = RT_CONN_FLAGS(sk);
 529                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 530         }
 531         flowi4_init_output(fl4, oif, mark, tos,
 532                            RT_SCOPE_UNIVERSE, prot,
 533                            flow_flags,
 534                            iph->daddr, iph->saddr, 0, 0,
 535                            sock_net_uid(net, sk));
 536 }
 537
 538 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 539                                const struct sock *sk)
 540 {
 541         const struct net *net = dev_net(skb->dev);
 542         const struct iphdr *iph = ip_hdr(skb);
 543         int oif = skb->dev->ifindex;
 544         u8 tos = RT_TOS(iph->tos);
 545         u8 prot = iph->protocol;
 546         u32 mark = skb->mark;
 547
 548         __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
 549 }
 550
 551 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 552 {
 553         const struct inet_sock *inet = inet_sk(sk);
 554         const struct ip_options_rcu *inet_opt;
 555         __be32 daddr = inet->inet_daddr;
 556
 557         rcu_read_lock();
 558         inet_opt = rcu_dereference(inet->inet_opt);
 559         if (inet_opt && inet_opt->opt.srr)
 560                 daddr = inet_opt->opt.faddr;
 561         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 562                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 563                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 564                            inet_sk_flowi_flags(sk),
 565                            daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
 566         rcu_read_unlock();
 567 }
 568
 569 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 570                                  const struct sk_buff *skb)
 571 {
 572         if (skb)
 573                 build_skb_flow_key(fl4, skb, sk);
 574         else
 575                 build_sk_flow_key(fl4, sk);
 576 }
 577
 578 static DEFINE_SPINLOCK(fnhe_lock);
 579
 580 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 581 {
 582         struct rtable *rt;
 583
 584         rt = rcu_dereference(fnhe->fnhe_rth_input);
 585         if (rt) {
 586                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 587                 dst_dev_put(&rt->dst);
 588                 dst_release(&rt->dst);
 589         }
 590         rt = rcu_dereference(fnhe->fnhe_rth_output);
 591         if (rt) {
 592                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 593                 dst_dev_put(&rt->dst);
 594                 dst_release(&rt->dst);
 595         }
 596 }
 597
 598 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 599 {
 600         struct fib_nh_exception *fnhe, *oldest;
 601
 602         oldest = rcu_dereference(hash->chain);
 603         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 604              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 605                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 606                         oldest = fnhe;
 607         }
 608         fnhe_flush_routes(oldest);
 609         return oldest;
 610 }
 611
 612 static inline u32 fnhe_hashfun(__be32 daddr)
 613 {
 614         static u32 fnhe_hashrnd __read_mostly;
 615         u32 hval;
 616
 617         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 618         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 619         return hash_32(hval, FNHE_HASH_SHIFT);
 620 }
 621
 622 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 623 {
 624         rt->rt_pmtu = fnhe->fnhe_pmtu;
 625         rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
 626         rt->dst.expires = fnhe->fnhe_expires;
 627
 628         if (fnhe->fnhe_gw) {
 629                 rt->rt_flags |= RTCF_REDIRECTED;
 630                 rt->rt_gateway = fnhe->fnhe_gw;
 631                 rt->rt_uses_gateway = 1;
 632         }
 633 }
 634
 635 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 636                                   u32 pmtu, bool lock, unsigned long expires)
 637 {
 638         struct fnhe_hash_bucket *hash;
 639         struct fib_nh_exception *fnhe;
 640         struct rtable *rt;
 641         u32 genid, hval;
 642         unsigned int i;
 643         int depth;
 644
 645         genid = fnhe_genid(dev_net(nh->nh_dev));
 646         hval = fnhe_hashfun(daddr);
 647
 648         spin_lock_bh(&fnhe_lock);
 649
 650         hash = rcu_dereference(nh->nh_exceptions);
 651         if (!hash) {
 652                 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
 653                 if (!hash)
 654                         goto out_unlock;
 655                 rcu_assign_pointer(nh->nh_exceptions, hash);
 656         }
 657
 658         hash += hval;
 659
 660         depth = 0;
 661         for (fnhe = rcu_dereference(hash->chain); fnhe;
 662              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 663                 if (fnhe->fnhe_daddr == daddr)
 664                         break;
 665                 depth++;
 666         }
 667
 668         if (fnhe) {
 669                 if (fnhe->fnhe_genid != genid)
 670                         fnhe->fnhe_genid = genid;
 671                 if (gw)
 672                         fnhe->fnhe_gw = gw;
 673                 if (pmtu) {
 674                         fnhe->fnhe_pmtu = pmtu;
 675                         fnhe->fnhe_mtu_locked = lock;
 676                 }
 677                 fnhe->fnhe_expires = max(1UL, expires);
 678                 /* Update all cached dsts too */
 679                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 680                 if (rt)
 681                         fill_route_from_fnhe(rt, fnhe);
 682                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 683                 if (rt)
 684                         fill_route_from_fnhe(rt, fnhe);
 685         } else {
 686                 if (depth > FNHE_RECLAIM_DEPTH)
 687                         fnhe = fnhe_oldest(hash);
 688                 else {
 689                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 690                         if (!fnhe)
 691                                 goto out_unlock;
 692
 693                         fnhe->fnhe_next = hash->chain;
 694                         rcu_assign_pointer(hash->chain, fnhe);
 695                 }
 696                 fnhe->fnhe_genid = genid;
 697                 fnhe->fnhe_daddr = daddr;
 698                 fnhe->fnhe_gw = gw;
 699                 fnhe->fnhe_pmtu = pmtu;
 700                 fnhe->fnhe_mtu_locked = lock;
 701                 fnhe->fnhe_expires = max(1UL, expires);
 702
 703                 /* Exception created; mark the cached routes for the nexthop
 704                  * stale, so anyone caching it rechecks if this exception
 705                  * applies to them.
 706                  */
 707                 rt = rcu_dereference(nh->nh_rth_input);
 708                 if (rt)
 709                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 710
 711                 for_each_possible_cpu(i) {
 712                         struct rtable __rcu **prt;
 713                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 714                         rt = rcu_dereference(*prt);
 715                         if (rt)
 716                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 717                 }
 718         }
 719
 720         fnhe->fnhe_stamp = jiffies;
 721
 722 out_unlock:
 723         spin_unlock_bh(&fnhe_lock);
 724 }
 725
 726 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 727                              bool kill_route)
 728 {
 729         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 730         __be32 old_gw = ip_hdr(skb)->saddr;
 731         struct net_device *dev = skb->dev;
 732         struct in_device *in_dev;
 733         struct fib_result res;
 734         struct neighbour *n;
 735         struct net *net;
 736
 737         switch (icmp_hdr(skb)->code & 7) {
 738         case ICMP_REDIR_NET:
 739         case ICMP_REDIR_NETTOS:
 740         case ICMP_REDIR_HOST:
 741         case ICMP_REDIR_HOSTTOS:
 742                 break;
 743
 744         default:
 745                 return;
 746         }
 747
 748         if (rt->rt_gateway != old_gw)
 749                 return;
 750
 751         in_dev = __in_dev_get_rcu(dev);
 752         if (!in_dev)
 753                 return;
 754
 755         net = dev_net(dev);
 756         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 757             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 758             ipv4_is_zeronet(new_gw))
 759                 goto reject_redirect;
 760
 761         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 762                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 763                         goto reject_redirect;
 764                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 765                         goto reject_redirect;
 766         } else {
 767                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 768                         goto reject_redirect;
 769         }
 770
 771         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 772         if (!n)
 773                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 774         if (!IS_ERR(n)) {
 775                 if (!(n->nud_state & NUD_VALID)) {
 776                         neigh_event_send(n, NULL);
 777                 } else {
 778                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 779                                 struct fib_nh *nh = &FIB_RES_NH(res);
 780
 781                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 782                                                 0, false,
 783                                                 jiffies + ip_rt_gc_timeout);
 784                         }
 785                         if (kill_route)
 786                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 787                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 788                 }
 789                 neigh_release(n);
 790         }
 791         return;
 792
 793 reject_redirect:
 794 #ifdef CONFIG_IP_ROUTE_VERBOSE
 795         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 796                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 797                 __be32 daddr = iph->daddr;
 798                 __be32 saddr = iph->saddr;
 799
 800                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 801                                      "  Advised path = %pI4 -> %pI4\n",
 802                                      &old_gw, dev->name, &new_gw,
 803                                      &saddr, &daddr);
 804         }
 805 #endif
 806         ;
 807 }
 808
 809 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 810 {
 811         struct rtable *rt;
 812         struct flowi4 fl4;
 813         const struct iphdr *iph = (const struct iphdr *) skb->data;
 814         struct net *net = dev_net(skb->dev);
 815         int oif = skb->dev->ifindex;
 816         u8 tos = RT_TOS(iph->tos);
 817         u8 prot = iph->protocol;
 818         u32 mark = skb->mark;
 819
 820         rt = (struct rtable *) dst;
 821
 822         __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
 823         __ip_do_redirect(rt, skb, &fl4, true);
 824 }
 825
 826 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 827 {
 828         struct rtable *rt = (struct rtable *)dst;
 829         struct dst_entry *ret = dst;
 830
 831         if (rt) {
 832                 if (dst->obsolete > 0) {
 833                         ip_rt_put(rt);
 834                         ret = NULL;
 835                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 836                            rt->dst.expires) {
 837                         ip_rt_put(rt);
 838                         ret = NULL;
 839                 }
 840         }
 841         return ret;
 842 }
 843
 844 /*
 845  * Algorithm:
 846  *      1. The first ip_rt_redirect_number redirects are sent
 847  *         with exponential backoff, then we stop sending them at all,
 848  *         assuming that the host ignores our redirects.
 849  *      2. If we did not see packets requiring redirects
 850  *         during ip_rt_redirect_silence, we assume that the host
 851  *         forgot redirected route and start to send redirects again.
 852  *
 853  * This algorithm is much cheaper and more intelligent than dumb load limiting
 854  * in icmp.c.
 855  *
 856  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 857  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 858  */
 859
 860 void ip_rt_send_redirect(struct sk_buff *skb)
 861 {
 862         struct rtable *rt = skb_rtable(skb);
 863         struct in_device *in_dev;
 864         struct inet_peer *peer;
 865         struct net *net;
 866         int log_martians;
 867         int vif;
 868
 869         rcu_read_lock();
 870         in_dev = __in_dev_get_rcu(rt->dst.dev);
 871         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 872                 rcu_read_unlock();
 873                 return;
 874         }
 875         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 876         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 877         rcu_read_unlock();
 878
 879         net = dev_net(rt->dst.dev);
 880         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 881         if (!peer) {
 882                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 883                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 884                 return;
 885         }
 886
 887         /* No redirected packets during ip_rt_redirect_silence;
 888          * reset the algorithm.
 889          */
 890         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) {
 891                 peer->rate_tokens = 0;
 892                 peer->n_redirects = 0;
 893         }
 894
 895         /* Too many ignored redirects; do not send anything
 896          * set dst.rate_last to the last seen redirected packet.
 897          */
 898         if (peer->n_redirects >= ip_rt_redirect_number) {
 899                 peer->rate_last = jiffies;
 900                 goto out_put_peer;
 901         }
 902
 903         /* Check for load limit; set rate_last to the latest sent
 904          * redirect.
 905          */
 906         if (peer->rate_tokens == 0 ||
 907             time_after(jiffies,
 908                        (peer->rate_last +
 909                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 910                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 911
 912                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 913                 peer->rate_last = jiffies;
 914                 ++peer->rate_tokens;
 915                 ++peer->n_redirects;
 916 #ifdef CONFIG_IP_ROUTE_VERBOSE
 917                 if (log_martians &&
 918                     peer->rate_tokens == ip_rt_redirect_number)
 919                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 920                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 921                                              &ip_hdr(skb)->daddr, &gw);
 922 #endif
 923         }
 924 out_put_peer:
 925         inet_putpeer(peer);
 926 }
 927
 928 static int ip_error(struct sk_buff *skb)
 929 {
 930         struct rtable *rt = skb_rtable(skb);
 931         struct net_device *dev = skb->dev;
 932         struct in_device *in_dev;
 933         struct inet_peer *peer;
 934         unsigned long now;
 935         struct net *net;
 936         bool send;
 937         int code;
 938
 939         if (netif_is_l3_master(skb->dev)) {
 940                 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
 941                 if (!dev)
 942                         goto out;
 943         }
 944
 945         in_dev = __in_dev_get_rcu(dev);
 946
 947         /* IP on this device is disabled. */
 948         if (!in_dev)
 949                 goto out;
 950
 951         net = dev_net(rt->dst.dev);
 952         if (!IN_DEV_FORWARD(in_dev)) {
 953                 switch (rt->dst.error) {
 954                 case EHOSTUNREACH:
 955                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 956                         break;
 957
 958                 case ENETUNREACH:
 959                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 960                         break;
 961                 }
 962                 goto out;
 963         }
 964
 965         switch (rt->dst.error) {
 966         case EINVAL:
 967         default:
 968                 goto out;
 969         case EHOSTUNREACH:
 970                 code = ICMP_HOST_UNREACH;
 971                 break;
 972         case ENETUNREACH:
 973                 code = ICMP_NET_UNREACH;
 974                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 975                 break;
 976         case EACCES:
 977                 code = ICMP_PKT_FILTERED;
 978                 break;
 979         }
 980
 981         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 982                                l3mdev_master_ifindex(skb->dev), 1);
 983
 984         send = true;
 985         if (peer) {
 986                 now = jiffies;
 987                 peer->rate_tokens += now - peer->rate_last;
 988                 if (peer->rate_tokens > ip_rt_error_burst)
 989                         peer->rate_tokens = ip_rt_error_burst;
 990                 peer->rate_last = now;
 991                 if (peer->rate_tokens >= ip_rt_error_cost)
 992                         peer->rate_tokens -= ip_rt_error_cost;
 993                 else
 994                         send = false;
 995                 inet_putpeer(peer);
 996         }
 997         if (send)
 998                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 999
1000 out:    kfree_skb(skb);
1001         return 0;
1002 }
1003
1004 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1005 {
1006         struct dst_entry *dst = &rt->dst;
1007         u32 old_mtu = ipv4_mtu(dst);
1008         struct fib_result res;
1009         bool lock = false;
1010
1011         if (ip_mtu_locked(dst))
1012                 return;
1013
1014         if (old_mtu < mtu)
1015                 return;
1016
1017         if (mtu < ip_rt_min_pmtu) {
1018                 lock = true;
1019                 mtu = min(old_mtu, ip_rt_min_pmtu);
1020         }
1021
1022         if (rt->rt_pmtu == mtu && !lock &&
1023             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1024                 return;
1025
1026         rcu_read_lock();
1027         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1028                 struct fib_nh *nh = &FIB_RES_NH(res);
1029
1030                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock,
1031                                       jiffies + ip_rt_mtu_expires);
1032         }
1033         rcu_read_unlock();
1034 }
1035
1036 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1037                               struct sk_buff *skb, u32 mtu)
1038 {
1039         struct rtable *rt = (struct rtable *) dst;
1040         struct flowi4 fl4;
1041
1042         ip_rt_build_flow_key(&fl4, sk, skb);
1043         __ip_rt_update_pmtu(rt, &fl4, mtu);
1044 }
1045
1046 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1047                       int oif, u8 protocol)
1048 {
1049         const struct iphdr *iph = (const struct iphdr *) skb->data;
1050         struct flowi4 fl4;
1051         struct rtable *rt;
1052         u32 mark = IP4_REPLY_MARK(net, skb->mark);
1053
1054         __build_flow_key(net, &fl4, NULL, iph, oif,
1055                          RT_TOS(iph->tos), protocol, mark, 0);
1056         rt = __ip_route_output_key(net, &fl4);
1057         if (!IS_ERR(rt)) {
1058                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1059                 ip_rt_put(rt);
1060         }
1061 }
1062 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1063
1064 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1065 {
1066         const struct iphdr *iph = (const struct iphdr *) skb->data;
1067         struct flowi4 fl4;
1068         struct rtable *rt;
1069
1070         __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1071
1072         if (!fl4.flowi4_mark)
1073                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1074
1075         rt = __ip_route_output_key(sock_net(sk), &fl4);
1076         if (!IS_ERR(rt)) {
1077                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1078                 ip_rt_put(rt);
1079         }
1080 }
1081
1082 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1083 {
1084         const struct iphdr *iph = (const struct iphdr *) skb->data;
1085         struct flowi4 fl4;
1086         struct rtable *rt;
1087         struct dst_entry *odst = NULL;
1088         bool new = false;
1089         struct net *net = sock_net(sk);
1090
1091         bh_lock_sock(sk);
1092
1093         if (!ip_sk_accept_pmtu(sk))
1094                 goto out;
1095
1096         odst = sk_dst_get(sk);
1097
1098         if (sock_owned_by_user(sk) || !odst) {
1099                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1100                 goto out;
1101         }
1102
1103         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1104
1105         rt = (struct rtable *)odst;
1106         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1107                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1108                 if (IS_ERR(rt))
1109                         goto out;
1110
1111                 new = true;
1112         }
1113
1114         __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu);
1115
1116         if (!dst_check(&rt->dst, 0)) {
1117                 if (new)
1118                         dst_release(&rt->dst);
1119
1120                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1121                 if (IS_ERR(rt))
1122                         goto out;
1123
1124                 new = true;
1125         }
1126
1127         if (new)
1128                 sk_dst_set(sk, &rt->dst);
1129
1130 out:
1131         bh_unlock_sock(sk);
1132         dst_release(odst);
1133 }
1134 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1135
1136 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1137                    int oif, u8 protocol)
1138 {
1139         const struct iphdr *iph = (const struct iphdr *) skb->data;
1140         struct flowi4 fl4;
1141         struct rtable *rt;
1142
1143         __build_flow_key(net, &fl4, NULL, iph, oif,
1144                          RT_TOS(iph->tos), protocol, 0, 0);
1145         rt = __ip_route_output_key(net, &fl4);
1146         if (!IS_ERR(rt)) {
1147                 __ip_do_redirect(rt, skb, &fl4, false);
1148                 ip_rt_put(rt);
1149         }
1150 }
1151 EXPORT_SYMBOL_GPL(ipv4_redirect);
1152
1153 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1154 {
1155         const struct iphdr *iph = (const struct iphdr *) skb->data;
1156         struct flowi4 fl4;
1157         struct rtable *rt;
1158         struct net *net = sock_net(sk);
1159
1160         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1161         rt = __ip_route_output_key(net, &fl4);
1162         if (!IS_ERR(rt)) {
1163                 __ip_do_redirect(rt, skb, &fl4, false);
1164                 ip_rt_put(rt);
1165         }
1166 }
1167 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1168
1169 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1170 {
1171         struct rtable *rt = (struct rtable *) dst;
1172
1173         /* All IPV4 dsts are created with ->obsolete set to the value
1174          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1175          * into this function always.
1176          *
1177          * When a PMTU/redirect information update invalidates a route,
1178          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1179          * DST_OBSOLETE_DEAD by dst_free().
1180          */
1181         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1182                 return NULL;
1183         return dst;
1184 }
1185
1186 static void ipv4_link_failure(struct sk_buff *skb)
1187 {
1188         struct ip_options opt;
1189         struct rtable *rt;
1190         int res;
1191
1192         /* Recompile ip options since IPCB may not be valid anymore.
1193          */
1194         memset(&opt, 0, sizeof(opt));
1195         opt.optlen = ip_hdr(skb)->ihl*4 - sizeof(struct iphdr);
1196
1197         rcu_read_lock();
1198         res = __ip_options_compile(dev_net(skb->dev), &opt, skb, NULL);
1199         rcu_read_unlock();
1200
1201         if (res)
1202                 return;
1203
1204         __icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0, &opt);
1205
1206         rt = skb_rtable(skb);
1207         if (rt)
1208                 dst_set_expires(&rt->dst, 0);
1209 }
1210
1211 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1212 {
1213         pr_debug("%s: %pI4 -> %pI4, %s\n",
1214                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1215                  skb->dev ? skb->dev->name : "?");
1216         kfree_skb(skb);
1217         WARN_ON(1);
1218         return 0;
1219 }
1220
1221 /*
1222    We do not cache source address of outgoing interface,
1223    because it is used only by IP RR, TS and SRR options,
1224    so that it out of fast path.
1225
1226    BTW remember: "addr" is allowed to be not aligned
1227    in IP options!
1228  */
1229
1230 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1231 {
1232         __be32 src;
1233
1234         if (rt_is_output_route(rt))
1235                 src = ip_hdr(skb)->saddr;
1236         else {
1237                 struct fib_result res;
1238                 struct iphdr *iph = ip_hdr(skb);
1239                 struct flowi4 fl4 = {
1240                         .daddr = iph->daddr,
1241                         .saddr = iph->saddr,
1242                         .flowi4_tos = RT_TOS(iph->tos),
1243                         .flowi4_oif = rt->dst.dev->ifindex,
1244                         .flowi4_iif = skb->dev->ifindex,
1245                         .flowi4_mark = skb->mark,
1246                 };
1247
1248                 rcu_read_lock();
1249                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1250                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1251                 else
1252                         src = inet_select_addr(rt->dst.dev,
1253                                                rt_nexthop(rt, iph->daddr),
1254                                                RT_SCOPE_UNIVERSE);
1255                 rcu_read_unlock();
1256         }
1257         memcpy(addr, &src, 4);
1258 }
1259
1260 #ifdef CONFIG_IP_ROUTE_CLASSID
1261 static void set_class_tag(struct rtable *rt, u32 tag)
1262 {
1263         if (!(rt->dst.tclassid & 0xFFFF))
1264                 rt->dst.tclassid |= tag & 0xFFFF;
1265         if (!(rt->dst.tclassid & 0xFFFF0000))
1266                 rt->dst.tclassid |= tag & 0xFFFF0000;
1267 }
1268 #endif
1269
1270 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1271 {
1272         unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1273         unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1274                                     ip_rt_min_advmss);
1275
1276         return min(advmss, IPV4_MAX_PMTU - header_size);
1277 }
1278
1279 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1280 {
1281         const struct rtable *rt = (const struct rtable *) dst;
1282         unsigned int mtu = rt->rt_pmtu;
1283
1284         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1285                 mtu = dst_metric_raw(dst, RTAX_MTU);
1286
1287         if (mtu)
1288                 return mtu;
1289
1290         mtu = READ_ONCE(dst->dev->mtu);
1291
1292         if (unlikely(ip_mtu_locked(dst))) {
1293                 if (rt->rt_uses_gateway && mtu > 576)
1294                         mtu = 576;
1295         }
1296
1297         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1298
1299         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1300 }
1301
1302 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1303 {
1304         struct fnhe_hash_bucket *hash;
1305         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1306         u32 hval = fnhe_hashfun(daddr);
1307
1308         spin_lock_bh(&fnhe_lock);
1309
1310         hash = rcu_dereference_protected(nh->nh_exceptions,
1311                                          lockdep_is_held(&fnhe_lock));
1312         hash += hval;
1313
1314         fnhe_p = &hash->chain;
1315         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1316         while (fnhe) {
1317                 if (fnhe->fnhe_daddr == daddr) {
1318                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1319                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1320                         /* set fnhe_daddr to 0 to ensure it won't bind with
1321                          * new dsts in rt_bind_exception().
1322                          */
1323                         fnhe->fnhe_daddr = 0;
1324                         fnhe_flush_routes(fnhe);
1325                         kfree_rcu(fnhe, rcu);
1326                         break;
1327                 }
1328                 fnhe_p = &fnhe->fnhe_next;
1329                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1330                                                  lockdep_is_held(&fnhe_lock));
1331         }
1332
1333         spin_unlock_bh(&fnhe_lock);
1334 }
1335
1336 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1337 {
1338         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1339         struct fib_nh_exception *fnhe;
1340         u32 hval;
1341
1342         if (!hash)
1343                 return NULL;
1344
1345         hval = fnhe_hashfun(daddr);
1346
1347         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1348              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1349                 if (fnhe->fnhe_daddr == daddr) {
1350                         if (fnhe->fnhe_expires &&
1351                             time_after(jiffies, fnhe->fnhe_expires)) {
1352                                 ip_del_fnhe(nh, daddr);
1353                                 break;
1354                         }
1355                         return fnhe;
1356                 }
1357         }
1358         return NULL;
1359 }
1360
1361 /* MTU selection:
1362  * 1. mtu on route is locked - use it
1363  * 2. mtu from nexthop exception
1364  * 3. mtu from egress device
1365  */
1366
1367 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
1368 {
1369         struct fib_info *fi = res->fi;
1370         struct fib_nh *nh = &fi->fib_nh[res->nh_sel];
1371         struct net_device *dev = nh->nh_dev;
1372         u32 mtu = 0;
1373
1374         if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
1375             fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
1376                 mtu = fi->fib_mtu;
1377
1378         if (likely(!mtu)) {
1379                 struct fib_nh_exception *fnhe;
1380
1381                 fnhe = find_exception(nh, daddr);
1382                 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
1383                         mtu = fnhe->fnhe_pmtu;
1384         }
1385
1386         if (likely(!mtu))
1387                 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
1388
1389         return mtu - lwtunnel_headroom(nh->nh_lwtstate, mtu);
1390 }
1391
1392 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1393                               __be32 daddr, const bool do_cache)
1394 {
1395         bool ret = false;
1396
1397         spin_lock_bh(&fnhe_lock);
1398
1399         if (daddr == fnhe->fnhe_daddr) {
1400                 struct rtable __rcu **porig;
1401                 struct rtable *orig;
1402                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1403
1404                 if (rt_is_input_route(rt))
1405                         porig = &fnhe->fnhe_rth_input;
1406                 else
1407                         porig = &fnhe->fnhe_rth_output;
1408                 orig = rcu_dereference(*porig);
1409
1410                 if (fnhe->fnhe_genid != genid) {
1411                         fnhe->fnhe_genid = genid;
1412                         fnhe->fnhe_gw = 0;
1413                         fnhe->fnhe_pmtu = 0;
1414                         fnhe->fnhe_expires = 0;
1415                         fnhe->fnhe_mtu_locked = false;
1416                         fnhe_flush_routes(fnhe);
1417                         orig = NULL;
1418                 }
1419                 fill_route_from_fnhe(rt, fnhe);
1420                 if (!rt->rt_gateway)
1421                         rt->rt_gateway = daddr;
1422
1423                 if (do_cache) {
1424                         dst_hold(&rt->dst);
1425                         rcu_assign_pointer(*porig, rt);
1426                         if (orig) {
1427                                 dst_dev_put(&orig->dst);
1428                                 dst_release(&orig->dst);
1429                         }
1430                         ret = true;
1431                 }
1432
1433                 fnhe->fnhe_stamp = jiffies;
1434         }
1435         spin_unlock_bh(&fnhe_lock);
1436
1437         return ret;
1438 }
1439
1440 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1441 {
1442         struct rtable *orig, *prev, **p;
1443         bool ret = true;
1444
1445         if (rt_is_input_route(rt)) {
1446                 p = (struct rtable **)&nh->nh_rth_input;
1447         } else {
1448                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1449         }
1450         orig = *p;
1451
1452         /* hold dst before doing cmpxchg() to avoid race condition
1453          * on this dst
1454          */
1455         dst_hold(&rt->dst);
1456         prev = cmpxchg(p, orig, rt);
1457         if (prev == orig) {
1458                 if (orig) {
1459                         dst_dev_put(&orig->dst);
1460                         dst_release(&orig->dst);
1461                 }
1462         } else {
1463                 dst_release(&rt->dst);
1464                 ret = false;
1465         }
1466
1467         return ret;
1468 }
1469
1470 struct uncached_list {
1471         spinlock_t              lock;
1472         struct list_head        head;
1473 };
1474
1475 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1476
1477 void rt_add_uncached_list(struct rtable *rt)
1478 {
1479         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1480
1481         rt->rt_uncached_list = ul;
1482
1483         spin_lock_bh(&ul->lock);
1484         list_add_tail(&rt->rt_uncached, &ul->head);
1485         spin_unlock_bh(&ul->lock);
1486 }
1487
1488 void rt_del_uncached_list(struct rtable *rt)
1489 {
1490         if (!list_empty(&rt->rt_uncached)) {
1491                 struct uncached_list *ul = rt->rt_uncached_list;
1492
1493                 spin_lock_bh(&ul->lock);
1494                 list_del(&rt->rt_uncached);
1495                 spin_unlock_bh(&ul->lock);
1496         }
1497 }
1498
1499 static void ipv4_dst_destroy(struct dst_entry *dst)
1500 {
1501         struct rtable *rt = (struct rtable *)dst;
1502
1503         ip_dst_metrics_put(dst);
1504         rt_del_uncached_list(rt);
1505 }
1506
1507 void rt_flush_dev(struct net_device *dev)
1508 {
1509         struct net *net = dev_net(dev);
1510         struct rtable *rt;
1511         int cpu;
1512
1513         for_each_possible_cpu(cpu) {
1514                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1515
1516                 spin_lock_bh(&ul->lock);
1517                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1518                         if (rt->dst.dev != dev)
1519                                 continue;
1520                         rt->dst.dev = net->loopback_dev;
1521                         dev_hold(rt->dst.dev);
1522                         dev_put(dev);
1523                 }
1524                 spin_unlock_bh(&ul->lock);
1525         }
1526 }
1527
1528 static bool rt_cache_valid(const struct rtable *rt)
1529 {
1530         return  rt &&
1531                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1532                 !rt_is_expired(rt);
1533 }
1534
1535 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1536                            const struct fib_result *res,
1537                            struct fib_nh_exception *fnhe,
1538                            struct fib_info *fi, u16 type, u32 itag,
1539                            const bool do_cache)
1540 {
1541         bool cached = false;
1542
1543         if (fi) {
1544                 struct fib_nh *nh = &FIB_RES_NH(*res);
1545
1546                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1547                         rt->rt_gateway = nh->nh_gw;
1548                         rt->rt_uses_gateway = 1;
1549                 }
1550                 ip_dst_init_metrics(&rt->dst, fi->fib_metrics);
1551
1552 #ifdef CONFIG_IP_ROUTE_CLASSID
1553                 rt->dst.tclassid = nh->nh_tclassid;
1554 #endif
1555                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1556                 if (unlikely(fnhe))
1557                         cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1558                 else if (do_cache)
1559                         cached = rt_cache_route(nh, rt);
1560                 if (unlikely(!cached)) {
1561                         /* Routes we intend to cache in nexthop exception or
1562                          * FIB nexthop have the DST_NOCACHE bit clear.
1563                          * However, if we are unsuccessful at storing this
1564                          * route into the cache we really need to set it.
1565                          */
1566                         if (!rt->rt_gateway)
1567                                 rt->rt_gateway = daddr;
1568                         rt_add_uncached_list(rt);
1569                 }
1570         } else
1571                 rt_add_uncached_list(rt);
1572
1573 #ifdef CONFIG_IP_ROUTE_CLASSID
1574 #ifdef CONFIG_IP_MULTIPLE_TABLES
1575         set_class_tag(rt, res->tclassid);
1576 #endif
1577         set_class_tag(rt, itag);
1578 #endif
1579 }
1580
1581 struct rtable *rt_dst_alloc(struct net_device *dev,
1582                             unsigned int flags, u16 type,
1583                             bool nopolicy, bool noxfrm, bool will_cache)
1584 {
1585         struct rtable *rt;
1586
1587         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1588                        (will_cache ? 0 : DST_HOST) |
1589                        (nopolicy ? DST_NOPOLICY : 0) |
1590                        (noxfrm ? DST_NOXFRM : 0));
1591
1592         if (rt) {
1593                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1594                 rt->rt_flags = flags;
1595                 rt->rt_type = type;
1596                 rt->rt_is_input = 0;
1597                 rt->rt_iif = 0;
1598                 rt->rt_pmtu = 0;
1599                 rt->rt_mtu_locked = 0;
1600                 rt->rt_gateway = 0;
1601                 rt->rt_uses_gateway = 0;
1602                 INIT_LIST_HEAD(&rt->rt_uncached);
1603
1604                 rt->dst.output = ip_output;
1605                 if (flags & RTCF_LOCAL)
1606                         rt->dst.input = ip_local_deliver;
1607         }
1608
1609         return rt;
1610 }
1611 EXPORT_SYMBOL(rt_dst_alloc);
1612
1613 /* called in rcu_read_lock() section */
1614 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1615                           u8 tos, struct net_device *dev,
1616                           struct in_device *in_dev, u32 *itag)
1617 {
1618         int err;
1619
1620         /* Primary sanity checks. */
1621         if (!in_dev)
1622                 return -EINVAL;
1623
1624         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1625             skb->protocol != htons(ETH_P_IP))
1626                 return -EINVAL;
1627
1628         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1629                 return -EINVAL;
1630
1631         if (ipv4_is_zeronet(saddr)) {
1632                 if (!ipv4_is_local_multicast(daddr) &&
1633                     ip_hdr(skb)->protocol != IPPROTO_IGMP)
1634                         return -EINVAL;
1635         } else {
1636                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1637                                           in_dev, itag);
1638                 if (err < 0)
1639                         return err;
1640         }
1641         return 0;
1642 }
1643
1644 /* called in rcu_read_lock() section */
1645 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1646                              u8 tos, struct net_device *dev, int our)
1647 {
1648         struct in_device *in_dev = __in_dev_get_rcu(dev);
1649         unsigned int flags = RTCF_MULTICAST;
1650         struct rtable *rth;
1651         u32 itag = 0;
1652         int err;
1653
1654         err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1655         if (err)
1656                 return err;
1657
1658         if (our)
1659                 flags |= RTCF_LOCAL;
1660
1661         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1662                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1663         if (!rth)
1664                 return -ENOBUFS;
1665
1666 #ifdef CONFIG_IP_ROUTE_CLASSID
1667         rth->dst.tclassid = itag;
1668 #endif
1669         rth->dst.output = ip_rt_bug;
1670         rth->rt_is_input= 1;
1671
1672 #ifdef CONFIG_IP_MROUTE
1673         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1674                 rth->dst.input = ip_mr_input;
1675 #endif
1676         RT_CACHE_STAT_INC(in_slow_mc);
1677
1678         skb_dst_set(skb, &rth->dst);
1679         return 0;
1680 }
1681
1682
1683 static void ip_handle_martian_source(struct net_device *dev,
1684                                      struct in_device *in_dev,
1685                                      struct sk_buff *skb,
1686                                      __be32 daddr,
1687                                      __be32 saddr)
1688 {
1689         RT_CACHE_STAT_INC(in_martian_src);
1690 #ifdef CONFIG_IP_ROUTE_VERBOSE
1691         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1692                 /*
1693                  *      RFC1812 recommendation, if source is martian,
1694                  *      the only hint is MAC header.
1695                  */
1696                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1697                         &daddr, &saddr, dev->name);
1698                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1699                         print_hex_dump(KERN_WARNING, "ll header: ",
1700                                        DUMP_PREFIX_OFFSET, 16, 1,
1701                                        skb_mac_header(skb),
1702                                        dev->hard_header_len, false);
1703                 }
1704         }
1705 #endif
1706 }
1707
1708 /* called in rcu_read_lock() section */
1709 static int __mkroute_input(struct sk_buff *skb,
1710                            const struct fib_result *res,
1711                            struct in_device *in_dev,
1712                            __be32 daddr, __be32 saddr, u32 tos)
1713 {
1714         struct fib_nh_exception *fnhe;
1715         struct rtable *rth;
1716         int err;
1717         struct in_device *out_dev;
1718         bool do_cache;
1719         u32 itag = 0;
1720
1721         /* get a working reference to the output device */
1722         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1723         if (!out_dev) {
1724                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1725                 return -EINVAL;
1726         }
1727
1728         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1729                                   in_dev->dev, in_dev, &itag);
1730         if (err < 0) {
1731                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1732                                          saddr);
1733
1734                 goto cleanup;
1735         }
1736
1737         do_cache = res->fi && !itag;
1738         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1739             skb->protocol == htons(ETH_P_IP) &&
1740             (IN_DEV_SHARED_MEDIA(out_dev) ||
1741              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1742                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1743
1744         if (skb->protocol != htons(ETH_P_IP)) {
1745                 /* Not IP (i.e. ARP). Do not create route, if it is
1746                  * invalid for proxy arp. DNAT routes are always valid.
1747                  *
1748                  * Proxy arp feature have been extended to allow, ARP
1749                  * replies back to the same interface, to support
1750                  * Private VLAN switch technologies. See arp.c.
1751                  */
1752                 if (out_dev == in_dev &&
1753                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1754                         err = -EINVAL;
1755                         goto cleanup;
1756                 }
1757         }
1758
1759         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1760         if (do_cache) {
1761                 if (fnhe)
1762                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1763                 else
1764                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1765                 if (rt_cache_valid(rth)) {
1766                         skb_dst_set_noref(skb, &rth->dst);
1767                         goto out;
1768                 }
1769         }
1770
1771         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1772                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1773                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1774         if (!rth) {
1775                 err = -ENOBUFS;
1776                 goto cleanup;
1777         }
1778
1779         rth->rt_is_input = 1;
1780         RT_CACHE_STAT_INC(in_slow_tot);
1781
1782         rth->dst.input = ip_forward;
1783
1784         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1785                        do_cache);
1786         lwtunnel_set_redirect(&rth->dst);
1787         skb_dst_set(skb, &rth->dst);
1788 out:
1789         err = 0;
1790  cleanup:
1791         return err;
1792 }
1793
1794 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1795 /* To make ICMP packets follow the right flow, the multipath hash is
1796  * calculated from the inner IP addresses.
1797  */
1798 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1799                                  struct flow_keys *hash_keys)
1800 {
1801         const struct iphdr *outer_iph = ip_hdr(skb);
1802         const struct iphdr *key_iph = outer_iph;
1803         const struct iphdr *inner_iph;
1804         const struct icmphdr *icmph;
1805         struct iphdr _inner_iph;
1806         struct icmphdr _icmph;
1807
1808         if (likely(outer_iph->protocol != IPPROTO_ICMP))
1809                 goto out;
1810
1811         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1812                 goto out;
1813
1814         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1815                                    &_icmph);
1816         if (!icmph)
1817                 goto out;
1818
1819         if (icmph->type != ICMP_DEST_UNREACH &&
1820             icmph->type != ICMP_REDIRECT &&
1821             icmph->type != ICMP_TIME_EXCEEDED &&
1822             icmph->type != ICMP_PARAMETERPROB)
1823                 goto out;
1824
1825         inner_iph = skb_header_pointer(skb,
1826                                        outer_iph->ihl * 4 + sizeof(_icmph),
1827                                        sizeof(_inner_iph), &_inner_iph);
1828         if (!inner_iph)
1829                 goto out;
1830
1831         key_iph = inner_iph;
1832 out:
1833         hash_keys->addrs.v4addrs.src = key_iph->saddr;
1834         hash_keys->addrs.v4addrs.dst = key_iph->daddr;
1835 }
1836
1837 /* if skb is set it will be used and fl4 can be NULL */
1838 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
1839                        const struct sk_buff *skb, struct flow_keys *flkeys)
1840 {
1841         u32 multipath_hash = fl4 ? fl4->flowi4_multipath_hash : 0;
1842         struct flow_keys hash_keys;
1843         u32 mhash;
1844
1845         switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1846         case 0:
1847                 memset(&hash_keys, 0, sizeof(hash_keys));
1848                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1849                 if (skb) {
1850                         ip_multipath_l3_keys(skb, &hash_keys);
1851                 } else {
1852                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1853                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1854                 }
1855                 break;
1856         case 1:
1857                 /* skb is currently provided only when forwarding */
1858                 if (skb) {
1859                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1860                         struct flow_keys keys;
1861
1862                         /* short-circuit if we already have L4 hash present */
1863                         if (skb->l4_hash)
1864                                 return skb_get_hash_raw(skb) >> 1;
1865
1866                         memset(&hash_keys, 0, sizeof(hash_keys));
1867
1868                         if (!flkeys) {
1869                                 skb_flow_dissect_flow_keys(skb, &keys, flag);
1870                                 flkeys = &keys;
1871                         }
1872
1873                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1874                         hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
1875                         hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
1876                         hash_keys.ports.src = flkeys->ports.src;
1877                         hash_keys.ports.dst = flkeys->ports.dst;
1878                         hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
1879                 } else {
1880                         memset(&hash_keys, 0, sizeof(hash_keys));
1881                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1882                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1883                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1884                         hash_keys.ports.src = fl4->fl4_sport;
1885                         hash_keys.ports.dst = fl4->fl4_dport;
1886                         hash_keys.basic.ip_proto = fl4->flowi4_proto;
1887                 }
1888                 break;
1889         }
1890         mhash = flow_hash_from_keys(&hash_keys);
1891
1892         if (multipath_hash)
1893                 mhash = jhash_2words(mhash, multipath_hash, 0);
1894
1895         return mhash >> 1;
1896 }
1897 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1898
1899 static int ip_mkroute_input(struct sk_buff *skb,
1900                             struct fib_result *res,
1901                             struct in_device *in_dev,
1902                             __be32 daddr, __be32 saddr, u32 tos,
1903                             struct flow_keys *hkeys)
1904 {
1905 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1906         if (res->fi && res->fi->fib_nhs > 1) {
1907                 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
1908
1909                 fib_select_multipath(res, h);
1910         }
1911 #endif
1912
1913         /* create a routing cache entry */
1914         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1915 }
1916
1917 /*
1918  *      NOTE. We drop all the packets that has local source
1919  *      addresses, because every properly looped back packet
1920  *      must have correct destination already attached by output routine.
1921  *
1922  *      Such approach solves two big problems:
1923  *      1. Not simplex devices are handled properly.
1924  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1925  *      called with rcu_read_lock()
1926  */
1927
1928 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1929                                u8 tos, struct net_device *dev,
1930                                struct fib_result *res)
1931 {
1932         struct in_device *in_dev = __in_dev_get_rcu(dev);
1933         struct flow_keys *flkeys = NULL, _flkeys;
1934         struct net    *net = dev_net(dev);
1935         struct ip_tunnel_info *tun_info;
1936         int             err = -EINVAL;
1937         unsigned int    flags = 0;
1938         u32             itag = 0;
1939         struct rtable   *rth;
1940         struct flowi4   fl4;
1941         bool do_cache;
1942
1943         /* IP on this device is disabled. */
1944
1945         if (!in_dev)
1946                 goto out;
1947
1948         /* Check for the most weird martians, which can be not detected
1949            by fib_lookup.
1950          */
1951
1952         tun_info = skb_tunnel_info(skb);
1953         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1954                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1955         else
1956                 fl4.flowi4_tun_key.tun_id = 0;
1957         skb_dst_drop(skb);
1958
1959         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1960                 goto martian_source;
1961
1962         res->fi = NULL;
1963         res->table = NULL;
1964         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1965                 goto brd_input;
1966
1967         /* Accept zero addresses only to limited broadcast;
1968          * I even do not know to fix it or not. Waiting for complains :-)
1969          */
1970         if (ipv4_is_zeronet(saddr))
1971                 goto martian_source;
1972
1973         if (ipv4_is_zeronet(daddr))
1974                 goto martian_destination;
1975
1976         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1977          * and call it once if daddr or/and saddr are loopback addresses
1978          */
1979         if (ipv4_is_loopback(daddr)) {
1980                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1981                         goto martian_destination;
1982         } else if (ipv4_is_loopback(saddr)) {
1983                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1984                         goto martian_source;
1985         }
1986
1987         /*
1988          *      Now we are ready to route packet.
1989          */
1990         fl4.flowi4_oif = 0;
1991         fl4.flowi4_iif = dev->ifindex;
1992         fl4.flowi4_mark = skb->mark;
1993         fl4.flowi4_tos = tos;
1994         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1995         fl4.flowi4_flags = 0;
1996         fl4.daddr = daddr;
1997         fl4.saddr = saddr;
1998         fl4.flowi4_uid = sock_net_uid(net, NULL);
1999
2000         if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
2001                 flkeys = &_flkeys;
2002         } else {
2003                 fl4.flowi4_proto = 0;
2004                 fl4.fl4_sport = 0;
2005                 fl4.fl4_dport = 0;
2006         }
2007
2008         err = fib_lookup(net, &fl4, res, 0);
2009         if (err != 0) {
2010                 if (!IN_DEV_FORWARD(in_dev))
2011                         err = -EHOSTUNREACH;
2012                 goto no_route;
2013         }
2014
2015         if (res->type == RTN_BROADCAST) {
2016                 if (IN_DEV_BFORWARD(in_dev))
2017                         goto make_route;
2018                 goto brd_input;
2019         }
2020
2021         if (res->type == RTN_LOCAL) {
2022                 err = fib_validate_source(skb, saddr, daddr, tos,
2023                                           0, dev, in_dev, &itag);
2024                 if (err < 0)
2025                         goto martian_source;
2026                 goto local_input;
2027         }
2028
2029         if (!IN_DEV_FORWARD(in_dev)) {
2030                 err = -EHOSTUNREACH;
2031                 goto no_route;
2032         }
2033         if (res->type != RTN_UNICAST)
2034                 goto martian_destination;
2035
2036 make_route:
2037         err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
2038 out:    return err;
2039
2040 brd_input:
2041         if (skb->protocol != htons(ETH_P_IP))
2042                 goto e_inval;
2043
2044         if (!ipv4_is_zeronet(saddr)) {
2045                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2046                                           in_dev, &itag);
2047                 if (err < 0)
2048                         goto martian_source;
2049         }
2050         flags |= RTCF_BROADCAST;
2051         res->type = RTN_BROADCAST;
2052         RT_CACHE_STAT_INC(in_brd);
2053
2054 local_input:
2055         do_cache = false;
2056         if (res->fi) {
2057                 if (!itag) {
2058                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
2059                         if (rt_cache_valid(rth)) {
2060                                 skb_dst_set_noref(skb, &rth->dst);
2061                                 err = 0;
2062                                 goto out;
2063                         }
2064                         do_cache = true;
2065                 }
2066         }
2067
2068         rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2069                            flags | RTCF_LOCAL, res->type,
2070                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2071         if (!rth)
2072                 goto e_nobufs;
2073
2074         rth->dst.output= ip_rt_bug;
2075 #ifdef CONFIG_IP_ROUTE_CLASSID
2076         rth->dst.tclassid = itag;
2077 #endif
2078         rth->rt_is_input = 1;
2079
2080         RT_CACHE_STAT_INC(in_slow_tot);
2081         if (res->type == RTN_UNREACHABLE) {
2082                 rth->dst.input= ip_error;
2083                 rth->dst.error= -err;
2084                 rth->rt_flags   &= ~RTCF_LOCAL;
2085         }
2086
2087         if (do_cache) {
2088                 struct fib_nh *nh = &FIB_RES_NH(*res);
2089
2090                 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2091                 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2092                         WARN_ON(rth->dst.input == lwtunnel_input);
2093                         rth->dst.lwtstate->orig_input = rth->dst.input;
2094                         rth->dst.input = lwtunnel_input;
2095                 }
2096
2097                 if (unlikely(!rt_cache_route(nh, rth)))
2098                         rt_add_uncached_list(rth);
2099         }
2100         skb_dst_set(skb, &rth->dst);
2101         err = 0;
2102         goto out;
2103
2104 no_route:
2105         RT_CACHE_STAT_INC(in_no_route);
2106         res->type = RTN_UNREACHABLE;
2107         res->fi = NULL;
2108         res->table = NULL;
2109         goto local_input;
2110
2111         /*
2112          *      Do not cache martian addresses: they should be logged (RFC1812)
2113          */
2114 martian_destination:
2115         RT_CACHE_STAT_INC(in_martian_dst);
2116 #ifdef CONFIG_IP_ROUTE_VERBOSE
2117         if (IN_DEV_LOG_MARTIANS(in_dev))
2118                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2119                                      &daddr, &saddr, dev->name);
2120 #endif
2121
2122 e_inval:
2123         err = -EINVAL;
2124         goto out;
2125
2126 e_nobufs:
2127         err = -ENOBUFS;
2128         goto out;
2129
2130 martian_source:
2131         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2132         goto out;
2133 }
2134
2135 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2136                          u8 tos, struct net_device *dev)
2137 {
2138         struct fib_result res;
2139         int err;
2140
2141         tos &= IPTOS_RT_MASK;
2142         rcu_read_lock();
2143         err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2144         rcu_read_unlock();
2145
2146         return err;
2147 }
2148 EXPORT_SYMBOL(ip_route_input_noref);
2149
2150 /* called with rcu_read_lock held */
2151 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2152                        u8 tos, struct net_device *dev, struct fib_result *res)
2153 {
2154         /* Multicast recognition logic is moved from route cache to here.
2155            The problem was that too many Ethernet cards have broken/missing
2156            hardware multicast filters :-( As result the host on multicasting
2157            network acquires a lot of useless route cache entries, sort of
2158            SDR messages from all the world. Now we try to get rid of them.
2159            Really, provided software IP multicast filter is organized
2160            reasonably (at least, hashed), it does not result in a slowdown
2161            comparing with route cache reject entries.
2162            Note, that multicast routers are not affected, because
2163            route cache entry is created eventually.
2164          */
2165         if (ipv4_is_multicast(daddr)) {
2166                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2167                 int our = 0;
2168                 int err = -EINVAL;
2169
2170                 if (!in_dev)
2171                         return err;
2172                 our = ip_check_mc_rcu(in_dev, daddr, saddr,
2173                                       ip_hdr(skb)->protocol);
2174
2175                 /* check l3 master if no match yet */
2176                 if (!our && netif_is_l3_slave(dev)) {
2177                         struct in_device *l3_in_dev;
2178
2179                         l3_in_dev = __in_dev_get_rcu(skb->dev);
2180                         if (l3_in_dev)
2181                                 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2182                                                       ip_hdr(skb)->protocol);
2183                 }
2184
2185                 if (our
2186 #ifdef CONFIG_IP_MROUTE
2187                         ||
2188                     (!ipv4_is_local_multicast(daddr) &&
2189                      IN_DEV_MFORWARD(in_dev))
2190 #endif
2191                    ) {
2192                         err = ip_route_input_mc(skb, daddr, saddr,
2193                                                 tos, dev, our);
2194                 }
2195                 return err;
2196         }
2197
2198         return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2199 }
2200
2201 /* called with rcu_read_lock() */
2202 static struct rtable *__mkroute_output(const struct fib_result *res,
2203                                        const struct flowi4 *fl4, int orig_oif,
2204                                        struct net_device *dev_out,
2205                                        unsigned int flags)
2206 {
2207         struct fib_info *fi = res->fi;
2208         struct fib_nh_exception *fnhe;
2209         struct in_device *in_dev;
2210         u16 type = res->type;
2211         struct rtable *rth;
2212         bool do_cache;
2213
2214         in_dev = __in_dev_get_rcu(dev_out);
2215         if (!in_dev)
2216                 return ERR_PTR(-EINVAL);
2217
2218         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2219                 if (ipv4_is_loopback(fl4->saddr) &&
2220                     !(dev_out->flags & IFF_LOOPBACK) &&
2221                     !netif_is_l3_master(dev_out))
2222                         return ERR_PTR(-EINVAL);
2223
2224         if (ipv4_is_lbcast(fl4->daddr))
2225                 type = RTN_BROADCAST;
2226         else if (ipv4_is_multicast(fl4->daddr))
2227                 type = RTN_MULTICAST;
2228         else if (ipv4_is_zeronet(fl4->daddr))
2229                 return ERR_PTR(-EINVAL);
2230
2231         if (dev_out->flags & IFF_LOOPBACK)
2232                 flags |= RTCF_LOCAL;
2233
2234         do_cache = true;
2235         if (type == RTN_BROADCAST) {
2236                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2237                 fi = NULL;
2238         } else if (type == RTN_MULTICAST) {
2239                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2240                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2241                                      fl4->flowi4_proto))
2242                         flags &= ~RTCF_LOCAL;
2243                 else
2244                         do_cache = false;
2245                 /* If multicast route do not exist use
2246                  * default one, but do not gateway in this case.
2247                  * Yes, it is hack.
2248                  */
2249                 if (fi && res->prefixlen < 4)
2250                         fi = NULL;
2251         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2252                    (orig_oif != dev_out->ifindex)) {
2253                 /* For local routes that require a particular output interface
2254                  * we do not want to cache the result.  Caching the result
2255                  * causes incorrect behaviour when there are multiple source
2256                  * addresses on the interface, the end result being that if the
2257                  * intended recipient is waiting on that interface for the
2258                  * packet he won't receive it because it will be delivered on
2259                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2260                  * be set to the loopback interface as well.
2261                  */
2262                 do_cache = false;
2263         }
2264
2265         fnhe = NULL;
2266         do_cache &= fi != NULL;
2267         if (fi) {
2268                 struct rtable __rcu **prth;
2269                 struct fib_nh *nh = &FIB_RES_NH(*res);
2270
2271                 fnhe = find_exception(nh, fl4->daddr);
2272                 if (!do_cache)
2273                         goto add;
2274                 if (fnhe) {
2275                         prth = &fnhe->fnhe_rth_output;
2276                 } else {
2277                         if (unlikely(fl4->flowi4_flags &
2278                                      FLOWI_FLAG_KNOWN_NH &&
2279                                      !(nh->nh_gw &&
2280                                        nh->nh_scope == RT_SCOPE_LINK))) {
2281                                 do_cache = false;
2282                                 goto add;
2283                         }
2284                         prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2285                 }
2286                 rth = rcu_dereference(*prth);
2287                 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2288                         return rth;
2289         }
2290
2291 add:
2292         rth = rt_dst_alloc(dev_out, flags, type,
2293                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2294                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2295                            do_cache);
2296         if (!rth)
2297                 return ERR_PTR(-ENOBUFS);
2298
2299         rth->rt_iif = orig_oif;
2300
2301         RT_CACHE_STAT_INC(out_slow_tot);
2302
2303         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2304                 if (flags & RTCF_LOCAL &&
2305                     !(dev_out->flags & IFF_LOOPBACK)) {
2306                         rth->dst.output = ip_mc_output;
2307                         RT_CACHE_STAT_INC(out_slow_mc);
2308                 }
2309 #ifdef CONFIG_IP_MROUTE
2310                 if (type == RTN_MULTICAST) {
2311                         if (IN_DEV_MFORWARD(in_dev) &&
2312                             !ipv4_is_local_multicast(fl4->daddr)) {
2313                                 rth->dst.input = ip_mr_input;
2314                                 rth->dst.output = ip_mc_output;
2315                         }
2316                 }
2317 #endif
2318         }
2319
2320         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2321         lwtunnel_set_redirect(&rth->dst);
2322
2323         return rth;
2324 }
2325
2326 /*
2327  * Major route resolver routine.
2328  */
2329
2330 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2331                                         const struct sk_buff *skb)
2332 {
2333         __u8 tos = RT_FL_TOS(fl4);
2334         struct fib_result res = {
2335                 .type           = RTN_UNSPEC,
2336                 .fi             = NULL,
2337                 .table          = NULL,
2338                 .tclassid       = 0,
2339         };
2340         struct rtable *rth;
2341
2342         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2343         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2344         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2345                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2346
2347         rcu_read_lock();
2348         rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2349         rcu_read_unlock();
2350
2351         return rth;
2352 }
2353 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2354
2355 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2356                                             struct fib_result *res,
2357                                             const struct sk_buff *skb)
2358 {
2359         struct net_device *dev_out = NULL;
2360         int orig_oif = fl4->flowi4_oif;
2361         unsigned int flags = 0;
2362         struct rtable *rth;
2363         int err = -ENETUNREACH;
2364
2365         if (fl4->saddr) {
2366                 rth = ERR_PTR(-EINVAL);
2367                 if (ipv4_is_multicast(fl4->saddr) ||
2368                     ipv4_is_lbcast(fl4->saddr) ||
2369                     ipv4_is_zeronet(fl4->saddr))
2370                         goto out;
2371
2372                 /* I removed check for oif == dev_out->oif here.
2373                    It was wrong for two reasons:
2374                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2375                       is assigned to multiple interfaces.
2376                    2. Moreover, we are allowed to send packets with saddr
2377                       of another iface. --ANK
2378                  */
2379
2380                 if (fl4->flowi4_oif == 0 &&
2381                     (ipv4_is_multicast(fl4->daddr) ||
2382                      ipv4_is_lbcast(fl4->daddr))) {
2383                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2384                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2385                         if (!dev_out)
2386                                 goto out;
2387
2388                         /* Special hack: user can direct multicasts
2389                            and limited broadcast via necessary interface
2390                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2391                            This hack is not just for fun, it allows
2392                            vic,vat and friends to work.
2393                            They bind socket to loopback, set ttl to zero
2394                            and expect that it will work.
2395                            From the viewpoint of routing cache they are broken,
2396                            because we are not allowed to build multicast path
2397                            with loopback source addr (look, routing cache
2398                            cannot know, that ttl is zero, so that packet
2399                            will not leave this host and route is valid).
2400                            Luckily, this hack is good workaround.
2401                          */
2402
2403                         fl4->flowi4_oif = dev_out->ifindex;
2404                         goto make_route;
2405                 }
2406
2407                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2408                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2409                         if (!__ip_dev_find(net, fl4->saddr, false))
2410                                 goto out;
2411                 }
2412         }
2413
2414
2415         if (fl4->flowi4_oif) {
2416                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2417                 rth = ERR_PTR(-ENODEV);
2418                 if (!dev_out)
2419                         goto out;
2420
2421                 /* RACE: Check return value of inet_select_addr instead. */
2422                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2423                         rth = ERR_PTR(-ENETUNREACH);
2424                         goto out;
2425                 }
2426                 if (ipv4_is_local_multicast(fl4->daddr) ||
2427                     ipv4_is_lbcast(fl4->daddr) ||
2428                     fl4->flowi4_proto == IPPROTO_IGMP) {
2429                         if (!fl4->saddr)
2430                                 fl4->saddr = inet_select_addr(dev_out, 0,
2431                                                               RT_SCOPE_LINK);
2432                         goto make_route;
2433                 }
2434                 if (!fl4->saddr) {
2435                         if (ipv4_is_multicast(fl4->daddr))
2436                                 fl4->saddr = inet_select_addr(dev_out, 0,
2437                                                               fl4->flowi4_scope);
2438                         else if (!fl4->daddr)
2439                                 fl4->saddr = inet_select_addr(dev_out, 0,
2440                                                               RT_SCOPE_HOST);
2441                 }
2442         }
2443
2444         if (!fl4->daddr) {
2445                 fl4->daddr = fl4->saddr;
2446                 if (!fl4->daddr)
2447                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2448                 dev_out = net->loopback_dev;
2449                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2450                 res->type = RTN_LOCAL;
2451                 flags |= RTCF_LOCAL;
2452                 goto make_route;
2453         }
2454
2455         err = fib_lookup(net, fl4, res, 0);
2456         if (err) {
2457                 res->fi = NULL;
2458                 res->table = NULL;
2459                 if (fl4->flowi4_oif &&
2460                     (ipv4_is_multicast(fl4->daddr) ||
2461                     !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2462                         /* Apparently, routing tables are wrong. Assume,
2463                            that the destination is on link.
2464
2465                            WHY? DW.
2466                            Because we are allowed to send to iface
2467                            even if it has NO routes and NO assigned
2468                            addresses. When oif is specified, routing
2469                            tables are looked up with only one purpose:
2470                            to catch if destination is gatewayed, rather than
2471                            direct. Moreover, if MSG_DONTROUTE is set,
2472                            we send packet, ignoring both routing tables
2473                            and ifaddr state. --ANK
2474
2475
2476                            We could make it even if oif is unknown,
2477                            likely IPv6, but we do not.
2478                          */
2479
2480                         if (fl4->saddr == 0)
2481                                 fl4->saddr = inet_select_addr(dev_out, 0,
2482                                                               RT_SCOPE_LINK);
2483                         res->type = RTN_UNICAST;
2484                         goto make_route;
2485                 }
2486                 rth = ERR_PTR(err);
2487                 goto out;
2488         }
2489
2490         if (res->type == RTN_LOCAL) {
2491                 if (!fl4->saddr) {
2492                         if (res->fi->fib_prefsrc)
2493                                 fl4->saddr = res->fi->fib_prefsrc;
2494                         else
2495                                 fl4->saddr = fl4->daddr;
2496                 }
2497
2498                 /* L3 master device is the loopback for that domain */
2499                 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2500                         net->loopback_dev;
2501
2502                 /* make sure orig_oif points to fib result device even
2503                  * though packet rx/tx happens over loopback or l3mdev
2504                  */
2505                 orig_oif = FIB_RES_OIF(*res);
2506
2507                 fl4->flowi4_oif = dev_out->ifindex;
2508                 flags |= RTCF_LOCAL;
2509                 goto make_route;
2510         }
2511
2512         fib_select_path(net, res, fl4, skb);
2513
2514         dev_out = FIB_RES_DEV(*res);
2515         fl4->flowi4_oif = dev_out->ifindex;
2516
2517
2518 make_route:
2519         rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2520
2521 out:
2522         return rth;
2523 }
2524
2525 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2526 {
2527         return NULL;
2528 }
2529
2530 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2531 {
2532         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2533
2534         return mtu ? : dst->dev->mtu;
2535 }
2536
2537 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2538                                           struct sk_buff *skb, u32 mtu)
2539 {
2540 }
2541
2542 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2543                                        struct sk_buff *skb)
2544 {
2545 }
2546
2547 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2548                                           unsigned long old)
2549 {
2550         return NULL;
2551 }
2552
2553 static struct dst_ops ipv4_dst_blackhole_ops = {
2554         .family                 =       AF_INET,
2555         .check                  =       ipv4_blackhole_dst_check,
2556         .mtu                    =       ipv4_blackhole_mtu,
2557         .default_advmss         =       ipv4_default_advmss,
2558         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2559         .redirect               =       ipv4_rt_blackhole_redirect,
2560         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2561         .neigh_lookup           =       ipv4_neigh_lookup,
2562 };
2563
2564 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2565 {
2566         struct rtable *ort = (struct rtable *) dst_orig;
2567         struct rtable *rt;
2568
2569         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2570         if (rt) {
2571                 struct dst_entry *new = &rt->dst;
2572
2573                 new->__use = 1;
2574                 new->input = dst_discard;
2575                 new->output = dst_discard_out;
2576
2577                 new->dev = net->loopback_dev;
2578                 if (new->dev)
2579                         dev_hold(new->dev);
2580
2581                 rt->rt_is_input = ort->rt_is_input;
2582                 rt->rt_iif = ort->rt_iif;
2583                 rt->rt_pmtu = ort->rt_pmtu;
2584                 rt->rt_mtu_locked = ort->rt_mtu_locked;
2585
2586                 rt->rt_genid = rt_genid_ipv4(net);
2587                 rt->rt_flags = ort->rt_flags;
2588                 rt->rt_type = ort->rt_type;
2589                 rt->rt_gateway = ort->rt_gateway;
2590                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2591
2592                 INIT_LIST_HEAD(&rt->rt_uncached);
2593         }
2594
2595         dst_release(dst_orig);
2596
2597         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2598 }
2599
2600 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2601                                     const struct sock *sk)
2602 {
2603         struct rtable *rt = __ip_route_output_key(net, flp4);
2604
2605         if (IS_ERR(rt))
2606                 return rt;
2607
2608         if (flp4->flowi4_proto)
2609                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2610                                                         flowi4_to_flowi(flp4),
2611                                                         sk, 0);
2612
2613         return rt;
2614 }
2615 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2616
2617 /* called with rcu_read_lock held */
2618 static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
2619                         struct rtable *rt, u32 table_id, struct flowi4 *fl4,
2620                         struct sk_buff *skb, u32 portid, u32 seq)
2621 {
2622         struct rtmsg *r;
2623         struct nlmsghdr *nlh;
2624         unsigned long expires = 0;
2625         u32 error;
2626         u32 metrics[RTAX_MAX];
2627
2628         nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2629         if (!nlh)
2630                 return -EMSGSIZE;
2631
2632         r = nlmsg_data(nlh);
2633         r->rtm_family    = AF_INET;
2634         r->rtm_dst_len  = 32;
2635         r->rtm_src_len  = 0;
2636         r->rtm_tos      = fl4->flowi4_tos;
2637         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2638         if (nla_put_u32(skb, RTA_TABLE, table_id))
2639                 goto nla_put_failure;
2640         r->rtm_type     = rt->rt_type;
2641         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2642         r->rtm_protocol = RTPROT_UNSPEC;
2643         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2644         if (rt->rt_flags & RTCF_NOTIFY)
2645                 r->rtm_flags |= RTM_F_NOTIFY;
2646         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2647                 r->rtm_flags |= RTCF_DOREDIRECT;
2648
2649         if (nla_put_in_addr(skb, RTA_DST, dst))
2650                 goto nla_put_failure;
2651         if (src) {
2652                 r->rtm_src_len = 32;
2653                 if (nla_put_in_addr(skb, RTA_SRC, src))
2654                         goto nla_put_failure;
2655         }
2656         if (rt->dst.dev &&
2657             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2658                 goto nla_put_failure;
2659 #ifdef CONFIG_IP_ROUTE_CLASSID
2660         if (rt->dst.tclassid &&
2661             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2662                 goto nla_put_failure;
2663 #endif
2664         if (!rt_is_input_route(rt) &&
2665             fl4->saddr != src) {
2666                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2667                         goto nla_put_failure;
2668         }
2669         if (rt->rt_uses_gateway &&
2670             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2671                 goto nla_put_failure;
2672
2673         expires = rt->dst.expires;
2674         if (expires) {
2675                 unsigned long now = jiffies;
2676
2677                 if (time_before(now, expires))
2678                         expires -= now;
2679                 else
2680                         expires = 0;
2681         }
2682
2683         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2684         if (rt->rt_pmtu && expires)
2685                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2686         if (rt->rt_mtu_locked && expires)
2687                 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
2688         if (rtnetlink_put_metrics(skb, metrics) < 0)
2689                 goto nla_put_failure;
2690
2691         if (fl4->flowi4_mark &&
2692             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2693                 goto nla_put_failure;
2694
2695         if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2696             nla_put_u32(skb, RTA_UID,
2697                         from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2698                 goto nla_put_failure;
2699
2700         error = rt->dst.error;
2701
2702         if (rt_is_input_route(rt)) {
2703 #ifdef CONFIG_IP_MROUTE
2704                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2705                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2706                         int err = ipmr_get_route(net, skb,
2707                                                  fl4->saddr, fl4->daddr,
2708                                                  r, portid);
2709
2710                         if (err <= 0) {
2711                                 if (err == 0)
2712                                         return 0;
2713                                 goto nla_put_failure;
2714                         }
2715                 } else
2716 #endif
2717                         if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
2718                                 goto nla_put_failure;
2719         }
2720
2721         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2722                 goto nla_put_failure;
2723
2724         nlmsg_end(skb, nlh);
2725         return 0;
2726
2727 nla_put_failure:
2728         nlmsg_cancel(skb, nlh);
2729         return -EMSGSIZE;
2730 }
2731
2732 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
2733                                                    u8 ip_proto, __be16 sport,
2734                                                    __be16 dport)
2735 {
2736         struct sk_buff *skb;
2737         struct iphdr *iph;
2738
2739         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2740         if (!skb)
2741                 return NULL;
2742
2743         /* Reserve room for dummy headers, this skb can pass
2744          * through good chunk of routing engine.
2745          */
2746         skb_reset_mac_header(skb);
2747         skb_reset_network_header(skb);
2748         skb->protocol = htons(ETH_P_IP);
2749         iph = skb_put(skb, sizeof(struct iphdr));
2750         iph->protocol = ip_proto;
2751         iph->saddr = src;
2752         iph->daddr = dst;
2753         iph->version = 0x4;
2754         iph->frag_off = 0;
2755         iph->ihl = 0x5;
2756         skb_set_transport_header(skb, skb->len);
2757
2758         switch (iph->protocol) {
2759         case IPPROTO_UDP: {
2760                 struct udphdr *udph;
2761
2762                 udph = skb_put_zero(skb, sizeof(struct udphdr));
2763                 udph->source = sport;
2764                 udph->dest = dport;
2765                 udph->len = sizeof(struct udphdr);
2766                 udph->check = 0;
2767                 break;
2768         }
2769         case IPPROTO_TCP: {
2770                 struct tcphdr *tcph;
2771
2772                 tcph = skb_put_zero(skb, sizeof(struct tcphdr));
2773                 tcph->source    = sport;
2774                 tcph->dest      = dport;
2775                 tcph->doff      = sizeof(struct tcphdr) / 4;
2776                 tcph->rst = 1;
2777                 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
2778                                             src, dst, 0);
2779                 break;
2780         }
2781         case IPPROTO_ICMP: {
2782                 struct icmphdr *icmph;
2783
2784                 icmph = skb_put_zero(skb, sizeof(struct icmphdr));
2785                 icmph->type = ICMP_ECHO;
2786                 icmph->code = 0;
2787         }
2788         }
2789
2790         return skb;
2791 }
2792
2793 static int inet_rtm_valid_getroute_req(struct sk_buff *skb,
2794                                        const struct nlmsghdr *nlh,
2795                                        struct nlattr **tb,
2796                                        struct netlink_ext_ack *extack)
2797 {
2798         struct rtmsg *rtm;
2799         int i, err;
2800
2801         if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
2802                 NL_SET_ERR_MSG(extack,
2803                                "ipv4: Invalid header for route get request");
2804                 return -EINVAL;
2805         }
2806
2807         if (!netlink_strict_get_check(skb))
2808                 return nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX,
2809                                    rtm_ipv4_policy, extack);
2810
2811         rtm = nlmsg_data(nlh);
2812         if ((rtm->rtm_src_len && rtm->rtm_src_len != 32) ||
2813             (rtm->rtm_dst_len && rtm->rtm_dst_len != 32) ||
2814             rtm->rtm_table || rtm->rtm_protocol ||
2815             rtm->rtm_scope || rtm->rtm_type) {
2816                 NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for route get request");
2817                 return -EINVAL;
2818         }
2819
2820         if (rtm->rtm_flags & ~(RTM_F_NOTIFY |
2821                                RTM_F_LOOKUP_TABLE |
2822                                RTM_F_FIB_MATCH)) {
2823                 NL_SET_ERR_MSG(extack, "ipv4: Unsupported rtm_flags for route get request");
2824                 return -EINVAL;
2825         }
2826
2827         err = nlmsg_parse_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
2828                                  rtm_ipv4_policy, extack);
2829         if (err)
2830                 return err;
2831
2832         if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
2833             (tb[RTA_DST] && !rtm->rtm_dst_len)) {
2834                 NL_SET_ERR_MSG(extack, "ipv4: rtm_src_len and rtm_dst_len must be 32 for IPv4");
2835                 return -EINVAL;
2836         }
2837
2838         for (i = 0; i <= RTA_MAX; i++) {
2839                 if (!tb[i])
2840                         continue;
2841
2842                 switch (i) {
2843                 case RTA_IIF:
2844                 case RTA_OIF:
2845                 case RTA_SRC:
2846                 case RTA_DST:
2847                 case RTA_IP_PROTO:
2848                 case RTA_SPORT:
2849                 case RTA_DPORT:
2850                 case RTA_MARK:
2851                 case RTA_UID:
2852                         break;
2853                 default:
2854                         NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in route get request");
2855                         return -EINVAL;
2856                 }
2857         }
2858
2859         return 0;
2860 }
2861
2862 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2863                              struct netlink_ext_ack *extack)
2864 {
2865         struct net *net = sock_net(in_skb->sk);
2866         struct nlattr *tb[RTA_MAX+1];
2867         u32 table_id = RT_TABLE_MAIN;
2868         __be16 sport = 0, dport = 0;
2869         struct fib_result res = {};
2870         u8 ip_proto = IPPROTO_UDP;
2871         struct rtable *rt = NULL;
2872         struct sk_buff *skb;
2873         struct rtmsg *rtm;
2874         struct flowi4 fl4 = {};
2875         __be32 dst = 0;
2876         __be32 src = 0;
2877         kuid_t uid;
2878         u32 iif;
2879         int err;
2880         int mark;
2881
2882         err = inet_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
2883         if (err < 0)
2884                 return err;
2885
2886         rtm = nlmsg_data(nlh);
2887         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2888         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2889         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2890         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2891         if (tb[RTA_UID])
2892                 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2893         else
2894                 uid = (iif ? INVALID_UID : current_uid());
2895
2896         if (tb[RTA_IP_PROTO]) {
2897                 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
2898                                                   &ip_proto, AF_INET, extack);
2899                 if (err)
2900                         return err;
2901         }
2902
2903         if (tb[RTA_SPORT])
2904                 sport = nla_get_be16(tb[RTA_SPORT]);
2905
2906         if (tb[RTA_DPORT])
2907                 dport = nla_get_be16(tb[RTA_DPORT]);
2908
2909         skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
2910         if (!skb)
2911                 return -ENOBUFS;
2912
2913         fl4.daddr = dst;
2914         fl4.saddr = src;
2915         fl4.flowi4_tos = rtm->rtm_tos;
2916         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2917         fl4.flowi4_mark = mark;
2918         fl4.flowi4_uid = uid;
2919         if (sport)
2920                 fl4.fl4_sport = sport;
2921         if (dport)
2922                 fl4.fl4_dport = dport;
2923         fl4.flowi4_proto = ip_proto;
2924
2925         rcu_read_lock();
2926
2927         if (iif) {
2928                 struct net_device *dev;
2929
2930                 dev = dev_get_by_index_rcu(net, iif);
2931                 if (!dev) {
2932                         err = -ENODEV;
2933                         goto errout_rcu;
2934                 }
2935
2936                 fl4.flowi4_iif = iif; /* for rt_fill_info */
2937                 skb->dev        = dev;
2938                 skb->mark       = mark;
2939                 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2940                                          dev, &res);
2941
2942                 rt = skb_rtable(skb);
2943                 if (err == 0 && rt->dst.error)
2944                         err = -rt->dst.error;
2945         } else {
2946                 fl4.flowi4_iif = LOOPBACK_IFINDEX;
2947                 skb->dev = net->loopback_dev;
2948                 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2949                 err = 0;
2950                 if (IS_ERR(rt))
2951                         err = PTR_ERR(rt);
2952                 else
2953                         skb_dst_set(skb, &rt->dst);
2954         }
2955
2956         if (err)
2957                 goto errout_rcu;
2958
2959         if (rtm->rtm_flags & RTM_F_NOTIFY)
2960                 rt->rt_flags |= RTCF_NOTIFY;
2961
2962         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2963                 table_id = res.table ? res.table->tb_id : 0;
2964
2965         /* reset skb for netlink reply msg */
2966         skb_trim(skb, 0);
2967         skb_reset_network_header(skb);
2968         skb_reset_transport_header(skb);
2969         skb_reset_mac_header(skb);
2970
2971         if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2972                 if (!res.fi) {
2973                         err = fib_props[res.type].error;
2974                         if (!err)
2975                                 err = -EHOSTUNREACH;
2976                         goto errout_rcu;
2977                 }
2978                 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2979                                     nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2980                                     rt->rt_type, res.prefix, res.prefixlen,
2981                                     fl4.flowi4_tos, res.fi, 0);
2982         } else {
2983                 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
2984                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2985         }
2986         if (err < 0)
2987                 goto errout_rcu;
2988
2989         rcu_read_unlock();
2990
2991         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2992
2993 errout_free:
2994         return err;
2995 errout_rcu:
2996         rcu_read_unlock();
2997         kfree_skb(skb);
2998         goto errout_free;
2999 }
3000
3001 void ip_rt_multicast_event(struct in_device *in_dev)
3002 {
3003         rt_cache_flush(dev_net(in_dev->dev));
3004 }
3005
3006 #ifdef CONFIG_SYSCTL
3007 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
3008 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
3009 static int ip_rt_gc_elasticity __read_mostly    = 8;
3010 static int ip_min_valid_pmtu __read_mostly      = IPV4_MIN_MTU;
3011
3012 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
3013                                         void __user *buffer,
3014                                         size_t *lenp, loff_t *ppos)
3015 {
3016         struct net *net = (struct net *)__ctl->extra1;
3017
3018         if (write) {
3019                 rt_cache_flush(net);
3020                 fnhe_genid_bump(net);
3021                 return 0;
3022         }
3023
3024         return -EINVAL;
3025 }
3026
3027 static struct ctl_table ipv4_route_table[] = {
3028         {
3029                 .procname       = "gc_thresh",
3030                 .data           = &ipv4_dst_ops.gc_thresh,
3031                 .maxlen         = sizeof(int),
3032                 .mode           = 0644,
3033                 .proc_handler   = proc_dointvec,
3034         },
3035         {
3036                 .procname       = "max_size",
3037                 .data           = &ip_rt_max_size,
3038                 .maxlen         = sizeof(int),
3039                 .mode           = 0644,
3040                 .proc_handler   = proc_dointvec,
3041         },
3042         {
3043                 /*  Deprecated. Use gc_min_interval_ms */
3044
3045                 .procname       = "gc_min_interval",
3046                 .data           = &ip_rt_gc_min_interval,
3047                 .maxlen         = sizeof(int),
3048                 .mode           = 0644,
3049                 .proc_handler   = proc_dointvec_jiffies,
3050         },
3051         {
3052                 .procname       = "gc_min_interval_ms",
3053                 .data           = &ip_rt_gc_min_interval,
3054                 .maxlen         = sizeof(int),
3055                 .mode           = 0644,
3056                 .proc_handler   = proc_dointvec_ms_jiffies,
3057         },
3058         {
3059                 .procname       = "gc_timeout",
3060                 .data           = &ip_rt_gc_timeout,
3061                 .maxlen         = sizeof(int),
3062                 .mode           = 0644,
3063                 .proc_handler   = proc_dointvec_jiffies,
3064         },
3065         {
3066                 .procname       = "gc_interval",
3067                 .data           = &ip_rt_gc_interval,
3068                 .maxlen         = sizeof(int),
3069                 .mode           = 0644,
3070                 .proc_handler   = proc_dointvec_jiffies,
3071         },
3072         {
3073                 .procname       = "redirect_load",
3074                 .data           = &ip_rt_redirect_load,
3075                 .maxlen         = sizeof(int),
3076                 .mode           = 0644,
3077                 .proc_handler   = proc_dointvec,
3078         },
3079         {
3080                 .procname       = "redirect_number",
3081                 .data           = &ip_rt_redirect_number,
3082                 .maxlen         = sizeof(int),
3083                 .mode           = 0644,
3084                 .proc_handler   = proc_dointvec,
3085         },
3086         {
3087                 .procname       = "redirect_silence",
3088                 .data           = &ip_rt_redirect_silence,
3089                 .maxlen         = sizeof(int),
3090                 .mode           = 0644,
3091                 .proc_handler   = proc_dointvec,
3092         },
3093         {
3094                 .procname       = "error_cost",
3095                 .data           = &ip_rt_error_cost,
3096                 .maxlen         = sizeof(int),
3097                 .mode           = 0644,
3098                 .proc_handler   = proc_dointvec,
3099         },
3100         {
3101                 .procname       = "error_burst",
3102                 .data           = &ip_rt_error_burst,
3103                 .maxlen         = sizeof(int),
3104                 .mode           = 0644,
3105                 .proc_handler   = proc_dointvec,
3106         },
3107         {
3108                 .procname       = "gc_elasticity",
3109                 .data           = &ip_rt_gc_elasticity,
3110                 .maxlen         = sizeof(int),
3111                 .mode           = 0644,
3112                 .proc_handler   = proc_dointvec,
3113         },
3114         {
3115                 .procname       = "mtu_expires",
3116                 .data           = &ip_rt_mtu_expires,
3117                 .maxlen         = sizeof(int),
3118                 .mode           = 0644,
3119                 .proc_handler   = proc_dointvec_jiffies,
3120         },
3121         {
3122                 .procname       = "min_pmtu",
3123                 .data           = &ip_rt_min_pmtu,
3124                 .maxlen         = sizeof(int),
3125                 .mode           = 0644,
3126                 .proc_handler   = proc_dointvec_minmax,
3127                 .extra1         = &ip_min_valid_pmtu,
3128         },
3129         {
3130                 .procname       = "min_adv_mss",
3131                 .data           = &ip_rt_min_advmss,
3132                 .maxlen         = sizeof(int),
3133                 .mode           = 0644,
3134                 .proc_handler   = proc_dointvec,
3135         },
3136         { }
3137 };
3138
3139 static struct ctl_table ipv4_route_flush_table[] = {
3140         {
3141                 .procname       = "flush",
3142                 .maxlen         = sizeof(int),
3143                 .mode           = 0200,
3144                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3145         },
3146         { },
3147 };
3148
3149 static __net_init int sysctl_route_net_init(struct net *net)
3150 {
3151         struct ctl_table *tbl;
3152
3153         tbl = ipv4_route_flush_table;
3154         if (!net_eq(net, &init_net)) {
3155                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3156                 if (!tbl)
3157                         goto err_dup;
3158
3159                 /* Don't export sysctls to unprivileged users */
3160                 if (net->user_ns != &init_user_ns)
3161                         tbl[0].procname = NULL;
3162         }
3163         tbl[0].extra1 = net;
3164
3165         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3166         if (!net->ipv4.route_hdr)
3167                 goto err_reg;
3168         return 0;
3169
3170 err_reg:
3171         if (tbl != ipv4_route_flush_table)
3172                 kfree(tbl);
3173 err_dup:
3174         return -ENOMEM;
3175 }
3176
3177 static __net_exit void sysctl_route_net_exit(struct net *net)
3178 {
3179         struct ctl_table *tbl;
3180
3181         tbl = net->ipv4.route_hdr->ctl_table_arg;
3182         unregister_net_sysctl_table(net->ipv4.route_hdr);
3183         BUG_ON(tbl == ipv4_route_flush_table);
3184         kfree(tbl);
3185 }
3186
3187 static __net_initdata struct pernet_operations sysctl_route_ops = {
3188         .init = sysctl_route_net_init,
3189         .exit = sysctl_route_net_exit,
3190 };
3191 #endif
3192
3193 static __net_init int rt_genid_init(struct net *net)
3194 {
3195         atomic_set(&net->ipv4.rt_genid, 0);
3196         atomic_set(&net->fnhe_genid, 0);
3197         atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3198         return 0;
3199 }
3200
3201 static __net_initdata struct pernet_operations rt_genid_ops = {
3202         .init = rt_genid_init,
3203 };
3204
3205 static int __net_init ipv4_inetpeer_init(struct net *net)
3206 {
3207         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3208
3209         if (!bp)
3210                 return -ENOMEM;
3211         inet_peer_base_init(bp);
3212         net->ipv4.peers = bp;
3213         return 0;
3214 }
3215
3216 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3217 {
3218         struct inet_peer_base *bp = net->ipv4.peers;
3219
3220         net->ipv4.peers = NULL;
3221         inetpeer_invalidate_tree(bp);
3222         kfree(bp);
3223 }
3224
3225 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3226         .init   =       ipv4_inetpeer_init,
3227         .exit   =       ipv4_inetpeer_exit,
3228 };
3229
3230 #ifdef CONFIG_IP_ROUTE_CLASSID
3231 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3232 #endif /* CONFIG_IP_ROUTE_CLASSID */
3233
3234 int __init ip_rt_init(void)
3235 {
3236         int cpu;
3237
3238         ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents),
3239                                   GFP_KERNEL);
3240         if (!ip_idents)
3241                 panic("IP: failed to allocate ip_idents\n");
3242
3243         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3244
3245         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3246         if (!ip_tstamps)
3247                 panic("IP: failed to allocate ip_tstamps\n");
3248
3249         for_each_possible_cpu(cpu) {
3250                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3251
3252                 INIT_LIST_HEAD(&ul->head);
3253                 spin_lock_init(&ul->lock);
3254         }
3255 #ifdef CONFIG_IP_ROUTE_CLASSID
3256         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3257         if (!ip_rt_acct)
3258                 panic("IP: failed to allocate ip_rt_acct\n");
3259 #endif
3260
3261         ipv4_dst_ops.kmem_cachep =
3262                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3263                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3264
3265         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3266
3267         if (dst_entries_init(&ipv4_dst_ops) < 0)
3268                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3269
3270         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3271                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3272
3273         ipv4_dst_ops.gc_thresh = ~0;
3274         ip_rt_max_size = INT_MAX;
3275
3276         devinet_init();
3277         ip_fib_init();
3278
3279         if (ip_rt_proc_init())
3280                 pr_err("Unable to create route proc files\n");
3281 #ifdef CONFIG_XFRM
3282         xfrm_init();
3283         xfrm4_init();
3284 #endif
3285         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3286                       RTNL_FLAG_DOIT_UNLOCKED);
3287
3288 #ifdef CONFIG_SYSCTL
3289         register_pernet_subsys(&sysctl_route_ops);
3290 #endif
3291         register_pernet_subsys(&rt_genid_ops);
3292         register_pernet_subsys(&ipv4_inetpeer_ops);
3293         return 0;
3294 }
3295
3296 #ifdef CONFIG_SYSCTL
3297 /*
3298  * We really need to sanitize the damn ipv4 init order, then all
3299  * this nonsense will go away.
3300  */
3301 void __init ip_static_sysctl_init(void)
3302 {
3303         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3304 }
3305 #endif