net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <[email protected]>
  10  *              Alan Cox, <[email protected]>
  11  *              Linus Torvalds, <[email protected]>
  12  *              Alexey Kuznetsov, <[email protected]>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              ([email protected])      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <net/dst.h>
  93 #include <net/net_namespace.h>
  94 #include <net/protocol.h>
  95 #include <net/ip.h>
  96 #include <net/route.h>
  97 #include <net/inetpeer.h>
  98 #include <net/sock.h>
  99 #include <net/ip_fib.h>
 100 #include <net/arp.h>
 101 #include <net/tcp.h>
 102 #include <net/icmp.h>
 103 #include <net/xfrm.h>
 104 #include <net/netevent.h>
 105 #include <net/rtnetlink.h>
 106 #ifdef CONFIG_SYSCTL
 107 #include <linux/sysctl.h>
 108 #include <linux/kmemleak.h>
 109 #endif
 110 #include <net/secure_seq.h>
 111
 112 #define RT_FL_TOS(oldflp4) \
 113         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 114
 115 #define IP_MAX_MTU      0xFFF0
 116
 117 #define RT_GC_TIMEOUT (300*HZ)
 118
 119 static int ip_rt_max_size;
 120 static int ip_rt_redirect_number __read_mostly  = 9;
 121 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 122 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 123 static int ip_rt_error_cost __read_mostly       = HZ;
 124 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 125 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 126 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 127 static int ip_rt_min_advmss __read_mostly       = 256;
 128
 129 /*
 130  *      Interface to generic destination cache.
 131  */
 132
 133 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 134 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 135 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 136 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 137 static void              ipv4_link_failure(struct sk_buff *skb);
 138 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 139                                            struct sk_buff *skb, u32 mtu);
 140 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 141                                         struct sk_buff *skb);
 142 static void             ipv4_dst_destroy(struct dst_entry *dst);
 143
 144 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
 145                             int how)
 146 {
 147 }
 148
 149 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 150 {
 151         WARN_ON(1);
 152         return NULL;
 153 }
 154
 155 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 156                                            struct sk_buff *skb,
 157                                            const void *daddr);
 158
 159 static struct dst_ops ipv4_dst_ops = {
 160         .family =               AF_INET,
 161         .protocol =             cpu_to_be16(ETH_P_IP),
 162         .check =                ipv4_dst_check,
 163         .default_advmss =       ipv4_default_advmss,
 164         .mtu =                  ipv4_mtu,
 165         .cow_metrics =          ipv4_cow_metrics,
 166         .destroy =              ipv4_dst_destroy,
 167         .ifdown =               ipv4_dst_ifdown,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174 };
 175
 176 #define ECN_OR_COST(class)      TC_PRIO_##class
 177
 178 const __u8 ip_tos2prio[16] = {
 179         TC_PRIO_BESTEFFORT,
 180         ECN_OR_COST(BESTEFFORT),
 181         TC_PRIO_BESTEFFORT,
 182         ECN_OR_COST(BESTEFFORT),
 183         TC_PRIO_BULK,
 184         ECN_OR_COST(BULK),
 185         TC_PRIO_BULK,
 186         ECN_OR_COST(BULK),
 187         TC_PRIO_INTERACTIVE,
 188         ECN_OR_COST(INTERACTIVE),
 189         TC_PRIO_INTERACTIVE,
 190         ECN_OR_COST(INTERACTIVE),
 191         TC_PRIO_INTERACTIVE_BULK,
 192         ECN_OR_COST(INTERACTIVE_BULK),
 193         TC_PRIO_INTERACTIVE_BULK,
 194         ECN_OR_COST(INTERACTIVE_BULK)
 195 };
 196 EXPORT_SYMBOL(ip_tos2prio);
 197
 198 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 199 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
 200
 201 #ifdef CONFIG_PROC_FS
 202 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 203 {
 204         if (*pos)
 205                 return NULL;
 206         return SEQ_START_TOKEN;
 207 }
 208
 209 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 210 {
 211         ++*pos;
 212         return NULL;
 213 }
 214
 215 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 216 {
 217 }
 218
 219 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 220 {
 221         if (v == SEQ_START_TOKEN)
 222                 seq_printf(seq, "%-127s\n",
 223                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 224                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 225                            "HHUptod\tSpecDst");
 226         return 0;
 227 }
 228
 229 static const struct seq_operations rt_cache_seq_ops = {
 230         .start  = rt_cache_seq_start,
 231         .next   = rt_cache_seq_next,
 232         .stop   = rt_cache_seq_stop,
 233         .show   = rt_cache_seq_show,
 234 };
 235
 236 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 237 {
 238         return seq_open(file, &rt_cache_seq_ops);
 239 }
 240
 241 static const struct file_operations rt_cache_seq_fops = {
 242         .owner   = THIS_MODULE,
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    st->in_hit,
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    st->out_hit,
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    st->gc_total,
 310                    st->gc_ignored,
 311                    st->gc_goal_miss,
 312                    st->gc_dst_overflow,
 313                    st->in_hlist_search,
 314                    st->out_hlist_search
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .owner   = THIS_MODULE,
 334         .open    = rt_cpu_seq_open,
 335         .read    = seq_read,
 336         .llseek  = seq_lseek,
 337         .release = seq_release,
 338 };
 339
 340 #ifdef CONFIG_IP_ROUTE_CLASSID
 341 static int rt_acct_proc_show(struct seq_file *m, void *v)
 342 {
 343         struct ip_rt_acct *dst, *src;
 344         unsigned int i, j;
 345
 346         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 347         if (!dst)
 348                 return -ENOMEM;
 349
 350         for_each_possible_cpu(i) {
 351                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 352                 for (j = 0; j < 256; j++) {
 353                         dst[j].o_bytes   += src[j].o_bytes;
 354                         dst[j].o_packets += src[j].o_packets;
 355                         dst[j].i_bytes   += src[j].i_bytes;
 356                         dst[j].i_packets += src[j].i_packets;
 357                 }
 358         }
 359
 360         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 361         kfree(dst);
 362         return 0;
 363 }
 364
 365 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 366 {
 367         return single_open(file, rt_acct_proc_show, NULL);
 368 }
 369
 370 static const struct file_operations rt_acct_proc_fops = {
 371         .owner          = THIS_MODULE,
 372         .open           = rt_acct_proc_open,
 373         .read           = seq_read,
 374         .llseek         = seq_lseek,
 375         .release        = single_release,
 376 };
 377 #endif
 378
 379 static int __net_init ip_rt_do_proc_init(struct net *net)
 380 {
 381         struct proc_dir_entry *pde;
 382
 383         pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
 384                           &rt_cache_seq_fops);
 385         if (!pde)
 386                 goto err1;
 387
 388         pde = proc_create("rt_cache", S_IRUGO,
 389                           net->proc_net_stat, &rt_cpu_seq_fops);
 390         if (!pde)
 391                 goto err2;
 392
 393 #ifdef CONFIG_IP_ROUTE_CLASSID
 394         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 395         if (!pde)
 396                 goto err3;
 397 #endif
 398         return 0;
 399
 400 #ifdef CONFIG_IP_ROUTE_CLASSID
 401 err3:
 402         remove_proc_entry("rt_cache", net->proc_net_stat);
 403 #endif
 404 err2:
 405         remove_proc_entry("rt_cache", net->proc_net);
 406 err1:
 407         return -ENOMEM;
 408 }
 409
 410 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 411 {
 412         remove_proc_entry("rt_cache", net->proc_net_stat);
 413         remove_proc_entry("rt_cache", net->proc_net);
 414 #ifdef CONFIG_IP_ROUTE_CLASSID
 415         remove_proc_entry("rt_acct", net->proc_net);
 416 #endif
 417 }
 418
 419 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 420         .init = ip_rt_do_proc_init,
 421         .exit = ip_rt_do_proc_exit,
 422 };
 423
 424 static int __init ip_rt_proc_init(void)
 425 {
 426         return register_pernet_subsys(&ip_rt_proc_ops);
 427 }
 428
 429 #else
 430 static inline int ip_rt_proc_init(void)
 431 {
 432         return 0;
 433 }
 434 #endif /* CONFIG_PROC_FS */
 435
 436 static inline bool rt_is_expired(const struct rtable *rth)
 437 {
 438         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
 439 }
 440
 441 void rt_cache_flush(struct net *net)
 442 {
 443         rt_genid_bump(net);
 444 }
 445
 446 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 447                                            struct sk_buff *skb,
 448                                            const void *daddr)
 449 {
 450         struct net_device *dev = dst->dev;
 451         const __be32 *pkey = daddr;
 452         const struct rtable *rt;
 453         struct neighbour *n;
 454
 455         rt = (const struct rtable *) dst;
 456         if (rt->rt_gateway)
 457                 pkey = (const __be32 *) &rt->rt_gateway;
 458         else if (skb)
 459                 pkey = &ip_hdr(skb)->daddr;
 460
 461         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 462         if (n)
 463                 return n;
 464         return neigh_create(&arp_tbl, pkey, dev);
 465 }
 466
 467 /*
 468  * Peer allocation may fail only in serious out-of-memory conditions.  However
 469  * we still can generate some output.
 470  * Random ID selection looks a bit dangerous because we have no chances to
 471  * select ID being unique in a reasonable period of time.
 472  * But broken packet identifier may be better than no packet at all.
 473  */
 474 static void ip_select_fb_ident(struct iphdr *iph)
 475 {
 476         static DEFINE_SPINLOCK(ip_fb_id_lock);
 477         static u32 ip_fallback_id;
 478         u32 salt;
 479
 480         spin_lock_bh(&ip_fb_id_lock);
 481         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
 482         iph->id = htons(salt & 0xFFFF);
 483         ip_fallback_id = salt;
 484         spin_unlock_bh(&ip_fb_id_lock);
 485 }
 486
 487 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
 488 {
 489         struct net *net = dev_net(dst->dev);
 490         struct inet_peer *peer;
 491
 492         peer = inet_getpeer_v4(net->ipv4.peers, iph->daddr, 1);
 493         if (peer) {
 494                 iph->id = htons(inet_getid(peer, more));
 495                 inet_putpeer(peer);
 496                 return;
 497         }
 498
 499         ip_select_fb_ident(iph);
 500 }
 501 EXPORT_SYMBOL(__ip_select_ident);
 502
 503 static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 504                              const struct iphdr *iph,
 505                              int oif, u8 tos,
 506                              u8 prot, u32 mark, int flow_flags)
 507 {
 508         if (sk) {
 509                 const struct inet_sock *inet = inet_sk(sk);
 510
 511                 oif = sk->sk_bound_dev_if;
 512                 mark = sk->sk_mark;
 513                 tos = RT_CONN_FLAGS(sk);
 514                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 515         }
 516         flowi4_init_output(fl4, oif, mark, tos,
 517                            RT_SCOPE_UNIVERSE, prot,
 518                            flow_flags,
 519                            iph->daddr, iph->saddr, 0, 0);
 520 }
 521
 522 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 523                                const struct sock *sk)
 524 {
 525         const struct iphdr *iph = ip_hdr(skb);
 526         int oif = skb->dev->ifindex;
 527         u8 tos = RT_TOS(iph->tos);
 528         u8 prot = iph->protocol;
 529         u32 mark = skb->mark;
 530
 531         __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
 532 }
 533
 534 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 535 {
 536         const struct inet_sock *inet = inet_sk(sk);
 537         const struct ip_options_rcu *inet_opt;
 538         __be32 daddr = inet->inet_daddr;
 539
 540         rcu_read_lock();
 541         inet_opt = rcu_dereference(inet->inet_opt);
 542         if (inet_opt && inet_opt->opt.srr)
 543                 daddr = inet_opt->opt.faddr;
 544         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 545                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 546                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 547                            inet_sk_flowi_flags(sk),
 548                            daddr, inet->inet_saddr, 0, 0);
 549         rcu_read_unlock();
 550 }
 551
 552 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 553                                  const struct sk_buff *skb)
 554 {
 555         if (skb)
 556                 build_skb_flow_key(fl4, skb, sk);
 557         else
 558                 build_sk_flow_key(fl4, sk);
 559 }
 560
 561 static inline void rt_free(struct rtable *rt)
 562 {
 563         call_rcu(&rt->dst.rcu_head, dst_rcu_free);
 564 }
 565
 566 static DEFINE_SPINLOCK(fnhe_lock);
 567
 568 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 569 {
 570         struct fib_nh_exception *fnhe, *oldest;
 571         struct rtable *orig;
 572
 573         oldest = rcu_dereference(hash->chain);
 574         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 575              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 576                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 577                         oldest = fnhe;
 578         }
 579         orig = rcu_dereference(oldest->fnhe_rth);
 580         if (orig) {
 581                 RCU_INIT_POINTER(oldest->fnhe_rth, NULL);
 582                 rt_free(orig);
 583         }
 584         return oldest;
 585 }
 586
 587 static inline u32 fnhe_hashfun(__be32 daddr)
 588 {
 589         u32 hval;
 590
 591         hval = (__force u32) daddr;
 592         hval ^= (hval >> 11) ^ (hval >> 22);
 593
 594         return hval & (FNHE_HASH_SIZE - 1);
 595 }
 596
 597 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 598                                   u32 pmtu, unsigned long expires)
 599 {
 600         struct fnhe_hash_bucket *hash;
 601         struct fib_nh_exception *fnhe;
 602         int depth;
 603         u32 hval = fnhe_hashfun(daddr);
 604
 605         spin_lock_bh(&fnhe_lock);
 606
 607         hash = nh->nh_exceptions;
 608         if (!hash) {
 609                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 610                 if (!hash)
 611                         goto out_unlock;
 612                 nh->nh_exceptions = hash;
 613         }
 614
 615         hash += hval;
 616
 617         depth = 0;
 618         for (fnhe = rcu_dereference(hash->chain); fnhe;
 619              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 620                 if (fnhe->fnhe_daddr == daddr)
 621                         break;
 622                 depth++;
 623         }
 624
 625         if (fnhe) {
 626                 if (gw)
 627                         fnhe->fnhe_gw = gw;
 628                 if (pmtu) {
 629                         fnhe->fnhe_pmtu = pmtu;
 630                         fnhe->fnhe_expires = expires;
 631                 }
 632         } else {
 633                 if (depth > FNHE_RECLAIM_DEPTH)
 634                         fnhe = fnhe_oldest(hash);
 635                 else {
 636                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 637                         if (!fnhe)
 638                                 goto out_unlock;
 639
 640                         fnhe->fnhe_next = hash->chain;
 641                         rcu_assign_pointer(hash->chain, fnhe);
 642                 }
 643                 fnhe->fnhe_daddr = daddr;
 644                 fnhe->fnhe_gw = gw;
 645                 fnhe->fnhe_pmtu = pmtu;
 646                 fnhe->fnhe_expires = expires;
 647         }
 648
 649         fnhe->fnhe_stamp = jiffies;
 650
 651 out_unlock:
 652         spin_unlock_bh(&fnhe_lock);
 653         return;
 654 }
 655
 656 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 657                              bool kill_route)
 658 {
 659         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 660         __be32 old_gw = ip_hdr(skb)->saddr;
 661         struct net_device *dev = skb->dev;
 662         struct in_device *in_dev;
 663         struct fib_result res;
 664         struct neighbour *n;
 665         struct net *net;
 666
 667         switch (icmp_hdr(skb)->code & 7) {
 668         case ICMP_REDIR_NET:
 669         case ICMP_REDIR_NETTOS:
 670         case ICMP_REDIR_HOST:
 671         case ICMP_REDIR_HOSTTOS:
 672                 break;
 673
 674         default:
 675                 return;
 676         }
 677
 678         if (rt->rt_gateway != old_gw)
 679                 return;
 680
 681         in_dev = __in_dev_get_rcu(dev);
 682         if (!in_dev)
 683                 return;
 684
 685         net = dev_net(dev);
 686         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 687             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 688             ipv4_is_zeronet(new_gw))
 689                 goto reject_redirect;
 690
 691         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 692                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 693                         goto reject_redirect;
 694                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 695                         goto reject_redirect;
 696         } else {
 697                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 698                         goto reject_redirect;
 699         }
 700
 701         n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
 702         if (n) {
 703                 if (!(n->nud_state & NUD_VALID)) {
 704                         neigh_event_send(n, NULL);
 705                 } else {
 706                         if (fib_lookup(net, fl4, &res) == 0) {
 707                                 struct fib_nh *nh = &FIB_RES_NH(res);
 708
 709                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 710                                                       0, 0);
 711                         }
 712                         if (kill_route)
 713                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 714                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 715                 }
 716                 neigh_release(n);
 717         }
 718         return;
 719
 720 reject_redirect:
 721 #ifdef CONFIG_IP_ROUTE_VERBOSE
 722         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 723                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 724                 __be32 daddr = iph->daddr;
 725                 __be32 saddr = iph->saddr;
 726
 727                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 728                                      "  Advised path = %pI4 -> %pI4\n",
 729                                      &old_gw, dev->name, &new_gw,
 730                                      &saddr, &daddr);
 731         }
 732 #endif
 733         ;
 734 }
 735
 736 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 737 {
 738         struct rtable *rt;
 739         struct flowi4 fl4;
 740
 741         rt = (struct rtable *) dst;
 742
 743         ip_rt_build_flow_key(&fl4, sk, skb);
 744         __ip_do_redirect(rt, skb, &fl4, true);
 745 }
 746
 747 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 748 {
 749         struct rtable *rt = (struct rtable *)dst;
 750         struct dst_entry *ret = dst;
 751
 752         if (rt) {
 753                 if (dst->obsolete > 0) {
 754                         ip_rt_put(rt);
 755                         ret = NULL;
 756                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 757                            rt->dst.expires) {
 758                         ip_rt_put(rt);
 759                         ret = NULL;
 760                 }
 761         }
 762         return ret;
 763 }
 764
 765 /*
 766  * Algorithm:
 767  *      1. The first ip_rt_redirect_number redirects are sent
 768  *         with exponential backoff, then we stop sending them at all,
 769  *         assuming that the host ignores our redirects.
 770  *      2. If we did not see packets requiring redirects
 771  *         during ip_rt_redirect_silence, we assume that the host
 772  *         forgot redirected route and start to send redirects again.
 773  *
 774  * This algorithm is much cheaper and more intelligent than dumb load limiting
 775  * in icmp.c.
 776  *
 777  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 778  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 779  */
 780
 781 void ip_rt_send_redirect(struct sk_buff *skb)
 782 {
 783         struct rtable *rt = skb_rtable(skb);
 784         struct in_device *in_dev;
 785         struct inet_peer *peer;
 786         struct net *net;
 787         int log_martians;
 788
 789         rcu_read_lock();
 790         in_dev = __in_dev_get_rcu(rt->dst.dev);
 791         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 792                 rcu_read_unlock();
 793                 return;
 794         }
 795         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 796         rcu_read_unlock();
 797
 798         net = dev_net(rt->dst.dev);
 799         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 800         if (!peer) {
 801                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 802                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 803                 return;
 804         }
 805
 806         /* No redirected packets during ip_rt_redirect_silence;
 807          * reset the algorithm.
 808          */
 809         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 810                 peer->rate_tokens = 0;
 811
 812         /* Too many ignored redirects; do not send anything
 813          * set dst.rate_last to the last seen redirected packet.
 814          */
 815         if (peer->rate_tokens >= ip_rt_redirect_number) {
 816                 peer->rate_last = jiffies;
 817                 goto out_put_peer;
 818         }
 819
 820         /* Check for load limit; set rate_last to the latest sent
 821          * redirect.
 822          */
 823         if (peer->rate_tokens == 0 ||
 824             time_after(jiffies,
 825                        (peer->rate_last +
 826                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 827                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 828
 829                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 830                 peer->rate_last = jiffies;
 831                 ++peer->rate_tokens;
 832 #ifdef CONFIG_IP_ROUTE_VERBOSE
 833                 if (log_martians &&
 834                     peer->rate_tokens == ip_rt_redirect_number)
 835                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 836                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 837                                              &ip_hdr(skb)->daddr, &gw);
 838 #endif
 839         }
 840 out_put_peer:
 841         inet_putpeer(peer);
 842 }
 843
 844 static int ip_error(struct sk_buff *skb)
 845 {
 846         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 847         struct rtable *rt = skb_rtable(skb);
 848         struct inet_peer *peer;
 849         unsigned long now;
 850         struct net *net;
 851         bool send;
 852         int code;
 853
 854         net = dev_net(rt->dst.dev);
 855         if (!IN_DEV_FORWARD(in_dev)) {
 856                 switch (rt->dst.error) {
 857                 case EHOSTUNREACH:
 858                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
 859                         break;
 860
 861                 case ENETUNREACH:
 862                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 863                         break;
 864                 }
 865                 goto out;
 866         }
 867
 868         switch (rt->dst.error) {
 869         case EINVAL:
 870         default:
 871                 goto out;
 872         case EHOSTUNREACH:
 873                 code = ICMP_HOST_UNREACH;
 874                 break;
 875         case ENETUNREACH:
 876                 code = ICMP_NET_UNREACH;
 877                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 878                 break;
 879         case EACCES:
 880                 code = ICMP_PKT_FILTERED;
 881                 break;
 882         }
 883
 884         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 885
 886         send = true;
 887         if (peer) {
 888                 now = jiffies;
 889                 peer->rate_tokens += now - peer->rate_last;
 890                 if (peer->rate_tokens > ip_rt_error_burst)
 891                         peer->rate_tokens = ip_rt_error_burst;
 892                 peer->rate_last = now;
 893                 if (peer->rate_tokens >= ip_rt_error_cost)
 894                         peer->rate_tokens -= ip_rt_error_cost;
 895                 else
 896                         send = false;
 897                 inet_putpeer(peer);
 898         }
 899         if (send)
 900                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 901
 902 out:    kfree_skb(skb);
 903         return 0;
 904 }
 905
 906 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 907 {
 908         struct dst_entry *dst = &rt->dst;
 909         struct fib_result res;
 910
 911         if (dst_metric_locked(dst, RTAX_MTU))
 912                 return;
 913
 914         if (dst->dev->mtu < mtu)
 915                 return;
 916
 917         if (mtu < ip_rt_min_pmtu)
 918                 mtu = ip_rt_min_pmtu;
 919
 920         if (!rt->rt_pmtu) {
 921                 dst->obsolete = DST_OBSOLETE_KILL;
 922         } else {
 923                 rt->rt_pmtu = mtu;
 924                 dst->expires = max(1UL, jiffies + ip_rt_mtu_expires);
 925         }
 926
 927         rcu_read_lock();
 928         if (fib_lookup(dev_net(dst->dev), fl4, &res) == 0) {
 929                 struct fib_nh *nh = &FIB_RES_NH(res);
 930
 931                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
 932                                       jiffies + ip_rt_mtu_expires);
 933         }
 934         rcu_read_unlock();
 935 }
 936
 937 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 938                               struct sk_buff *skb, u32 mtu)
 939 {
 940         struct rtable *rt = (struct rtable *) dst;
 941         struct flowi4 fl4;
 942
 943         ip_rt_build_flow_key(&fl4, sk, skb);
 944         __ip_rt_update_pmtu(rt, &fl4, mtu);
 945 }
 946
 947 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
 948                       int oif, u32 mark, u8 protocol, int flow_flags)
 949 {
 950         const struct iphdr *iph = (const struct iphdr *) skb->data;
 951         struct flowi4 fl4;
 952         struct rtable *rt;
 953
 954         __build_flow_key(&fl4, NULL, iph, oif,
 955                          RT_TOS(iph->tos), protocol, mark, flow_flags);
 956         rt = __ip_route_output_key(net, &fl4);
 957         if (!IS_ERR(rt)) {
 958                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 959                 ip_rt_put(rt);
 960         }
 961 }
 962 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
 963
 964 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 965 {
 966         const struct iphdr *iph = (const struct iphdr *) skb->data;
 967         struct flowi4 fl4;
 968         struct rtable *rt;
 969
 970         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
 971         rt = __ip_route_output_key(sock_net(sk), &fl4);
 972         if (!IS_ERR(rt)) {
 973                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 974                 ip_rt_put(rt);
 975         }
 976 }
 977
 978 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 979 {
 980         const struct iphdr *iph = (const struct iphdr *) skb->data;
 981         struct flowi4 fl4;
 982         struct rtable *rt;
 983         struct dst_entry *dst;
 984         bool new = false;
 985
 986         bh_lock_sock(sk);
 987         rt = (struct rtable *) __sk_dst_get(sk);
 988
 989         if (sock_owned_by_user(sk) || !rt) {
 990                 __ipv4_sk_update_pmtu(skb, sk, mtu);
 991                 goto out;
 992         }
 993
 994         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
 995
 996         if (!__sk_dst_check(sk, 0)) {
 997                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
 998                 if (IS_ERR(rt))
 999                         goto out;
1000
1001                 new = true;
1002         }
1003
1004         __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
1005
1006         dst = dst_check(&rt->dst, 0);
1007         if (!dst) {
1008                 if (new)
1009                         dst_release(&rt->dst);
1010
1011                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1012                 if (IS_ERR(rt))
1013                         goto out;
1014
1015                 new = true;
1016         }
1017
1018         if (new)
1019                 __sk_dst_set(sk, &rt->dst);
1020
1021 out:
1022         bh_unlock_sock(sk);
1023 }
1024 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1025
1026 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1027                    int oif, u32 mark, u8 protocol, int flow_flags)
1028 {
1029         const struct iphdr *iph = (const struct iphdr *) skb->data;
1030         struct flowi4 fl4;
1031         struct rtable *rt;
1032
1033         __build_flow_key(&fl4, NULL, iph, oif,
1034                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1035         rt = __ip_route_output_key(net, &fl4);
1036         if (!IS_ERR(rt)) {
1037                 __ip_do_redirect(rt, skb, &fl4, false);
1038                 ip_rt_put(rt);
1039         }
1040 }
1041 EXPORT_SYMBOL_GPL(ipv4_redirect);
1042
1043 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1044 {
1045         const struct iphdr *iph = (const struct iphdr *) skb->data;
1046         struct flowi4 fl4;
1047         struct rtable *rt;
1048
1049         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1050         rt = __ip_route_output_key(sock_net(sk), &fl4);
1051         if (!IS_ERR(rt)) {
1052                 __ip_do_redirect(rt, skb, &fl4, false);
1053                 ip_rt_put(rt);
1054         }
1055 }
1056 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1057
1058 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1059 {
1060         struct rtable *rt = (struct rtable *) dst;
1061
1062         /* All IPV4 dsts are created with ->obsolete set to the value
1063          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1064          * into this function always.
1065          *
1066          * When a PMTU/redirect information update invalidates a
1067          * route, this is indicated by setting obsolete to
1068          * DST_OBSOLETE_KILL.
1069          */
1070         if (dst->obsolete == DST_OBSOLETE_KILL || rt_is_expired(rt))
1071                 return NULL;
1072         return dst;
1073 }
1074
1075 static void ipv4_link_failure(struct sk_buff *skb)
1076 {
1077         struct rtable *rt;
1078
1079         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1080
1081         rt = skb_rtable(skb);
1082         if (rt)
1083                 dst_set_expires(&rt->dst, 0);
1084 }
1085
1086 static int ip_rt_bug(struct sk_buff *skb)
1087 {
1088         pr_debug("%s: %pI4 -> %pI4, %s\n",
1089                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1090                  skb->dev ? skb->dev->name : "?");
1091         kfree_skb(skb);
1092         WARN_ON(1);
1093         return 0;
1094 }
1095
1096 /*
1097    We do not cache source address of outgoing interface,
1098    because it is used only by IP RR, TS and SRR options,
1099    so that it out of fast path.
1100
1101    BTW remember: "addr" is allowed to be not aligned
1102    in IP options!
1103  */
1104
1105 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1106 {
1107         __be32 src;
1108
1109         if (rt_is_output_route(rt))
1110                 src = ip_hdr(skb)->saddr;
1111         else {
1112                 struct fib_result res;
1113                 struct flowi4 fl4;
1114                 struct iphdr *iph;
1115
1116                 iph = ip_hdr(skb);
1117
1118                 memset(&fl4, 0, sizeof(fl4));
1119                 fl4.daddr = iph->daddr;
1120                 fl4.saddr = iph->saddr;
1121                 fl4.flowi4_tos = RT_TOS(iph->tos);
1122                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1123                 fl4.flowi4_iif = skb->dev->ifindex;
1124                 fl4.flowi4_mark = skb->mark;
1125
1126                 rcu_read_lock();
1127                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1128                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1129                 else
1130                         src = inet_select_addr(rt->dst.dev,
1131                                                rt_nexthop(rt, iph->daddr),
1132                                                RT_SCOPE_UNIVERSE);
1133                 rcu_read_unlock();
1134         }
1135         memcpy(addr, &src, 4);
1136 }
1137
1138 #ifdef CONFIG_IP_ROUTE_CLASSID
1139 static void set_class_tag(struct rtable *rt, u32 tag)
1140 {
1141         if (!(rt->dst.tclassid & 0xFFFF))
1142                 rt->dst.tclassid |= tag & 0xFFFF;
1143         if (!(rt->dst.tclassid & 0xFFFF0000))
1144                 rt->dst.tclassid |= tag & 0xFFFF0000;
1145 }
1146 #endif
1147
1148 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1149 {
1150         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1151
1152         if (advmss == 0) {
1153                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1154                                ip_rt_min_advmss);
1155                 if (advmss > 65535 - 40)
1156                         advmss = 65535 - 40;
1157         }
1158         return advmss;
1159 }
1160
1161 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1162 {
1163         const struct rtable *rt = (const struct rtable *) dst;
1164         unsigned int mtu = rt->rt_pmtu;
1165
1166         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1167                 mtu = dst_metric_raw(dst, RTAX_MTU);
1168
1169         if (mtu)
1170                 return mtu;
1171
1172         mtu = dst->dev->mtu;
1173
1174         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1175                 if (rt->rt_uses_gateway && mtu > 576)
1176                         mtu = 576;
1177         }
1178
1179         if (mtu > IP_MAX_MTU)
1180                 mtu = IP_MAX_MTU;
1181
1182         return mtu;
1183 }
1184
1185 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1186 {
1187         struct fnhe_hash_bucket *hash = nh->nh_exceptions;
1188         struct fib_nh_exception *fnhe;
1189         u32 hval;
1190
1191         if (!hash)
1192                 return NULL;
1193
1194         hval = fnhe_hashfun(daddr);
1195
1196         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1197              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1198                 if (fnhe->fnhe_daddr == daddr)
1199                         return fnhe;
1200         }
1201         return NULL;
1202 }
1203
1204 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1205                               __be32 daddr)
1206 {
1207         bool ret = false;
1208
1209         spin_lock_bh(&fnhe_lock);
1210
1211         if (daddr == fnhe->fnhe_daddr) {
1212                 struct rtable *orig = rcu_dereference(fnhe->fnhe_rth);
1213                 if (orig && rt_is_expired(orig)) {
1214                         fnhe->fnhe_gw = 0;
1215                         fnhe->fnhe_pmtu = 0;
1216                         fnhe->fnhe_expires = 0;
1217                 }
1218                 if (fnhe->fnhe_pmtu) {
1219                         unsigned long expires = fnhe->fnhe_expires;
1220                         unsigned long diff = expires - jiffies;
1221
1222                         if (time_before(jiffies, expires)) {
1223                                 rt->rt_pmtu = fnhe->fnhe_pmtu;
1224                                 dst_set_expires(&rt->dst, diff);
1225                         }
1226                 }
1227                 if (fnhe->fnhe_gw) {
1228                         rt->rt_flags |= RTCF_REDIRECTED;
1229                         rt->rt_gateway = fnhe->fnhe_gw;
1230                         rt->rt_uses_gateway = 1;
1231                 } else if (!rt->rt_gateway)
1232                         rt->rt_gateway = daddr;
1233
1234                 rcu_assign_pointer(fnhe->fnhe_rth, rt);
1235                 if (orig)
1236                         rt_free(orig);
1237
1238                 fnhe->fnhe_stamp = jiffies;
1239                 ret = true;
1240         }
1241         spin_unlock_bh(&fnhe_lock);
1242
1243         return ret;
1244 }
1245
1246 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1247 {
1248         struct rtable *orig, *prev, **p;
1249         bool ret = true;
1250
1251         if (rt_is_input_route(rt)) {
1252                 p = (struct rtable **)&nh->nh_rth_input;
1253         } else {
1254                 p = (struct rtable **)__this_cpu_ptr(nh->nh_pcpu_rth_output);
1255         }
1256         orig = *p;
1257
1258         prev = cmpxchg(p, orig, rt);
1259         if (prev == orig) {
1260                 if (orig)
1261                         rt_free(orig);
1262         } else
1263                 ret = false;
1264
1265         return ret;
1266 }
1267
1268 static DEFINE_SPINLOCK(rt_uncached_lock);
1269 static LIST_HEAD(rt_uncached_list);
1270
1271 static void rt_add_uncached_list(struct rtable *rt)
1272 {
1273         spin_lock_bh(&rt_uncached_lock);
1274         list_add_tail(&rt->rt_uncached, &rt_uncached_list);
1275         spin_unlock_bh(&rt_uncached_lock);
1276 }
1277
1278 static void ipv4_dst_destroy(struct dst_entry *dst)
1279 {
1280         struct rtable *rt = (struct rtable *) dst;
1281
1282         if (!list_empty(&rt->rt_uncached)) {
1283                 spin_lock_bh(&rt_uncached_lock);
1284                 list_del(&rt->rt_uncached);
1285                 spin_unlock_bh(&rt_uncached_lock);
1286         }
1287 }
1288
1289 void rt_flush_dev(struct net_device *dev)
1290 {
1291         if (!list_empty(&rt_uncached_list)) {
1292                 struct net *net = dev_net(dev);
1293                 struct rtable *rt;
1294
1295                 spin_lock_bh(&rt_uncached_lock);
1296                 list_for_each_entry(rt, &rt_uncached_list, rt_uncached) {
1297                         if (rt->dst.dev != dev)
1298                                 continue;
1299                         rt->dst.dev = net->loopback_dev;
1300                         dev_hold(rt->dst.dev);
1301                         dev_put(dev);
1302                 }
1303                 spin_unlock_bh(&rt_uncached_lock);
1304         }
1305 }
1306
1307 static bool rt_cache_valid(const struct rtable *rt)
1308 {
1309         return  rt &&
1310                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1311                 !rt_is_expired(rt);
1312 }
1313
1314 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1315                            const struct fib_result *res,
1316                            struct fib_nh_exception *fnhe,
1317                            struct fib_info *fi, u16 type, u32 itag)
1318 {
1319         bool cached = false;
1320
1321         if (fi) {
1322                 struct fib_nh *nh = &FIB_RES_NH(*res);
1323
1324                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1325                         rt->rt_gateway = nh->nh_gw;
1326                         rt->rt_uses_gateway = 1;
1327                 }
1328                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1329 #ifdef CONFIG_IP_ROUTE_CLASSID
1330                 rt->dst.tclassid = nh->nh_tclassid;
1331 #endif
1332                 if (unlikely(fnhe))
1333                         cached = rt_bind_exception(rt, fnhe, daddr);
1334                 else if (!(rt->dst.flags & DST_NOCACHE))
1335                         cached = rt_cache_route(nh, rt);
1336                 if (unlikely(!cached)) {
1337                         /* Routes we intend to cache in nexthop exception or
1338                          * FIB nexthop have the DST_NOCACHE bit clear.
1339                          * However, if we are unsuccessful at storing this
1340                          * route into the cache we really need to set it.
1341                          */
1342                         rt->dst.flags |= DST_NOCACHE;
1343                         if (!rt->rt_gateway)
1344                                 rt->rt_gateway = daddr;
1345                         rt_add_uncached_list(rt);
1346                 }
1347         } else
1348                 rt_add_uncached_list(rt);
1349
1350 #ifdef CONFIG_IP_ROUTE_CLASSID
1351 #ifdef CONFIG_IP_MULTIPLE_TABLES
1352         set_class_tag(rt, res->tclassid);
1353 #endif
1354         set_class_tag(rt, itag);
1355 #endif
1356 }
1357
1358 static struct rtable *rt_dst_alloc(struct net_device *dev,
1359                                    bool nopolicy, bool noxfrm, bool will_cache)
1360 {
1361         return dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1362                          (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
1363                          (nopolicy ? DST_NOPOLICY : 0) |
1364                          (noxfrm ? DST_NOXFRM : 0));
1365 }
1366
1367 /* called in rcu_read_lock() section */
1368 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1369                                 u8 tos, struct net_device *dev, int our)
1370 {
1371         struct rtable *rth;
1372         struct in_device *in_dev = __in_dev_get_rcu(dev);
1373         u32 itag = 0;
1374         int err;
1375
1376         /* Primary sanity checks. */
1377
1378         if (in_dev == NULL)
1379                 return -EINVAL;
1380
1381         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1382             skb->protocol != htons(ETH_P_IP))
1383                 goto e_inval;
1384
1385         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1386                 if (ipv4_is_loopback(saddr))
1387                         goto e_inval;
1388
1389         if (ipv4_is_zeronet(saddr)) {
1390                 if (!ipv4_is_local_multicast(daddr))
1391                         goto e_inval;
1392         } else {
1393                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1394                                           in_dev, &itag);
1395                 if (err < 0)
1396                         goto e_err;
1397         }
1398         rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
1399                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1400         if (!rth)
1401                 goto e_nobufs;
1402
1403 #ifdef CONFIG_IP_ROUTE_CLASSID
1404         rth->dst.tclassid = itag;
1405 #endif
1406         rth->dst.output = ip_rt_bug;
1407
1408         rth->rt_genid   = rt_genid(dev_net(dev));
1409         rth->rt_flags   = RTCF_MULTICAST;
1410         rth->rt_type    = RTN_MULTICAST;
1411         rth->rt_is_input= 1;
1412         rth->rt_iif     = 0;
1413         rth->rt_pmtu    = 0;
1414         rth->rt_gateway = 0;
1415         rth->rt_uses_gateway = 0;
1416         INIT_LIST_HEAD(&rth->rt_uncached);
1417         if (our) {
1418                 rth->dst.input= ip_local_deliver;
1419                 rth->rt_flags |= RTCF_LOCAL;
1420         }
1421
1422 #ifdef CONFIG_IP_MROUTE
1423         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1424                 rth->dst.input = ip_mr_input;
1425 #endif
1426         RT_CACHE_STAT_INC(in_slow_mc);
1427
1428         skb_dst_set(skb, &rth->dst);
1429         return 0;
1430
1431 e_nobufs:
1432         return -ENOBUFS;
1433 e_inval:
1434         return -EINVAL;
1435 e_err:
1436         return err;
1437 }
1438
1439
1440 static void ip_handle_martian_source(struct net_device *dev,
1441                                      struct in_device *in_dev,
1442                                      struct sk_buff *skb,
1443                                      __be32 daddr,
1444                                      __be32 saddr)
1445 {
1446         RT_CACHE_STAT_INC(in_martian_src);
1447 #ifdef CONFIG_IP_ROUTE_VERBOSE
1448         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1449                 /*
1450                  *      RFC1812 recommendation, if source is martian,
1451                  *      the only hint is MAC header.
1452                  */
1453                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1454                         &daddr, &saddr, dev->name);
1455                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1456                         print_hex_dump(KERN_WARNING, "ll header: ",
1457                                        DUMP_PREFIX_OFFSET, 16, 1,
1458                                        skb_mac_header(skb),
1459                                        dev->hard_header_len, true);
1460                 }
1461         }
1462 #endif
1463 }
1464
1465 /* called in rcu_read_lock() section */
1466 static int __mkroute_input(struct sk_buff *skb,
1467                            const struct fib_result *res,
1468                            struct in_device *in_dev,
1469                            __be32 daddr, __be32 saddr, u32 tos)
1470 {
1471         struct rtable *rth;
1472         int err;
1473         struct in_device *out_dev;
1474         unsigned int flags = 0;
1475         bool do_cache;
1476         u32 itag;
1477
1478         /* get a working reference to the output device */
1479         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1480         if (out_dev == NULL) {
1481                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1482                 return -EINVAL;
1483         }
1484
1485         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1486                                   in_dev->dev, in_dev, &itag);
1487         if (err < 0) {
1488                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1489                                          saddr);
1490
1491                 goto cleanup;
1492         }
1493
1494         do_cache = res->fi && !itag;
1495         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1496             (IN_DEV_SHARED_MEDIA(out_dev) ||
1497              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res)))) {
1498                 flags |= RTCF_DOREDIRECT;
1499                 do_cache = false;
1500         }
1501
1502         if (skb->protocol != htons(ETH_P_IP)) {
1503                 /* Not IP (i.e. ARP). Do not create route, if it is
1504                  * invalid for proxy arp. DNAT routes are always valid.
1505                  *
1506                  * Proxy arp feature have been extended to allow, ARP
1507                  * replies back to the same interface, to support
1508                  * Private VLAN switch technologies. See arp.c.
1509                  */
1510                 if (out_dev == in_dev &&
1511                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1512                         err = -EINVAL;
1513                         goto cleanup;
1514                 }
1515         }
1516
1517         if (do_cache) {
1518                 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1519                 if (rt_cache_valid(rth)) {
1520                         skb_dst_set_noref(skb, &rth->dst);
1521                         goto out;
1522                 }
1523         }
1524
1525         rth = rt_dst_alloc(out_dev->dev,
1526                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1527                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1528         if (!rth) {
1529                 err = -ENOBUFS;
1530                 goto cleanup;
1531         }
1532
1533         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
1534         rth->rt_flags = flags;
1535         rth->rt_type = res->type;
1536         rth->rt_is_input = 1;
1537         rth->rt_iif     = 0;
1538         rth->rt_pmtu    = 0;
1539         rth->rt_gateway = 0;
1540         rth->rt_uses_gateway = 0;
1541         INIT_LIST_HEAD(&rth->rt_uncached);
1542
1543         rth->dst.input = ip_forward;
1544         rth->dst.output = ip_output;
1545
1546         rt_set_nexthop(rth, daddr, res, NULL, res->fi, res->type, itag);
1547         skb_dst_set(skb, &rth->dst);
1548 out:
1549         err = 0;
1550  cleanup:
1551         return err;
1552 }
1553
1554 static int ip_mkroute_input(struct sk_buff *skb,
1555                             struct fib_result *res,
1556                             const struct flowi4 *fl4,
1557                             struct in_device *in_dev,
1558                             __be32 daddr, __be32 saddr, u32 tos)
1559 {
1560 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1561         if (res->fi && res->fi->fib_nhs > 1)
1562                 fib_select_multipath(res);
1563 #endif
1564
1565         /* create a routing cache entry */
1566         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1567 }
1568
1569 /*
1570  *      NOTE. We drop all the packets that has local source
1571  *      addresses, because every properly looped back packet
1572  *      must have correct destination already attached by output routine.
1573  *
1574  *      Such approach solves two big problems:
1575  *      1. Not simplex devices are handled properly.
1576  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1577  *      called with rcu_read_lock()
1578  */
1579
1580 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1581                                u8 tos, struct net_device *dev)
1582 {
1583         struct fib_result res;
1584         struct in_device *in_dev = __in_dev_get_rcu(dev);
1585         struct flowi4   fl4;
1586         unsigned int    flags = 0;
1587         u32             itag = 0;
1588         struct rtable   *rth;
1589         int             err = -EINVAL;
1590         struct net    *net = dev_net(dev);
1591         bool do_cache;
1592
1593         /* IP on this device is disabled. */
1594
1595         if (!in_dev)
1596                 goto out;
1597
1598         /* Check for the most weird martians, which can be not detected
1599            by fib_lookup.
1600          */
1601
1602         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1603                 goto martian_source;
1604
1605         res.fi = NULL;
1606         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1607                 goto brd_input;
1608
1609         /* Accept zero addresses only to limited broadcast;
1610          * I even do not know to fix it or not. Waiting for complains :-)
1611          */
1612         if (ipv4_is_zeronet(saddr))
1613                 goto martian_source;
1614
1615         if (ipv4_is_zeronet(daddr))
1616                 goto martian_destination;
1617
1618         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1619          * and call it once if daddr or/and saddr are loopback addresses
1620          */
1621         if (ipv4_is_loopback(daddr)) {
1622                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1623                         goto martian_destination;
1624         } else if (ipv4_is_loopback(saddr)) {
1625                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1626                         goto martian_source;
1627         }
1628
1629         /*
1630          *      Now we are ready to route packet.
1631          */
1632         fl4.flowi4_oif = 0;
1633         fl4.flowi4_iif = dev->ifindex;
1634         fl4.flowi4_mark = skb->mark;
1635         fl4.flowi4_tos = tos;
1636         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1637         fl4.daddr = daddr;
1638         fl4.saddr = saddr;
1639         err = fib_lookup(net, &fl4, &res);
1640         if (err != 0)
1641                 goto no_route;
1642
1643         RT_CACHE_STAT_INC(in_slow_tot);
1644
1645         if (res.type == RTN_BROADCAST)
1646                 goto brd_input;
1647
1648         if (res.type == RTN_LOCAL) {
1649                 err = fib_validate_source(skb, saddr, daddr, tos,
1650                                           LOOPBACK_IFINDEX,
1651                                           dev, in_dev, &itag);
1652                 if (err < 0)
1653                         goto martian_source_keep_err;
1654                 goto local_input;
1655         }
1656
1657         if (!IN_DEV_FORWARD(in_dev))
1658                 goto no_route;
1659         if (res.type != RTN_UNICAST)
1660                 goto martian_destination;
1661
1662         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1663 out:    return err;
1664
1665 brd_input:
1666         if (skb->protocol != htons(ETH_P_IP))
1667                 goto e_inval;
1668
1669         if (!ipv4_is_zeronet(saddr)) {
1670                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1671                                           in_dev, &itag);
1672                 if (err < 0)
1673                         goto martian_source_keep_err;
1674         }
1675         flags |= RTCF_BROADCAST;
1676         res.type = RTN_BROADCAST;
1677         RT_CACHE_STAT_INC(in_brd);
1678
1679 local_input:
1680         do_cache = false;
1681         if (res.fi) {
1682                 if (!itag) {
1683                         rth = rcu_dereference(FIB_RES_NH(res).nh_rth_input);
1684                         if (rt_cache_valid(rth)) {
1685                                 skb_dst_set_noref(skb, &rth->dst);
1686                                 err = 0;
1687                                 goto out;
1688                         }
1689                         do_cache = true;
1690                 }
1691         }
1692
1693         rth = rt_dst_alloc(net->loopback_dev,
1694                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
1695         if (!rth)
1696                 goto e_nobufs;
1697
1698         rth->dst.input= ip_local_deliver;
1699         rth->dst.output= ip_rt_bug;
1700 #ifdef CONFIG_IP_ROUTE_CLASSID
1701         rth->dst.tclassid = itag;
1702 #endif
1703
1704         rth->rt_genid = rt_genid(net);
1705         rth->rt_flags   = flags|RTCF_LOCAL;
1706         rth->rt_type    = res.type;
1707         rth->rt_is_input = 1;
1708         rth->rt_iif     = 0;
1709         rth->rt_pmtu    = 0;
1710         rth->rt_gateway = 0;
1711         rth->rt_uses_gateway = 0;
1712         INIT_LIST_HEAD(&rth->rt_uncached);
1713         if (res.type == RTN_UNREACHABLE) {
1714                 rth->dst.input= ip_error;
1715                 rth->dst.error= -err;
1716                 rth->rt_flags   &= ~RTCF_LOCAL;
1717         }
1718         if (do_cache)
1719                 rt_cache_route(&FIB_RES_NH(res), rth);
1720         skb_dst_set(skb, &rth->dst);
1721         err = 0;
1722         goto out;
1723
1724 no_route:
1725         RT_CACHE_STAT_INC(in_no_route);
1726         res.type = RTN_UNREACHABLE;
1727         if (err == -ESRCH)
1728                 err = -ENETUNREACH;
1729         goto local_input;
1730
1731         /*
1732          *      Do not cache martian addresses: they should be logged (RFC1812)
1733          */
1734 martian_destination:
1735         RT_CACHE_STAT_INC(in_martian_dst);
1736 #ifdef CONFIG_IP_ROUTE_VERBOSE
1737         if (IN_DEV_LOG_MARTIANS(in_dev))
1738                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
1739                                      &daddr, &saddr, dev->name);
1740 #endif
1741
1742 e_inval:
1743         err = -EINVAL;
1744         goto out;
1745
1746 e_nobufs:
1747         err = -ENOBUFS;
1748         goto out;
1749
1750 martian_source:
1751         err = -EINVAL;
1752 martian_source_keep_err:
1753         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
1754         goto out;
1755 }
1756
1757 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1758                          u8 tos, struct net_device *dev)
1759 {
1760         int res;
1761
1762         rcu_read_lock();
1763
1764         /* Multicast recognition logic is moved from route cache to here.
1765            The problem was that too many Ethernet cards have broken/missing
1766            hardware multicast filters :-( As result the host on multicasting
1767            network acquires a lot of useless route cache entries, sort of
1768            SDR messages from all the world. Now we try to get rid of them.
1769            Really, provided software IP multicast filter is organized
1770            reasonably (at least, hashed), it does not result in a slowdown
1771            comparing with route cache reject entries.
1772            Note, that multicast routers are not affected, because
1773            route cache entry is created eventually.
1774          */
1775         if (ipv4_is_multicast(daddr)) {
1776                 struct in_device *in_dev = __in_dev_get_rcu(dev);
1777
1778                 if (in_dev) {
1779                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
1780                                                   ip_hdr(skb)->protocol);
1781                         if (our
1782 #ifdef CONFIG_IP_MROUTE
1783                                 ||
1784                             (!ipv4_is_local_multicast(daddr) &&
1785                              IN_DEV_MFORWARD(in_dev))
1786 #endif
1787                            ) {
1788                                 int res = ip_route_input_mc(skb, daddr, saddr,
1789                                                             tos, dev, our);
1790                                 rcu_read_unlock();
1791                                 return res;
1792                         }
1793                 }
1794                 rcu_read_unlock();
1795                 return -EINVAL;
1796         }
1797         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
1798         rcu_read_unlock();
1799         return res;
1800 }
1801 EXPORT_SYMBOL(ip_route_input_noref);
1802
1803 /* called with rcu_read_lock() */
1804 static struct rtable *__mkroute_output(const struct fib_result *res,
1805                                        const struct flowi4 *fl4, int orig_oif,
1806                                        struct net_device *dev_out,
1807                                        unsigned int flags)
1808 {
1809         struct fib_info *fi = res->fi;
1810         struct fib_nh_exception *fnhe;
1811         struct in_device *in_dev;
1812         u16 type = res->type;
1813         struct rtable *rth;
1814         bool do_cache;
1815
1816         in_dev = __in_dev_get_rcu(dev_out);
1817         if (!in_dev)
1818                 return ERR_PTR(-EINVAL);
1819
1820         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1821                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
1822                         return ERR_PTR(-EINVAL);
1823
1824         if (ipv4_is_lbcast(fl4->daddr))
1825                 type = RTN_BROADCAST;
1826         else if (ipv4_is_multicast(fl4->daddr))
1827                 type = RTN_MULTICAST;
1828         else if (ipv4_is_zeronet(fl4->daddr))
1829                 return ERR_PTR(-EINVAL);
1830
1831         if (dev_out->flags & IFF_LOOPBACK)
1832                 flags |= RTCF_LOCAL;
1833
1834         do_cache = true;
1835         if (type == RTN_BROADCAST) {
1836                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
1837                 fi = NULL;
1838         } else if (type == RTN_MULTICAST) {
1839                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
1840                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
1841                                      fl4->flowi4_proto))
1842                         flags &= ~RTCF_LOCAL;
1843                 else
1844                         do_cache = false;
1845                 /* If multicast route do not exist use
1846                  * default one, but do not gateway in this case.
1847                  * Yes, it is hack.
1848                  */
1849                 if (fi && res->prefixlen < 4)
1850                         fi = NULL;
1851         }
1852
1853         fnhe = NULL;
1854         do_cache &= fi != NULL;
1855         if (do_cache) {
1856                 struct rtable __rcu **prth;
1857                 struct fib_nh *nh = &FIB_RES_NH(*res);
1858
1859                 fnhe = find_exception(nh, fl4->daddr);
1860                 if (fnhe)
1861                         prth = &fnhe->fnhe_rth;
1862                 else {
1863                         if (unlikely(fl4->flowi4_flags &
1864                                      FLOWI_FLAG_KNOWN_NH &&
1865                                      !(nh->nh_gw &&
1866                                        nh->nh_scope == RT_SCOPE_LINK))) {
1867                                 do_cache = false;
1868                                 goto add;
1869                         }
1870                         prth = __this_cpu_ptr(nh->nh_pcpu_rth_output);
1871                 }
1872                 rth = rcu_dereference(*prth);
1873                 if (rt_cache_valid(rth)) {
1874                         dst_hold(&rth->dst);
1875                         return rth;
1876                 }
1877         }
1878
1879 add:
1880         rth = rt_dst_alloc(dev_out,
1881                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1882                            IN_DEV_CONF_GET(in_dev, NOXFRM),
1883                            do_cache);
1884         if (!rth)
1885                 return ERR_PTR(-ENOBUFS);
1886
1887         rth->dst.output = ip_output;
1888
1889         rth->rt_genid = rt_genid(dev_net(dev_out));
1890         rth->rt_flags   = flags;
1891         rth->rt_type    = type;
1892         rth->rt_is_input = 0;
1893         rth->rt_iif     = orig_oif ? : 0;
1894         rth->rt_pmtu    = 0;
1895         rth->rt_gateway = 0;
1896         rth->rt_uses_gateway = 0;
1897         INIT_LIST_HEAD(&rth->rt_uncached);
1898
1899         RT_CACHE_STAT_INC(out_slow_tot);
1900
1901         if (flags & RTCF_LOCAL)
1902                 rth->dst.input = ip_local_deliver;
1903         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
1904                 if (flags & RTCF_LOCAL &&
1905                     !(dev_out->flags & IFF_LOOPBACK)) {
1906                         rth->dst.output = ip_mc_output;
1907                         RT_CACHE_STAT_INC(out_slow_mc);
1908                 }
1909 #ifdef CONFIG_IP_MROUTE
1910                 if (type == RTN_MULTICAST) {
1911                         if (IN_DEV_MFORWARD(in_dev) &&
1912                             !ipv4_is_local_multicast(fl4->daddr)) {
1913                                 rth->dst.input = ip_mr_input;
1914                                 rth->dst.output = ip_mc_output;
1915                         }
1916                 }
1917 #endif
1918         }
1919
1920         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
1921
1922         return rth;
1923 }
1924
1925 /*
1926  * Major route resolver routine.
1927  */
1928
1929 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *fl4)
1930 {
1931         struct net_device *dev_out = NULL;
1932         __u8 tos = RT_FL_TOS(fl4);
1933         unsigned int flags = 0;
1934         struct fib_result res;
1935         struct rtable *rth;
1936         int orig_oif;
1937
1938         res.tclassid    = 0;
1939         res.fi          = NULL;
1940         res.table       = NULL;
1941
1942         orig_oif = fl4->flowi4_oif;
1943
1944         fl4->flowi4_iif = LOOPBACK_IFINDEX;
1945         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
1946         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
1947                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
1948
1949         rcu_read_lock();
1950         if (fl4->saddr) {
1951                 rth = ERR_PTR(-EINVAL);
1952                 if (ipv4_is_multicast(fl4->saddr) ||
1953                     ipv4_is_lbcast(fl4->saddr) ||
1954                     ipv4_is_zeronet(fl4->saddr))
1955                         goto out;
1956
1957                 /* I removed check for oif == dev_out->oif here.
1958                    It was wrong for two reasons:
1959                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
1960                       is assigned to multiple interfaces.
1961                    2. Moreover, we are allowed to send packets with saddr
1962                       of another iface. --ANK
1963                  */
1964
1965                 if (fl4->flowi4_oif == 0 &&
1966                     (ipv4_is_multicast(fl4->daddr) ||
1967                      ipv4_is_lbcast(fl4->daddr))) {
1968                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
1969                         dev_out = __ip_dev_find(net, fl4->saddr, false);
1970                         if (dev_out == NULL)
1971                                 goto out;
1972
1973                         /* Special hack: user can direct multicasts
1974                            and limited broadcast via necessary interface
1975                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
1976                            This hack is not just for fun, it allows
1977                            vic,vat and friends to work.
1978                            They bind socket to loopback, set ttl to zero
1979                            and expect that it will work.
1980                            From the viewpoint of routing cache they are broken,
1981                            because we are not allowed to build multicast path
1982                            with loopback source addr (look, routing cache
1983                            cannot know, that ttl is zero, so that packet
1984                            will not leave this host and route is valid).
1985                            Luckily, this hack is good workaround.
1986                          */
1987
1988                         fl4->flowi4_oif = dev_out->ifindex;
1989                         goto make_route;
1990                 }
1991
1992                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
1993                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
1994                         if (!__ip_dev_find(net, fl4->saddr, false))
1995                                 goto out;
1996                 }
1997         }
1998
1999
2000         if (fl4->flowi4_oif) {
2001                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2002                 rth = ERR_PTR(-ENODEV);
2003                 if (dev_out == NULL)
2004                         goto out;
2005
2006                 /* RACE: Check return value of inet_select_addr instead. */
2007                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2008                         rth = ERR_PTR(-ENETUNREACH);
2009                         goto out;
2010                 }
2011                 if (ipv4_is_local_multicast(fl4->daddr) ||
2012                     ipv4_is_lbcast(fl4->daddr)) {
2013                         if (!fl4->saddr)
2014                                 fl4->saddr = inet_select_addr(dev_out, 0,
2015                                                               RT_SCOPE_LINK);
2016                         goto make_route;
2017                 }
2018                 if (fl4->saddr) {
2019                         if (ipv4_is_multicast(fl4->daddr))
2020                                 fl4->saddr = inet_select_addr(dev_out, 0,
2021                                                               fl4->flowi4_scope);
2022                         else if (!fl4->daddr)
2023                                 fl4->saddr = inet_select_addr(dev_out, 0,
2024                                                               RT_SCOPE_HOST);
2025                 }
2026         }
2027
2028         if (!fl4->daddr) {
2029                 fl4->daddr = fl4->saddr;
2030                 if (!fl4->daddr)
2031                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2032                 dev_out = net->loopback_dev;
2033                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2034                 res.type = RTN_LOCAL;
2035                 flags |= RTCF_LOCAL;
2036                 goto make_route;
2037         }
2038
2039         if (fib_lookup(net, fl4, &res)) {
2040                 res.fi = NULL;
2041                 res.table = NULL;
2042                 if (fl4->flowi4_oif) {
2043                         /* Apparently, routing tables are wrong. Assume,
2044                            that the destination is on link.
2045
2046                            WHY? DW.
2047                            Because we are allowed to send to iface
2048                            even if it has NO routes and NO assigned
2049                            addresses. When oif is specified, routing
2050                            tables are looked up with only one purpose:
2051                            to catch if destination is gatewayed, rather than
2052                            direct. Moreover, if MSG_DONTROUTE is set,
2053                            we send packet, ignoring both routing tables
2054                            and ifaddr state. --ANK
2055
2056
2057                            We could make it even if oif is unknown,
2058                            likely IPv6, but we do not.
2059                          */
2060
2061                         if (fl4->saddr == 0)
2062                                 fl4->saddr = inet_select_addr(dev_out, 0,
2063                                                               RT_SCOPE_LINK);
2064                         res.type = RTN_UNICAST;
2065                         goto make_route;
2066                 }
2067                 rth = ERR_PTR(-ENETUNREACH);
2068                 goto out;
2069         }
2070
2071         if (res.type == RTN_LOCAL) {
2072                 if (!fl4->saddr) {
2073                         if (res.fi->fib_prefsrc)
2074                                 fl4->saddr = res.fi->fib_prefsrc;
2075                         else
2076                                 fl4->saddr = fl4->daddr;
2077                 }
2078                 dev_out = net->loopback_dev;
2079                 fl4->flowi4_oif = dev_out->ifindex;
2080                 flags |= RTCF_LOCAL;
2081                 goto make_route;
2082         }
2083
2084 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2085         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2086                 fib_select_multipath(&res);
2087         else
2088 #endif
2089         if (!res.prefixlen &&
2090             res.table->tb_num_default > 1 &&
2091             res.type == RTN_UNICAST && !fl4->flowi4_oif)
2092                 fib_select_default(&res);
2093
2094         if (!fl4->saddr)
2095                 fl4->saddr = FIB_RES_PREFSRC(net, res);
2096
2097         dev_out = FIB_RES_DEV(res);
2098         fl4->flowi4_oif = dev_out->ifindex;
2099
2100
2101 make_route:
2102         rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
2103
2104 out:
2105         rcu_read_unlock();
2106         return rth;
2107 }
2108 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2109
2110 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2111 {
2112         return NULL;
2113 }
2114
2115 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2116 {
2117         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2118
2119         return mtu ? : dst->dev->mtu;
2120 }
2121
2122 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2123                                           struct sk_buff *skb, u32 mtu)
2124 {
2125 }
2126
2127 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2128                                        struct sk_buff *skb)
2129 {
2130 }
2131
2132 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2133                                           unsigned long old)
2134 {
2135         return NULL;
2136 }
2137
2138 static struct dst_ops ipv4_dst_blackhole_ops = {
2139         .family                 =       AF_INET,
2140         .protocol               =       cpu_to_be16(ETH_P_IP),
2141         .check                  =       ipv4_blackhole_dst_check,
2142         .mtu                    =       ipv4_blackhole_mtu,
2143         .default_advmss         =       ipv4_default_advmss,
2144         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2145         .redirect               =       ipv4_rt_blackhole_redirect,
2146         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2147         .neigh_lookup           =       ipv4_neigh_lookup,
2148 };
2149
2150 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2151 {
2152         struct rtable *ort = (struct rtable *) dst_orig;
2153         struct rtable *rt;
2154
2155         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2156         if (rt) {
2157                 struct dst_entry *new = &rt->dst;
2158
2159                 new->__use = 1;
2160                 new->input = dst_discard;
2161                 new->output = dst_discard;
2162
2163                 new->dev = ort->dst.dev;
2164                 if (new->dev)
2165                         dev_hold(new->dev);
2166
2167                 rt->rt_is_input = ort->rt_is_input;
2168                 rt->rt_iif = ort->rt_iif;
2169                 rt->rt_pmtu = ort->rt_pmtu;
2170
2171                 rt->rt_genid = rt_genid(net);
2172                 rt->rt_flags = ort->rt_flags;
2173                 rt->rt_type = ort->rt_type;
2174                 rt->rt_gateway = ort->rt_gateway;
2175                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2176
2177                 INIT_LIST_HEAD(&rt->rt_uncached);
2178
2179                 dst_free(new);
2180         }
2181
2182         dst_release(dst_orig);
2183
2184         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2185 }
2186
2187 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2188                                     struct sock *sk)
2189 {
2190         struct rtable *rt = __ip_route_output_key(net, flp4);
2191
2192         if (IS_ERR(rt))
2193                 return rt;
2194
2195         if (flp4->flowi4_proto)
2196                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2197                                                    flowi4_to_flowi(flp4),
2198                                                    sk, 0);
2199
2200         return rt;
2201 }
2202 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2203
2204 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src,
2205                         struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2206                         u32 seq, int event, int nowait, unsigned int flags)
2207 {
2208         struct rtable *rt = skb_rtable(skb);
2209         struct rtmsg *r;
2210         struct nlmsghdr *nlh;
2211         unsigned long expires = 0;
2212         u32 error;
2213         u32 metrics[RTAX_MAX];
2214
2215         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
2216         if (nlh == NULL)
2217                 return -EMSGSIZE;
2218
2219         r = nlmsg_data(nlh);
2220         r->rtm_family    = AF_INET;
2221         r->rtm_dst_len  = 32;
2222         r->rtm_src_len  = 0;
2223         r->rtm_tos      = fl4->flowi4_tos;
2224         r->rtm_table    = RT_TABLE_MAIN;
2225         if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2226                 goto nla_put_failure;
2227         r->rtm_type     = rt->rt_type;
2228         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2229         r->rtm_protocol = RTPROT_UNSPEC;
2230         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2231         if (rt->rt_flags & RTCF_NOTIFY)
2232                 r->rtm_flags |= RTM_F_NOTIFY;
2233
2234         if (nla_put_be32(skb, RTA_DST, dst))
2235                 goto nla_put_failure;
2236         if (src) {
2237                 r->rtm_src_len = 32;
2238                 if (nla_put_be32(skb, RTA_SRC, src))
2239                         goto nla_put_failure;
2240         }
2241         if (rt->dst.dev &&
2242             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2243                 goto nla_put_failure;
2244 #ifdef CONFIG_IP_ROUTE_CLASSID
2245         if (rt->dst.tclassid &&
2246             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2247                 goto nla_put_failure;
2248 #endif
2249         if (!rt_is_input_route(rt) &&
2250             fl4->saddr != src) {
2251                 if (nla_put_be32(skb, RTA_PREFSRC, fl4->saddr))
2252                         goto nla_put_failure;
2253         }
2254         if (rt->rt_uses_gateway &&
2255             nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2256                 goto nla_put_failure;
2257
2258         expires = rt->dst.expires;
2259         if (expires) {
2260                 unsigned long now = jiffies;
2261
2262                 if (time_before(now, expires))
2263                         expires -= now;
2264                 else
2265                         expires = 0;
2266         }
2267
2268         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2269         if (rt->rt_pmtu && expires)
2270                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2271         if (rtnetlink_put_metrics(skb, metrics) < 0)
2272                 goto nla_put_failure;
2273
2274         if (fl4->flowi4_mark &&
2275             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2276                 goto nla_put_failure;
2277
2278         error = rt->dst.error;
2279
2280         if (rt_is_input_route(rt)) {
2281 #ifdef CONFIG_IP_MROUTE
2282                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2283                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2284                         int err = ipmr_get_route(net, skb,
2285                                                  fl4->saddr, fl4->daddr,
2286                                                  r, nowait);
2287                         if (err <= 0) {
2288                                 if (!nowait) {
2289                                         if (err == 0)
2290                                                 return 0;
2291                                         goto nla_put_failure;
2292                                 } else {
2293                                         if (err == -EMSGSIZE)
2294                                                 goto nla_put_failure;
2295                                         error = err;
2296                                 }
2297                         }
2298                 } else
2299 #endif
2300                         if (nla_put_u32(skb, RTA_IIF, rt->rt_iif))
2301                                 goto nla_put_failure;
2302         }
2303
2304         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2305                 goto nla_put_failure;
2306
2307         return nlmsg_end(skb, nlh);
2308
2309 nla_put_failure:
2310         nlmsg_cancel(skb, nlh);
2311         return -EMSGSIZE;
2312 }
2313
2314 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
2315 {
2316         struct net *net = sock_net(in_skb->sk);
2317         struct rtmsg *rtm;
2318         struct nlattr *tb[RTA_MAX+1];
2319         struct rtable *rt = NULL;
2320         struct flowi4 fl4;
2321         __be32 dst = 0;
2322         __be32 src = 0;
2323         u32 iif;
2324         int err;
2325         int mark;
2326         struct sk_buff *skb;
2327
2328         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2329         if (err < 0)
2330                 goto errout;
2331
2332         rtm = nlmsg_data(nlh);
2333
2334         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2335         if (skb == NULL) {
2336                 err = -ENOBUFS;
2337                 goto errout;
2338         }
2339
2340         /* Reserve room for dummy headers, this skb can pass
2341            through good chunk of routing engine.
2342          */
2343         skb_reset_mac_header(skb);
2344         skb_reset_network_header(skb);
2345
2346         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2347         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2348         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2349
2350         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2351         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2352         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2353         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2354
2355         memset(&fl4, 0, sizeof(fl4));
2356         fl4.daddr = dst;
2357         fl4.saddr = src;
2358         fl4.flowi4_tos = rtm->rtm_tos;
2359         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2360         fl4.flowi4_mark = mark;
2361
2362         if (iif) {
2363                 struct net_device *dev;
2364
2365                 dev = __dev_get_by_index(net, iif);
2366                 if (dev == NULL) {
2367                         err = -ENODEV;
2368                         goto errout_free;
2369                 }
2370
2371                 skb->protocol   = htons(ETH_P_IP);
2372                 skb->dev        = dev;
2373                 skb->mark       = mark;
2374                 local_bh_disable();
2375                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2376                 local_bh_enable();
2377
2378                 rt = skb_rtable(skb);
2379                 if (err == 0 && rt->dst.error)
2380                         err = -rt->dst.error;
2381         } else {
2382                 rt = ip_route_output_key(net, &fl4);
2383
2384                 err = 0;
2385                 if (IS_ERR(rt))
2386                         err = PTR_ERR(rt);
2387         }
2388
2389         if (err)
2390                 goto errout_free;
2391
2392         skb_dst_set(skb, &rt->dst);
2393         if (rtm->rtm_flags & RTM_F_NOTIFY)
2394                 rt->rt_flags |= RTCF_NOTIFY;
2395
2396         err = rt_fill_info(net, dst, src, &fl4, skb,
2397                            NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2398                            RTM_NEWROUTE, 0, 0);
2399         if (err <= 0)
2400                 goto errout_free;
2401
2402         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2403 errout:
2404         return err;
2405
2406 errout_free:
2407         kfree_skb(skb);
2408         goto errout;
2409 }
2410
2411 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
2412 {
2413         return skb->len;
2414 }
2415
2416 void ip_rt_multicast_event(struct in_device *in_dev)
2417 {
2418         rt_cache_flush(dev_net(in_dev->dev));
2419 }
2420
2421 #ifdef CONFIG_SYSCTL
2422 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
2423 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2424 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2425 static int ip_rt_gc_elasticity __read_mostly    = 8;
2426
2427 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
2428                                         void __user *buffer,
2429                                         size_t *lenp, loff_t *ppos)
2430 {
2431         if (write) {
2432                 rt_cache_flush((struct net *)__ctl->extra1);
2433                 return 0;
2434         }
2435
2436         return -EINVAL;
2437 }
2438
2439 static ctl_table ipv4_route_table[] = {
2440         {
2441                 .procname       = "gc_thresh",
2442                 .data           = &ipv4_dst_ops.gc_thresh,
2443                 .maxlen         = sizeof(int),
2444                 .mode           = 0644,
2445                 .proc_handler   = proc_dointvec,
2446         },
2447         {
2448                 .procname       = "max_size",
2449                 .data           = &ip_rt_max_size,
2450                 .maxlen         = sizeof(int),
2451                 .mode           = 0644,
2452                 .proc_handler   = proc_dointvec,
2453         },
2454         {
2455                 /*  Deprecated. Use gc_min_interval_ms */
2456
2457                 .procname       = "gc_min_interval",
2458                 .data           = &ip_rt_gc_min_interval,
2459                 .maxlen         = sizeof(int),
2460                 .mode           = 0644,
2461                 .proc_handler   = proc_dointvec_jiffies,
2462         },
2463         {
2464                 .procname       = "gc_min_interval_ms",
2465                 .data           = &ip_rt_gc_min_interval,
2466                 .maxlen         = sizeof(int),
2467                 .mode           = 0644,
2468                 .proc_handler   = proc_dointvec_ms_jiffies,
2469         },
2470         {
2471                 .procname       = "gc_timeout",
2472                 .data           = &ip_rt_gc_timeout,
2473                 .maxlen         = sizeof(int),
2474                 .mode           = 0644,
2475                 .proc_handler   = proc_dointvec_jiffies,
2476         },
2477         {
2478                 .procname       = "gc_interval",
2479                 .data           = &ip_rt_gc_interval,
2480                 .maxlen         = sizeof(int),
2481                 .mode           = 0644,
2482                 .proc_handler   = proc_dointvec_jiffies,
2483         },
2484         {
2485                 .procname       = "redirect_load",
2486                 .data           = &ip_rt_redirect_load,
2487                 .maxlen         = sizeof(int),
2488                 .mode           = 0644,
2489                 .proc_handler   = proc_dointvec,
2490         },
2491         {
2492                 .procname       = "redirect_number",
2493                 .data           = &ip_rt_redirect_number,
2494                 .maxlen         = sizeof(int),
2495                 .mode           = 0644,
2496                 .proc_handler   = proc_dointvec,
2497         },
2498         {
2499                 .procname       = "redirect_silence",
2500                 .data           = &ip_rt_redirect_silence,
2501                 .maxlen         = sizeof(int),
2502                 .mode           = 0644,
2503                 .proc_handler   = proc_dointvec,
2504         },
2505         {
2506                 .procname       = "error_cost",
2507                 .data           = &ip_rt_error_cost,
2508                 .maxlen         = sizeof(int),
2509                 .mode           = 0644,
2510                 .proc_handler   = proc_dointvec,
2511         },
2512         {
2513                 .procname       = "error_burst",
2514                 .data           = &ip_rt_error_burst,
2515                 .maxlen         = sizeof(int),
2516                 .mode           = 0644,
2517                 .proc_handler   = proc_dointvec,
2518         },
2519         {
2520                 .procname       = "gc_elasticity",
2521                 .data           = &ip_rt_gc_elasticity,
2522                 .maxlen         = sizeof(int),
2523                 .mode           = 0644,
2524                 .proc_handler   = proc_dointvec,
2525         },
2526         {
2527                 .procname       = "mtu_expires",
2528                 .data           = &ip_rt_mtu_expires,
2529                 .maxlen         = sizeof(int),
2530                 .mode           = 0644,
2531                 .proc_handler   = proc_dointvec_jiffies,
2532         },
2533         {
2534                 .procname       = "min_pmtu",
2535                 .data           = &ip_rt_min_pmtu,
2536                 .maxlen         = sizeof(int),
2537                 .mode           = 0644,
2538                 .proc_handler   = proc_dointvec,
2539         },
2540         {
2541                 .procname       = "min_adv_mss",
2542                 .data           = &ip_rt_min_advmss,
2543                 .maxlen         = sizeof(int),
2544                 .mode           = 0644,
2545                 .proc_handler   = proc_dointvec,
2546         },
2547         { }
2548 };
2549
2550 static struct ctl_table ipv4_route_flush_table[] = {
2551         {
2552                 .procname       = "flush",
2553                 .maxlen         = sizeof(int),
2554                 .mode           = 0200,
2555                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2556         },
2557         { },
2558 };
2559
2560 static __net_init int sysctl_route_net_init(struct net *net)
2561 {
2562         struct ctl_table *tbl;
2563
2564         tbl = ipv4_route_flush_table;
2565         if (!net_eq(net, &init_net)) {
2566                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2567                 if (tbl == NULL)
2568                         goto err_dup;
2569
2570                 /* Don't export sysctls to unprivileged users */
2571                 if (net->user_ns != &init_user_ns)
2572                         tbl[0].procname = NULL;
2573         }
2574         tbl[0].extra1 = net;
2575
2576         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2577         if (net->ipv4.route_hdr == NULL)
2578                 goto err_reg;
2579         return 0;
2580
2581 err_reg:
2582         if (tbl != ipv4_route_flush_table)
2583                 kfree(tbl);
2584 err_dup:
2585         return -ENOMEM;
2586 }
2587
2588 static __net_exit void sysctl_route_net_exit(struct net *net)
2589 {
2590         struct ctl_table *tbl;
2591
2592         tbl = net->ipv4.route_hdr->ctl_table_arg;
2593         unregister_net_sysctl_table(net->ipv4.route_hdr);
2594         BUG_ON(tbl == ipv4_route_flush_table);
2595         kfree(tbl);
2596 }
2597
2598 static __net_initdata struct pernet_operations sysctl_route_ops = {
2599         .init = sysctl_route_net_init,
2600         .exit = sysctl_route_net_exit,
2601 };
2602 #endif
2603
2604 static __net_init int rt_genid_init(struct net *net)
2605 {
2606         atomic_set(&net->rt_genid, 0);
2607         get_random_bytes(&net->ipv4.dev_addr_genid,
2608                          sizeof(net->ipv4.dev_addr_genid));
2609         return 0;
2610 }
2611
2612 static __net_initdata struct pernet_operations rt_genid_ops = {
2613         .init = rt_genid_init,
2614 };
2615
2616 static int __net_init ipv4_inetpeer_init(struct net *net)
2617 {
2618         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
2619
2620         if (!bp)
2621                 return -ENOMEM;
2622         inet_peer_base_init(bp);
2623         net->ipv4.peers = bp;
2624         return 0;
2625 }
2626
2627 static void __net_exit ipv4_inetpeer_exit(struct net *net)
2628 {
2629         struct inet_peer_base *bp = net->ipv4.peers;
2630
2631         net->ipv4.peers = NULL;
2632         inetpeer_invalidate_tree(bp);
2633         kfree(bp);
2634 }
2635
2636 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
2637         .init   =       ipv4_inetpeer_init,
2638         .exit   =       ipv4_inetpeer_exit,
2639 };
2640
2641 #ifdef CONFIG_IP_ROUTE_CLASSID
2642 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2643 #endif /* CONFIG_IP_ROUTE_CLASSID */
2644
2645 int __init ip_rt_init(void)
2646 {
2647         int rc = 0;
2648
2649 #ifdef CONFIG_IP_ROUTE_CLASSID
2650         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
2651         if (!ip_rt_acct)
2652                 panic("IP: failed to allocate ip_rt_acct\n");
2653 #endif
2654
2655         ipv4_dst_ops.kmem_cachep =
2656                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2657                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2658
2659         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
2660
2661         if (dst_entries_init(&ipv4_dst_ops) < 0)
2662                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
2663
2664         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
2665                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
2666
2667         ipv4_dst_ops.gc_thresh = ~0;
2668         ip_rt_max_size = INT_MAX;
2669
2670         devinet_init();
2671         ip_fib_init();
2672
2673         if (ip_rt_proc_init())
2674                 pr_err("Unable to create route proc files\n");
2675 #ifdef CONFIG_XFRM
2676         xfrm_init();
2677         xfrm4_init();
2678 #endif
2679         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2680
2681 #ifdef CONFIG_SYSCTL
2682         register_pernet_subsys(&sysctl_route_ops);
2683 #endif
2684         register_pernet_subsys(&rt_genid_ops);
2685         register_pernet_subsys(&ipv4_inetpeer_ops);
2686         return rc;
2687 }
2688
2689 #ifdef CONFIG_SYSCTL
2690 /*
2691  * We really need to sanitize the damn ipv4 init order, then all
2692  * this nonsense will go away.
2693  */
2694 void __init ip_static_sysctl_init(void)
2695 {
2696         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
2697 }
2698 #endif