net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <[email protected]>
  10  *              Alan Cox, <[email protected]>
  11  *              Linus Torvalds, <[email protected]>
  12  *              Alexey Kuznetsov, <[email protected]>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              ([email protected])      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <linux/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #include <linux/kmemleak.h>
 112 #endif
 113 #include <net/secure_seq.h>
 114 #include <net/ip_tunnels.h>
 115 #include <net/l3mdev.h>
 116
 117 #include "fib_lookup.h"
 118
 119 #define RT_FL_TOS(oldflp4) \
 120         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 121
 122 #define RT_GC_TIMEOUT (300*HZ)
 123
 124 static int ip_rt_max_size;
 125 static int ip_rt_redirect_number __read_mostly  = 9;
 126 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 127 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 128 static int ip_rt_error_cost __read_mostly       = HZ;
 129 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 130 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 131 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 132 static int ip_rt_min_advmss __read_mostly       = 256;
 133
 134 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 135 /*
 136  *      Interface to generic destination cache.
 137  */
 138
 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 140 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 141 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 143 static void              ipv4_link_failure(struct sk_buff *skb);
 144 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 145                                            struct sk_buff *skb, u32 mtu);
 146 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 147                                         struct sk_buff *skb);
 148 static void             ipv4_dst_destroy(struct dst_entry *dst);
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
 160
 161 static struct dst_ops ipv4_dst_ops = {
 162         .family =               AF_INET,
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174         .confirm_neigh =        ipv4_confirm_neigh,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .owner   = THIS_MODULE,
 244         .open    = rt_cache_seq_open,
 245         .read    = seq_read,
 246         .llseek  = seq_lseek,
 247         .release = seq_release,
 248 };
 249
 250
 251 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 252 {
 253         int cpu;
 254
 255         if (*pos == 0)
 256                 return SEQ_START_TOKEN;
 257
 258         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 259                 if (!cpu_possible(cpu))
 260                         continue;
 261                 *pos = cpu+1;
 262                 return &per_cpu(rt_cache_stat, cpu);
 263         }
 264         return NULL;
 265 }
 266
 267 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 268 {
 269         int cpu;
 270
 271         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 272                 if (!cpu_possible(cpu))
 273                         continue;
 274                 *pos = cpu+1;
 275                 return &per_cpu(rt_cache_stat, cpu);
 276         }
 277         return NULL;
 278
 279 }
 280
 281 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 282 {
 283
 284 }
 285
 286 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 287 {
 288         struct rt_cache_stat *st = v;
 289
 290         if (v == SEQ_START_TOKEN) {
 291                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 292                 return 0;
 293         }
 294
 295         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 296                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 297                    dst_entries_get_slow(&ipv4_dst_ops),
 298                    0, /* st->in_hit */
 299                    st->in_slow_tot,
 300                    st->in_slow_mc,
 301                    st->in_no_route,
 302                    st->in_brd,
 303                    st->in_martian_dst,
 304                    st->in_martian_src,
 305
 306                    0, /* st->out_hit */
 307                    st->out_slow_tot,
 308                    st->out_slow_mc,
 309
 310                    0, /* st->gc_total */
 311                    0, /* st->gc_ignored */
 312                    0, /* st->gc_goal_miss */
 313                    0, /* st->gc_dst_overflow */
 314                    0, /* st->in_hlist_search */
 315                    0  /* st->out_hlist_search */
 316                 );
 317         return 0;
 318 }
 319
 320 static const struct seq_operations rt_cpu_seq_ops = {
 321         .start  = rt_cpu_seq_start,
 322         .next   = rt_cpu_seq_next,
 323         .stop   = rt_cpu_seq_stop,
 324         .show   = rt_cpu_seq_show,
 325 };
 326
 327
 328 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 329 {
 330         return seq_open(file, &rt_cpu_seq_ops);
 331 }
 332
 333 static const struct file_operations rt_cpu_seq_fops = {
 334         .owner   = THIS_MODULE,
 335         .open    = rt_cpu_seq_open,
 336         .read    = seq_read,
 337         .llseek  = seq_lseek,
 338         .release = seq_release,
 339 };
 340
 341 #ifdef CONFIG_IP_ROUTE_CLASSID
 342 static int rt_acct_proc_show(struct seq_file *m, void *v)
 343 {
 344         struct ip_rt_acct *dst, *src;
 345         unsigned int i, j;
 346
 347         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 348         if (!dst)
 349                 return -ENOMEM;
 350
 351         for_each_possible_cpu(i) {
 352                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 353                 for (j = 0; j < 256; j++) {
 354                         dst[j].o_bytes   += src[j].o_bytes;
 355                         dst[j].o_packets += src[j].o_packets;
 356                         dst[j].i_bytes   += src[j].i_bytes;
 357                         dst[j].i_packets += src[j].i_packets;
 358                 }
 359         }
 360
 361         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 362         kfree(dst);
 363         return 0;
 364 }
 365
 366 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 367 {
 368         return single_open(file, rt_acct_proc_show, NULL);
 369 }
 370
 371 static const struct file_operations rt_acct_proc_fops = {
 372         .owner          = THIS_MODULE,
 373         .open           = rt_acct_proc_open,
 374         .read           = seq_read,
 375         .llseek         = seq_lseek,
 376         .release        = single_release,
 377 };
 378 #endif
 379
 380 static int __net_init ip_rt_do_proc_init(struct net *net)
 381 {
 382         struct proc_dir_entry *pde;
 383
 384         pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
 385                           &rt_cache_seq_fops);
 386         if (!pde)
 387                 goto err1;
 388
 389         pde = proc_create("rt_cache", S_IRUGO,
 390                           net->proc_net_stat, &rt_cpu_seq_fops);
 391         if (!pde)
 392                 goto err2;
 393
 394 #ifdef CONFIG_IP_ROUTE_CLASSID
 395         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 396         if (!pde)
 397                 goto err3;
 398 #endif
 399         return 0;
 400
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402 err3:
 403         remove_proc_entry("rt_cache", net->proc_net_stat);
 404 #endif
 405 err2:
 406         remove_proc_entry("rt_cache", net->proc_net);
 407 err1:
 408         return -ENOMEM;
 409 }
 410
 411 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 412 {
 413         remove_proc_entry("rt_cache", net->proc_net_stat);
 414         remove_proc_entry("rt_cache", net->proc_net);
 415 #ifdef CONFIG_IP_ROUTE_CLASSID
 416         remove_proc_entry("rt_acct", net->proc_net);
 417 #endif
 418 }
 419
 420 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 421         .init = ip_rt_do_proc_init,
 422         .exit = ip_rt_do_proc_exit,
 423 };
 424
 425 static int __init ip_rt_proc_init(void)
 426 {
 427         return register_pernet_subsys(&ip_rt_proc_ops);
 428 }
 429
 430 #else
 431 static inline int ip_rt_proc_init(void)
 432 {
 433         return 0;
 434 }
 435 #endif /* CONFIG_PROC_FS */
 436
 437 static inline bool rt_is_expired(const struct rtable *rth)
 438 {
 439         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 440 }
 441
 442 void rt_cache_flush(struct net *net)
 443 {
 444         rt_genid_bump_ipv4(net);
 445 }
 446
 447 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 448                                            struct sk_buff *skb,
 449                                            const void *daddr)
 450 {
 451         struct net_device *dev = dst->dev;
 452         const __be32 *pkey = daddr;
 453         const struct rtable *rt;
 454         struct neighbour *n;
 455
 456         rt = (const struct rtable *) dst;
 457         if (rt->rt_gateway)
 458                 pkey = (const __be32 *) &rt->rt_gateway;
 459         else if (skb)
 460                 pkey = &ip_hdr(skb)->daddr;
 461
 462         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 463         if (n)
 464                 return n;
 465         return neigh_create(&arp_tbl, pkey, dev);
 466 }
 467
 468 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 469 {
 470         struct net_device *dev = dst->dev;
 471         const __be32 *pkey = daddr;
 472         const struct rtable *rt;
 473
 474         rt = (const struct rtable *)dst;
 475         if (rt->rt_gateway)
 476                 pkey = (const __be32 *)&rt->rt_gateway;
 477         else if (!daddr ||
 478                  (rt->rt_flags &
 479                   (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
 480                 return;
 481
 482         __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
 483 }
 484
 485 #define IP_IDENTS_SZ 2048u
 486
 487 static atomic_t *ip_idents __read_mostly;
 488 static u32 *ip_tstamps __read_mostly;
 489
 490 /* In order to protect privacy, we add a perturbation to identifiers
 491  * if one generator is seldom used. This makes hard for an attacker
 492  * to infer how many packets were sent between two points in time.
 493  */
 494 u32 ip_idents_reserve(u32 hash, int segs)
 495 {
 496         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 497         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 498         u32 old = READ_ONCE(*p_tstamp);
 499         u32 now = (u32)jiffies;
 500         u32 new, delta = 0;
 501
 502         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 503                 delta = prandom_u32_max(now - old);
 504
 505         /* Do not use atomic_add_return() as it makes UBSAN unhappy */
 506         do {
 507                 old = (u32)atomic_read(p_id);
 508                 new = old + delta + segs;
 509         } while (atomic_cmpxchg(p_id, old, new) != old);
 510
 511         return new - segs;
 512 }
 513 EXPORT_SYMBOL(ip_idents_reserve);
 514
 515 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 516 {
 517         static u32 ip_idents_hashrnd __read_mostly;
 518         u32 hash, id;
 519
 520         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 521
 522         hash = jhash_3words((__force u32)iph->daddr,
 523                             (__force u32)iph->saddr,
 524                             iph->protocol ^ net_hash_mix(net),
 525                             ip_idents_hashrnd);
 526         id = ip_idents_reserve(hash, segs);
 527         iph->id = htons(id);
 528 }
 529 EXPORT_SYMBOL(__ip_select_ident);
 530
 531 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
 532                              const struct sock *sk,
 533                              const struct iphdr *iph,
 534                              int oif, u8 tos,
 535                              u8 prot, u32 mark, int flow_flags)
 536 {
 537         if (sk) {
 538                 const struct inet_sock *inet = inet_sk(sk);
 539
 540                 oif = sk->sk_bound_dev_if;
 541                 mark = sk->sk_mark;
 542                 tos = RT_CONN_FLAGS(sk);
 543                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 544         }
 545         flowi4_init_output(fl4, oif, mark, tos,
 546                            RT_SCOPE_UNIVERSE, prot,
 547                            flow_flags,
 548                            iph->daddr, iph->saddr, 0, 0,
 549                            sock_net_uid(net, sk));
 550 }
 551
 552 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 553                                const struct sock *sk)
 554 {
 555         const struct net *net = dev_net(skb->dev);
 556         const struct iphdr *iph = ip_hdr(skb);
 557         int oif = skb->dev->ifindex;
 558         u8 tos = RT_TOS(iph->tos);
 559         u8 prot = iph->protocol;
 560         u32 mark = skb->mark;
 561
 562         __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
 563 }
 564
 565 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 566 {
 567         const struct inet_sock *inet = inet_sk(sk);
 568         const struct ip_options_rcu *inet_opt;
 569         __be32 daddr = inet->inet_daddr;
 570
 571         rcu_read_lock();
 572         inet_opt = rcu_dereference(inet->inet_opt);
 573         if (inet_opt && inet_opt->opt.srr)
 574                 daddr = inet_opt->opt.faddr;
 575         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 576                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 577                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 578                            inet_sk_flowi_flags(sk),
 579                            daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
 580         rcu_read_unlock();
 581 }
 582
 583 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 584                                  const struct sk_buff *skb)
 585 {
 586         if (skb)
 587                 build_skb_flow_key(fl4, skb, sk);
 588         else
 589                 build_sk_flow_key(fl4, sk);
 590 }
 591
 592 static DEFINE_SPINLOCK(fnhe_lock);
 593
 594 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 595 {
 596         struct rtable *rt;
 597
 598         rt = rcu_dereference(fnhe->fnhe_rth_input);
 599         if (rt) {
 600                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 601                 dst_dev_put(&rt->dst);
 602                 dst_release(&rt->dst);
 603         }
 604         rt = rcu_dereference(fnhe->fnhe_rth_output);
 605         if (rt) {
 606                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 607                 dst_dev_put(&rt->dst);
 608                 dst_release(&rt->dst);
 609         }
 610 }
 611
 612 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 613 {
 614         struct fib_nh_exception *fnhe, *oldest;
 615
 616         oldest = rcu_dereference(hash->chain);
 617         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 618              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 619                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 620                         oldest = fnhe;
 621         }
 622         fnhe_flush_routes(oldest);
 623         return oldest;
 624 }
 625
 626 static inline u32 fnhe_hashfun(__be32 daddr)
 627 {
 628         static u32 fnhe_hashrnd __read_mostly;
 629         u32 hval;
 630
 631         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 632         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 633         return hash_32(hval, FNHE_HASH_SHIFT);
 634 }
 635
 636 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 637 {
 638         rt->rt_pmtu = fnhe->fnhe_pmtu;
 639         rt->dst.expires = fnhe->fnhe_expires;
 640
 641         if (fnhe->fnhe_gw) {
 642                 rt->rt_flags |= RTCF_REDIRECTED;
 643                 rt->rt_gateway = fnhe->fnhe_gw;
 644                 rt->rt_uses_gateway = 1;
 645         }
 646 }
 647
 648 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 649                                   u32 pmtu, unsigned long expires)
 650 {
 651         struct fnhe_hash_bucket *hash;
 652         struct fib_nh_exception *fnhe;
 653         struct rtable *rt;
 654         u32 genid, hval;
 655         unsigned int i;
 656         int depth;
 657
 658         genid = fnhe_genid(dev_net(nh->nh_dev));
 659         hval = fnhe_hashfun(daddr);
 660
 661         spin_lock_bh(&fnhe_lock);
 662
 663         hash = rcu_dereference(nh->nh_exceptions);
 664         if (!hash) {
 665                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 666                 if (!hash)
 667                         goto out_unlock;
 668                 rcu_assign_pointer(nh->nh_exceptions, hash);
 669         }
 670
 671         hash += hval;
 672
 673         depth = 0;
 674         for (fnhe = rcu_dereference(hash->chain); fnhe;
 675              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 676                 if (fnhe->fnhe_daddr == daddr)
 677                         break;
 678                 depth++;
 679         }
 680
 681         if (fnhe) {
 682                 if (fnhe->fnhe_genid != genid)
 683                         fnhe->fnhe_genid = genid;
 684                 if (gw)
 685                         fnhe->fnhe_gw = gw;
 686                 if (pmtu)
 687                         fnhe->fnhe_pmtu = pmtu;
 688                 fnhe->fnhe_expires = max(1UL, expires);
 689                 /* Update all cached dsts too */
 690                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 691                 if (rt)
 692                         fill_route_from_fnhe(rt, fnhe);
 693                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 694                 if (rt)
 695                         fill_route_from_fnhe(rt, fnhe);
 696         } else {
 697                 if (depth > FNHE_RECLAIM_DEPTH)
 698                         fnhe = fnhe_oldest(hash);
 699                 else {
 700                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 701                         if (!fnhe)
 702                                 goto out_unlock;
 703
 704                         fnhe->fnhe_next = hash->chain;
 705                         rcu_assign_pointer(hash->chain, fnhe);
 706                 }
 707                 fnhe->fnhe_genid = genid;
 708                 fnhe->fnhe_daddr = daddr;
 709                 fnhe->fnhe_gw = gw;
 710                 fnhe->fnhe_pmtu = pmtu;
 711                 fnhe->fnhe_expires = expires;
 712
 713                 /* Exception created; mark the cached routes for the nexthop
 714                  * stale, so anyone caching it rechecks if this exception
 715                  * applies to them.
 716                  */
 717                 rt = rcu_dereference(nh->nh_rth_input);
 718                 if (rt)
 719                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 720
 721                 for_each_possible_cpu(i) {
 722                         struct rtable __rcu **prt;
 723                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 724                         rt = rcu_dereference(*prt);
 725                         if (rt)
 726                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 727                 }
 728         }
 729
 730         fnhe->fnhe_stamp = jiffies;
 731
 732 out_unlock:
 733         spin_unlock_bh(&fnhe_lock);
 734 }
 735
 736 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 737                              bool kill_route)
 738 {
 739         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 740         __be32 old_gw = ip_hdr(skb)->saddr;
 741         struct net_device *dev = skb->dev;
 742         struct in_device *in_dev;
 743         struct fib_result res;
 744         struct neighbour *n;
 745         struct net *net;
 746
 747         switch (icmp_hdr(skb)->code & 7) {
 748         case ICMP_REDIR_NET:
 749         case ICMP_REDIR_NETTOS:
 750         case ICMP_REDIR_HOST:
 751         case ICMP_REDIR_HOSTTOS:
 752                 break;
 753
 754         default:
 755                 return;
 756         }
 757
 758         if (rt->rt_gateway != old_gw)
 759                 return;
 760
 761         in_dev = __in_dev_get_rcu(dev);
 762         if (!in_dev)
 763                 return;
 764
 765         net = dev_net(dev);
 766         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 767             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 768             ipv4_is_zeronet(new_gw))
 769                 goto reject_redirect;
 770
 771         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 772                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 773                         goto reject_redirect;
 774                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 775                         goto reject_redirect;
 776         } else {
 777                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 778                         goto reject_redirect;
 779         }
 780
 781         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 782         if (!n)
 783                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 784         if (!IS_ERR(n)) {
 785                 if (!(n->nud_state & NUD_VALID)) {
 786                         neigh_event_send(n, NULL);
 787                 } else {
 788                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 789                                 struct fib_nh *nh = &FIB_RES_NH(res);
 790
 791                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 792                                                 0, jiffies + ip_rt_gc_timeout);
 793                         }
 794                         if (kill_route)
 795                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 796                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 797                 }
 798                 neigh_release(n);
 799         }
 800         return;
 801
 802 reject_redirect:
 803 #ifdef CONFIG_IP_ROUTE_VERBOSE
 804         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 805                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 806                 __be32 daddr = iph->daddr;
 807                 __be32 saddr = iph->saddr;
 808
 809                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 810                                      "  Advised path = %pI4 -> %pI4\n",
 811                                      &old_gw, dev->name, &new_gw,
 812                                      &saddr, &daddr);
 813         }
 814 #endif
 815         ;
 816 }
 817
 818 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 819 {
 820         struct rtable *rt;
 821         struct flowi4 fl4;
 822         const struct iphdr *iph = (const struct iphdr *) skb->data;
 823         struct net *net = dev_net(skb->dev);
 824         int oif = skb->dev->ifindex;
 825         u8 tos = RT_TOS(iph->tos);
 826         u8 prot = iph->protocol;
 827         u32 mark = skb->mark;
 828
 829         rt = (struct rtable *) dst;
 830
 831         __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
 832         __ip_do_redirect(rt, skb, &fl4, true);
 833 }
 834
 835 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 836 {
 837         struct rtable *rt = (struct rtable *)dst;
 838         struct dst_entry *ret = dst;
 839
 840         if (rt) {
 841                 if (dst->obsolete > 0) {
 842                         ip_rt_put(rt);
 843                         ret = NULL;
 844                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 845                            rt->dst.expires) {
 846                         ip_rt_put(rt);
 847                         ret = NULL;
 848                 }
 849         }
 850         return ret;
 851 }
 852
 853 /*
 854  * Algorithm:
 855  *      1. The first ip_rt_redirect_number redirects are sent
 856  *         with exponential backoff, then we stop sending them at all,
 857  *         assuming that the host ignores our redirects.
 858  *      2. If we did not see packets requiring redirects
 859  *         during ip_rt_redirect_silence, we assume that the host
 860  *         forgot redirected route and start to send redirects again.
 861  *
 862  * This algorithm is much cheaper and more intelligent than dumb load limiting
 863  * in icmp.c.
 864  *
 865  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 866  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 867  */
 868
 869 void ip_rt_send_redirect(struct sk_buff *skb)
 870 {
 871         struct rtable *rt = skb_rtable(skb);
 872         struct in_device *in_dev;
 873         struct inet_peer *peer;
 874         struct net *net;
 875         int log_martians;
 876         int vif;
 877
 878         rcu_read_lock();
 879         in_dev = __in_dev_get_rcu(rt->dst.dev);
 880         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 881                 rcu_read_unlock();
 882                 return;
 883         }
 884         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 885         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 886         rcu_read_unlock();
 887
 888         net = dev_net(rt->dst.dev);
 889         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 890         if (!peer) {
 891                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 892                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 893                 return;
 894         }
 895
 896         /* No redirected packets during ip_rt_redirect_silence;
 897          * reset the algorithm.
 898          */
 899         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 900                 peer->rate_tokens = 0;
 901
 902         /* Too many ignored redirects; do not send anything
 903          * set dst.rate_last to the last seen redirected packet.
 904          */
 905         if (peer->rate_tokens >= ip_rt_redirect_number) {
 906                 peer->rate_last = jiffies;
 907                 goto out_put_peer;
 908         }
 909
 910         /* Check for load limit; set rate_last to the latest sent
 911          * redirect.
 912          */
 913         if (peer->rate_tokens == 0 ||
 914             time_after(jiffies,
 915                        (peer->rate_last +
 916                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 917                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 918
 919                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 920                 peer->rate_last = jiffies;
 921                 ++peer->rate_tokens;
 922 #ifdef CONFIG_IP_ROUTE_VERBOSE
 923                 if (log_martians &&
 924                     peer->rate_tokens == ip_rt_redirect_number)
 925                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 926                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 927                                              &ip_hdr(skb)->daddr, &gw);
 928 #endif
 929         }
 930 out_put_peer:
 931         inet_putpeer(peer);
 932 }
 933
 934 static int ip_error(struct sk_buff *skb)
 935 {
 936         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 937         struct rtable *rt = skb_rtable(skb);
 938         struct inet_peer *peer;
 939         unsigned long now;
 940         struct net *net;
 941         bool send;
 942         int code;
 943
 944         /* IP on this device is disabled. */
 945         if (!in_dev)
 946                 goto out;
 947
 948         net = dev_net(rt->dst.dev);
 949         if (!IN_DEV_FORWARD(in_dev)) {
 950                 switch (rt->dst.error) {
 951                 case EHOSTUNREACH:
 952                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 953                         break;
 954
 955                 case ENETUNREACH:
 956                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 957                         break;
 958                 }
 959                 goto out;
 960         }
 961
 962         switch (rt->dst.error) {
 963         case EINVAL:
 964         default:
 965                 goto out;
 966         case EHOSTUNREACH:
 967                 code = ICMP_HOST_UNREACH;
 968                 break;
 969         case ENETUNREACH:
 970                 code = ICMP_NET_UNREACH;
 971                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 972                 break;
 973         case EACCES:
 974                 code = ICMP_PKT_FILTERED;
 975                 break;
 976         }
 977
 978         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 979                                l3mdev_master_ifindex(skb->dev), 1);
 980
 981         send = true;
 982         if (peer) {
 983                 now = jiffies;
 984                 peer->rate_tokens += now - peer->rate_last;
 985                 if (peer->rate_tokens > ip_rt_error_burst)
 986                         peer->rate_tokens = ip_rt_error_burst;
 987                 peer->rate_last = now;
 988                 if (peer->rate_tokens >= ip_rt_error_cost)
 989                         peer->rate_tokens -= ip_rt_error_cost;
 990                 else
 991                         send = false;
 992                 inet_putpeer(peer);
 993         }
 994         if (send)
 995                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 996
 997 out:    kfree_skb(skb);
 998         return 0;
 999 }
1000
1001 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1002 {
1003         struct dst_entry *dst = &rt->dst;
1004         struct fib_result res;
1005
1006         if (dst_metric_locked(dst, RTAX_MTU))
1007                 return;
1008
1009         if (ipv4_mtu(dst) < mtu)
1010                 return;
1011
1012         if (mtu < ip_rt_min_pmtu)
1013                 mtu = ip_rt_min_pmtu;
1014
1015         if (rt->rt_pmtu == mtu &&
1016             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1017                 return;
1018
1019         rcu_read_lock();
1020         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1021                 struct fib_nh *nh = &FIB_RES_NH(res);
1022
1023                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
1024                                       jiffies + ip_rt_mtu_expires);
1025         }
1026         rcu_read_unlock();
1027 }
1028
1029 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1030                               struct sk_buff *skb, u32 mtu)
1031 {
1032         struct rtable *rt = (struct rtable *) dst;
1033         struct flowi4 fl4;
1034
1035         ip_rt_build_flow_key(&fl4, sk, skb);
1036         __ip_rt_update_pmtu(rt, &fl4, mtu);
1037 }
1038
1039 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1040                       int oif, u32 mark, u8 protocol, int flow_flags)
1041 {
1042         const struct iphdr *iph = (const struct iphdr *) skb->data;
1043         struct flowi4 fl4;
1044         struct rtable *rt;
1045
1046         if (!mark)
1047                 mark = IP4_REPLY_MARK(net, skb->mark);
1048
1049         __build_flow_key(net, &fl4, NULL, iph, oif,
1050                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1051         rt = __ip_route_output_key(net, &fl4);
1052         if (!IS_ERR(rt)) {
1053                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1054                 ip_rt_put(rt);
1055         }
1056 }
1057 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1058
1059 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1060 {
1061         const struct iphdr *iph = (const struct iphdr *) skb->data;
1062         struct flowi4 fl4;
1063         struct rtable *rt;
1064
1065         __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1066
1067         if (!fl4.flowi4_mark)
1068                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1069
1070         rt = __ip_route_output_key(sock_net(sk), &fl4);
1071         if (!IS_ERR(rt)) {
1072                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1073                 ip_rt_put(rt);
1074         }
1075 }
1076
1077 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1078 {
1079         const struct iphdr *iph = (const struct iphdr *) skb->data;
1080         struct flowi4 fl4;
1081         struct rtable *rt;
1082         struct dst_entry *odst = NULL;
1083         bool new = false;
1084         struct net *net = sock_net(sk);
1085
1086         bh_lock_sock(sk);
1087
1088         if (!ip_sk_accept_pmtu(sk))
1089                 goto out;
1090
1091         odst = sk_dst_get(sk);
1092
1093         if (sock_owned_by_user(sk) || !odst) {
1094                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1095                 goto out;
1096         }
1097
1098         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1099
1100         rt = (struct rtable *)odst;
1101         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1102                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1103                 if (IS_ERR(rt))
1104                         goto out;
1105
1106                 new = true;
1107         }
1108
1109         __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
1110
1111         if (!dst_check(&rt->dst, 0)) {
1112                 if (new)
1113                         dst_release(&rt->dst);
1114
1115                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1116                 if (IS_ERR(rt))
1117                         goto out;
1118
1119                 new = true;
1120         }
1121
1122         if (new)
1123                 sk_dst_set(sk, &rt->dst);
1124
1125 out:
1126         bh_unlock_sock(sk);
1127         dst_release(odst);
1128 }
1129 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1130
1131 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1132                    int oif, u32 mark, u8 protocol, int flow_flags)
1133 {
1134         const struct iphdr *iph = (const struct iphdr *) skb->data;
1135         struct flowi4 fl4;
1136         struct rtable *rt;
1137
1138         __build_flow_key(net, &fl4, NULL, iph, oif,
1139                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1140         rt = __ip_route_output_key(net, &fl4);
1141         if (!IS_ERR(rt)) {
1142                 __ip_do_redirect(rt, skb, &fl4, false);
1143                 ip_rt_put(rt);
1144         }
1145 }
1146 EXPORT_SYMBOL_GPL(ipv4_redirect);
1147
1148 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1149 {
1150         const struct iphdr *iph = (const struct iphdr *) skb->data;
1151         struct flowi4 fl4;
1152         struct rtable *rt;
1153         struct net *net = sock_net(sk);
1154
1155         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1156         rt = __ip_route_output_key(net, &fl4);
1157         if (!IS_ERR(rt)) {
1158                 __ip_do_redirect(rt, skb, &fl4, false);
1159                 ip_rt_put(rt);
1160         }
1161 }
1162 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1163
1164 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1165 {
1166         struct rtable *rt = (struct rtable *) dst;
1167
1168         /* All IPV4 dsts are created with ->obsolete set to the value
1169          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1170          * into this function always.
1171          *
1172          * When a PMTU/redirect information update invalidates a route,
1173          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1174          * DST_OBSOLETE_DEAD by dst_free().
1175          */
1176         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1177                 return NULL;
1178         return dst;
1179 }
1180
1181 static void ipv4_link_failure(struct sk_buff *skb)
1182 {
1183         struct rtable *rt;
1184
1185         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1186
1187         rt = skb_rtable(skb);
1188         if (rt)
1189                 dst_set_expires(&rt->dst, 0);
1190 }
1191
1192 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1193 {
1194         pr_debug("%s: %pI4 -> %pI4, %s\n",
1195                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1196                  skb->dev ? skb->dev->name : "?");
1197         kfree_skb(skb);
1198         WARN_ON(1);
1199         return 0;
1200 }
1201
1202 /*
1203    We do not cache source address of outgoing interface,
1204    because it is used only by IP RR, TS and SRR options,
1205    so that it out of fast path.
1206
1207    BTW remember: "addr" is allowed to be not aligned
1208    in IP options!
1209  */
1210
1211 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1212 {
1213         __be32 src;
1214
1215         if (rt_is_output_route(rt))
1216                 src = ip_hdr(skb)->saddr;
1217         else {
1218                 struct fib_result res;
1219                 struct flowi4 fl4;
1220                 struct iphdr *iph;
1221
1222                 iph = ip_hdr(skb);
1223
1224                 memset(&fl4, 0, sizeof(fl4));
1225                 fl4.daddr = iph->daddr;
1226                 fl4.saddr = iph->saddr;
1227                 fl4.flowi4_tos = RT_TOS(iph->tos);
1228                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1229                 fl4.flowi4_iif = skb->dev->ifindex;
1230                 fl4.flowi4_mark = skb->mark;
1231
1232                 rcu_read_lock();
1233                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1234                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1235                 else
1236                         src = inet_select_addr(rt->dst.dev,
1237                                                rt_nexthop(rt, iph->daddr),
1238                                                RT_SCOPE_UNIVERSE);
1239                 rcu_read_unlock();
1240         }
1241         memcpy(addr, &src, 4);
1242 }
1243
1244 #ifdef CONFIG_IP_ROUTE_CLASSID
1245 static void set_class_tag(struct rtable *rt, u32 tag)
1246 {
1247         if (!(rt->dst.tclassid & 0xFFFF))
1248                 rt->dst.tclassid |= tag & 0xFFFF;
1249         if (!(rt->dst.tclassid & 0xFFFF0000))
1250                 rt->dst.tclassid |= tag & 0xFFFF0000;
1251 }
1252 #endif
1253
1254 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1255 {
1256         unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1257         unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1258                                     ip_rt_min_advmss);
1259
1260         return min(advmss, IPV4_MAX_PMTU - header_size);
1261 }
1262
1263 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1264 {
1265         const struct rtable *rt = (const struct rtable *) dst;
1266         unsigned int mtu = rt->rt_pmtu;
1267
1268         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1269                 mtu = dst_metric_raw(dst, RTAX_MTU);
1270
1271         if (mtu)
1272                 return mtu;
1273
1274         mtu = READ_ONCE(dst->dev->mtu);
1275
1276         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1277                 if (rt->rt_uses_gateway && mtu > 576)
1278                         mtu = 576;
1279         }
1280
1281         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1282
1283         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1284 }
1285
1286 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1287 {
1288         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1289         struct fib_nh_exception *fnhe;
1290         u32 hval;
1291
1292         if (!hash)
1293                 return NULL;
1294
1295         hval = fnhe_hashfun(daddr);
1296
1297         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1298              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1299                 if (fnhe->fnhe_daddr == daddr)
1300                         return fnhe;
1301         }
1302         return NULL;
1303 }
1304
1305 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1306                               __be32 daddr, const bool do_cache)
1307 {
1308         bool ret = false;
1309
1310         spin_lock_bh(&fnhe_lock);
1311
1312         if (daddr == fnhe->fnhe_daddr) {
1313                 struct rtable __rcu **porig;
1314                 struct rtable *orig;
1315                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1316
1317                 if (rt_is_input_route(rt))
1318                         porig = &fnhe->fnhe_rth_input;
1319                 else
1320                         porig = &fnhe->fnhe_rth_output;
1321                 orig = rcu_dereference(*porig);
1322
1323                 if (fnhe->fnhe_genid != genid) {
1324                         fnhe->fnhe_genid = genid;
1325                         fnhe->fnhe_gw = 0;
1326                         fnhe->fnhe_pmtu = 0;
1327                         fnhe->fnhe_expires = 0;
1328                         fnhe_flush_routes(fnhe);
1329                         orig = NULL;
1330                 }
1331                 fill_route_from_fnhe(rt, fnhe);
1332                 if (!rt->rt_gateway)
1333                         rt->rt_gateway = daddr;
1334
1335                 if (do_cache) {
1336                         dst_hold(&rt->dst);
1337                         rcu_assign_pointer(*porig, rt);
1338                         if (orig) {
1339                                 dst_dev_put(&orig->dst);
1340                                 dst_release(&orig->dst);
1341                         }
1342                         ret = true;
1343                 }
1344
1345                 fnhe->fnhe_stamp = jiffies;
1346         }
1347         spin_unlock_bh(&fnhe_lock);
1348
1349         return ret;
1350 }
1351
1352 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1353 {
1354         struct rtable *orig, *prev, **p;
1355         bool ret = true;
1356
1357         if (rt_is_input_route(rt)) {
1358                 p = (struct rtable **)&nh->nh_rth_input;
1359         } else {
1360                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1361         }
1362         orig = *p;
1363
1364         /* hold dst before doing cmpxchg() to avoid race condition
1365          * on this dst
1366          */
1367         dst_hold(&rt->dst);
1368         prev = cmpxchg(p, orig, rt);
1369         if (prev == orig) {
1370                 if (orig) {
1371                         dst_dev_put(&orig->dst);
1372                         dst_release(&orig->dst);
1373                 }
1374         } else {
1375                 dst_release(&rt->dst);
1376                 ret = false;
1377         }
1378
1379         return ret;
1380 }
1381
1382 struct uncached_list {
1383         spinlock_t              lock;
1384         struct list_head        head;
1385 };
1386
1387 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1388
1389 static void rt_add_uncached_list(struct rtable *rt)
1390 {
1391         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1392
1393         rt->rt_uncached_list = ul;
1394
1395         spin_lock_bh(&ul->lock);
1396         list_add_tail(&rt->rt_uncached, &ul->head);
1397         spin_unlock_bh(&ul->lock);
1398 }
1399
1400 static void ipv4_dst_destroy(struct dst_entry *dst)
1401 {
1402         struct dst_metrics *p = (struct dst_metrics *)DST_METRICS_PTR(dst);
1403         struct rtable *rt = (struct rtable *) dst;
1404
1405         if (p != &dst_default_metrics && refcount_dec_and_test(&p->refcnt))
1406                 kfree(p);
1407
1408         if (!list_empty(&rt->rt_uncached)) {
1409                 struct uncached_list *ul = rt->rt_uncached_list;
1410
1411                 spin_lock_bh(&ul->lock);
1412                 list_del(&rt->rt_uncached);
1413                 spin_unlock_bh(&ul->lock);
1414         }
1415 }
1416
1417 void rt_flush_dev(struct net_device *dev)
1418 {
1419         struct net *net = dev_net(dev);
1420         struct rtable *rt;
1421         int cpu;
1422
1423         for_each_possible_cpu(cpu) {
1424                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1425
1426                 spin_lock_bh(&ul->lock);
1427                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1428                         if (rt->dst.dev != dev)
1429                                 continue;
1430                         rt->dst.dev = net->loopback_dev;
1431                         dev_hold(rt->dst.dev);
1432                         dev_put(dev);
1433                 }
1434                 spin_unlock_bh(&ul->lock);
1435         }
1436 }
1437
1438 static bool rt_cache_valid(const struct rtable *rt)
1439 {
1440         return  rt &&
1441                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1442                 !rt_is_expired(rt);
1443 }
1444
1445 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1446                            const struct fib_result *res,
1447                            struct fib_nh_exception *fnhe,
1448                            struct fib_info *fi, u16 type, u32 itag,
1449                            const bool do_cache)
1450 {
1451         bool cached = false;
1452
1453         if (fi) {
1454                 struct fib_nh *nh = &FIB_RES_NH(*res);
1455
1456                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1457                         rt->rt_gateway = nh->nh_gw;
1458                         rt->rt_uses_gateway = 1;
1459                 }
1460                 dst_init_metrics(&rt->dst, fi->fib_metrics->metrics, true);
1461                 if (fi->fib_metrics != &dst_default_metrics) {
1462                         rt->dst._metrics |= DST_METRICS_REFCOUNTED;
1463                         refcount_inc(&fi->fib_metrics->refcnt);
1464                 }
1465 #ifdef CONFIG_IP_ROUTE_CLASSID
1466                 rt->dst.tclassid = nh->nh_tclassid;
1467 #endif
1468                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1469                 if (unlikely(fnhe))
1470                         cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1471                 else if (do_cache)
1472                         cached = rt_cache_route(nh, rt);
1473                 if (unlikely(!cached)) {
1474                         /* Routes we intend to cache in nexthop exception or
1475                          * FIB nexthop have the DST_NOCACHE bit clear.
1476                          * However, if we are unsuccessful at storing this
1477                          * route into the cache we really need to set it.
1478                          */
1479                         if (!rt->rt_gateway)
1480                                 rt->rt_gateway = daddr;
1481                         rt_add_uncached_list(rt);
1482                 }
1483         } else
1484                 rt_add_uncached_list(rt);
1485
1486 #ifdef CONFIG_IP_ROUTE_CLASSID
1487 #ifdef CONFIG_IP_MULTIPLE_TABLES
1488         set_class_tag(rt, res->tclassid);
1489 #endif
1490         set_class_tag(rt, itag);
1491 #endif
1492 }
1493
1494 struct rtable *rt_dst_alloc(struct net_device *dev,
1495                             unsigned int flags, u16 type,
1496                             bool nopolicy, bool noxfrm, bool will_cache)
1497 {
1498         struct rtable *rt;
1499
1500         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1501                        (will_cache ? 0 : DST_HOST) |
1502                        (nopolicy ? DST_NOPOLICY : 0) |
1503                        (noxfrm ? DST_NOXFRM : 0));
1504
1505         if (rt) {
1506                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1507                 rt->rt_flags = flags;
1508                 rt->rt_type = type;
1509                 rt->rt_is_input = 0;
1510                 rt->rt_iif = 0;
1511                 rt->rt_pmtu = 0;
1512                 rt->rt_gateway = 0;
1513                 rt->rt_uses_gateway = 0;
1514                 rt->rt_table_id = 0;
1515                 INIT_LIST_HEAD(&rt->rt_uncached);
1516
1517                 rt->dst.output = ip_output;
1518                 if (flags & RTCF_LOCAL)
1519                         rt->dst.input = ip_local_deliver;
1520         }
1521
1522         return rt;
1523 }
1524 EXPORT_SYMBOL(rt_dst_alloc);
1525
1526 /* called in rcu_read_lock() section */
1527 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1528                           u8 tos, struct net_device *dev,
1529                           struct in_device *in_dev, u32 *itag)
1530 {
1531         int err;
1532
1533         /* Primary sanity checks. */
1534         if (!in_dev)
1535                 return -EINVAL;
1536
1537         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1538             skb->protocol != htons(ETH_P_IP))
1539                 return -EINVAL;
1540
1541         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1542                 return -EINVAL;
1543
1544         if (ipv4_is_zeronet(saddr)) {
1545                 if (!ipv4_is_local_multicast(daddr))
1546                         return -EINVAL;
1547         } else {
1548                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1549                                           in_dev, itag);
1550                 if (err < 0)
1551                         return err;
1552         }
1553         return 0;
1554 }
1555
1556 /* called in rcu_read_lock() section */
1557 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1558                              u8 tos, struct net_device *dev, int our)
1559 {
1560         struct in_device *in_dev = __in_dev_get_rcu(dev);
1561         unsigned int flags = RTCF_MULTICAST;
1562         struct rtable *rth;
1563         u32 itag = 0;
1564         int err;
1565
1566         err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1567         if (err)
1568                 return err;
1569
1570         if (our)
1571                 flags |= RTCF_LOCAL;
1572
1573         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1574                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1575         if (!rth)
1576                 return -ENOBUFS;
1577
1578 #ifdef CONFIG_IP_ROUTE_CLASSID
1579         rth->dst.tclassid = itag;
1580 #endif
1581         rth->dst.output = ip_rt_bug;
1582         rth->rt_is_input= 1;
1583
1584 #ifdef CONFIG_IP_MROUTE
1585         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1586                 rth->dst.input = ip_mr_input;
1587 #endif
1588         RT_CACHE_STAT_INC(in_slow_mc);
1589
1590         skb_dst_set(skb, &rth->dst);
1591         return 0;
1592 }
1593
1594
1595 static void ip_handle_martian_source(struct net_device *dev,
1596                                      struct in_device *in_dev,
1597                                      struct sk_buff *skb,
1598                                      __be32 daddr,
1599                                      __be32 saddr)
1600 {
1601         RT_CACHE_STAT_INC(in_martian_src);
1602 #ifdef CONFIG_IP_ROUTE_VERBOSE
1603         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1604                 /*
1605                  *      RFC1812 recommendation, if source is martian,
1606                  *      the only hint is MAC header.
1607                  */
1608                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1609                         &daddr, &saddr, dev->name);
1610                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1611                         print_hex_dump(KERN_WARNING, "ll header: ",
1612                                        DUMP_PREFIX_OFFSET, 16, 1,
1613                                        skb_mac_header(skb),
1614                                        dev->hard_header_len, true);
1615                 }
1616         }
1617 #endif
1618 }
1619
1620 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1621 {
1622         struct fnhe_hash_bucket *hash;
1623         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1624         u32 hval = fnhe_hashfun(daddr);
1625
1626         spin_lock_bh(&fnhe_lock);
1627
1628         hash = rcu_dereference_protected(nh->nh_exceptions,
1629                                          lockdep_is_held(&fnhe_lock));
1630         hash += hval;
1631
1632         fnhe_p = &hash->chain;
1633         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1634         while (fnhe) {
1635                 if (fnhe->fnhe_daddr == daddr) {
1636                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1637                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1638                         fnhe_flush_routes(fnhe);
1639                         kfree_rcu(fnhe, rcu);
1640                         break;
1641                 }
1642                 fnhe_p = &fnhe->fnhe_next;
1643                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1644                                                  lockdep_is_held(&fnhe_lock));
1645         }
1646
1647         spin_unlock_bh(&fnhe_lock);
1648 }
1649
1650 static void set_lwt_redirect(struct rtable *rth)
1651 {
1652         if (lwtunnel_output_redirect(rth->dst.lwtstate)) {
1653                 rth->dst.lwtstate->orig_output = rth->dst.output;
1654                 rth->dst.output = lwtunnel_output;
1655         }
1656
1657         if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
1658                 rth->dst.lwtstate->orig_input = rth->dst.input;
1659                 rth->dst.input = lwtunnel_input;
1660         }
1661 }
1662
1663 /* called in rcu_read_lock() section */
1664 static int __mkroute_input(struct sk_buff *skb,
1665                            const struct fib_result *res,
1666                            struct in_device *in_dev,
1667                            __be32 daddr, __be32 saddr, u32 tos)
1668 {
1669         struct fib_nh_exception *fnhe;
1670         struct rtable *rth;
1671         int err;
1672         struct in_device *out_dev;
1673         bool do_cache;
1674         u32 itag = 0;
1675
1676         /* get a working reference to the output device */
1677         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1678         if (!out_dev) {
1679                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1680                 return -EINVAL;
1681         }
1682
1683         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1684                                   in_dev->dev, in_dev, &itag);
1685         if (err < 0) {
1686                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1687                                          saddr);
1688
1689                 goto cleanup;
1690         }
1691
1692         do_cache = res->fi && !itag;
1693         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1694             skb->protocol == htons(ETH_P_IP) &&
1695             (IN_DEV_SHARED_MEDIA(out_dev) ||
1696              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1697                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1698
1699         if (skb->protocol != htons(ETH_P_IP)) {
1700                 /* Not IP (i.e. ARP). Do not create route, if it is
1701                  * invalid for proxy arp. DNAT routes are always valid.
1702                  *
1703                  * Proxy arp feature have been extended to allow, ARP
1704                  * replies back to the same interface, to support
1705                  * Private VLAN switch technologies. See arp.c.
1706                  */
1707                 if (out_dev == in_dev &&
1708                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1709                         err = -EINVAL;
1710                         goto cleanup;
1711                 }
1712         }
1713
1714         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1715         if (do_cache) {
1716                 if (fnhe) {
1717                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1718                         if (rth && rth->dst.expires &&
1719                             time_after(jiffies, rth->dst.expires)) {
1720                                 ip_del_fnhe(&FIB_RES_NH(*res), daddr);
1721                                 fnhe = NULL;
1722                         } else {
1723                                 goto rt_cache;
1724                         }
1725                 }
1726
1727                 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1728
1729 rt_cache:
1730                 if (rt_cache_valid(rth)) {
1731                         skb_dst_set_noref(skb, &rth->dst);
1732                         goto out;
1733                 }
1734         }
1735
1736         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1737                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1738                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1739         if (!rth) {
1740                 err = -ENOBUFS;
1741                 goto cleanup;
1742         }
1743
1744         rth->rt_is_input = 1;
1745         if (res->table)
1746                 rth->rt_table_id = res->table->tb_id;
1747         RT_CACHE_STAT_INC(in_slow_tot);
1748
1749         rth->dst.input = ip_forward;
1750
1751         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1752                        do_cache);
1753         set_lwt_redirect(rth);
1754         skb_dst_set(skb, &rth->dst);
1755 out:
1756         err = 0;
1757  cleanup:
1758         return err;
1759 }
1760
1761 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1762 /* To make ICMP packets follow the right flow, the multipath hash is
1763  * calculated from the inner IP addresses.
1764  */
1765 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1766                                  struct flow_keys *hash_keys)
1767 {
1768         const struct iphdr *outer_iph = ip_hdr(skb);
1769         const struct iphdr *inner_iph;
1770         const struct icmphdr *icmph;
1771         struct iphdr _inner_iph;
1772         struct icmphdr _icmph;
1773
1774         hash_keys->addrs.v4addrs.src = outer_iph->saddr;
1775         hash_keys->addrs.v4addrs.dst = outer_iph->daddr;
1776         if (likely(outer_iph->protocol != IPPROTO_ICMP))
1777                 return;
1778
1779         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1780                 return;
1781
1782         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1783                                    &_icmph);
1784         if (!icmph)
1785                 return;
1786
1787         if (icmph->type != ICMP_DEST_UNREACH &&
1788             icmph->type != ICMP_REDIRECT &&
1789             icmph->type != ICMP_TIME_EXCEEDED &&
1790             icmph->type != ICMP_PARAMETERPROB)
1791                 return;
1792
1793         inner_iph = skb_header_pointer(skb,
1794                                        outer_iph->ihl * 4 + sizeof(_icmph),
1795                                        sizeof(_inner_iph), &_inner_iph);
1796         if (!inner_iph)
1797                 return;
1798         hash_keys->addrs.v4addrs.src = inner_iph->saddr;
1799         hash_keys->addrs.v4addrs.dst = inner_iph->daddr;
1800 }
1801
1802 /* if skb is set it will be used and fl4 can be NULL */
1803 int fib_multipath_hash(const struct fib_info *fi, const struct flowi4 *fl4,
1804                        const struct sk_buff *skb)
1805 {
1806         struct net *net = fi->fib_net;
1807         struct flow_keys hash_keys;
1808         u32 mhash;
1809
1810         switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1811         case 0:
1812                 memset(&hash_keys, 0, sizeof(hash_keys));
1813                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1814                 if (skb) {
1815                         ip_multipath_l3_keys(skb, &hash_keys);
1816                 } else {
1817                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1818                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1819                 }
1820                 break;
1821         case 1:
1822                 /* skb is currently provided only when forwarding */
1823                 if (skb) {
1824                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1825                         struct flow_keys keys;
1826
1827                         /* short-circuit if we already have L4 hash present */
1828                         if (skb->l4_hash)
1829                                 return skb_get_hash_raw(skb) >> 1;
1830                         memset(&hash_keys, 0, sizeof(hash_keys));
1831                         skb_flow_dissect_flow_keys(skb, &keys, flag);
1832                         hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
1833                         hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
1834                         hash_keys.ports.src = keys.ports.src;
1835                         hash_keys.ports.dst = keys.ports.dst;
1836                         hash_keys.basic.ip_proto = keys.basic.ip_proto;
1837                 } else {
1838                         memset(&hash_keys, 0, sizeof(hash_keys));
1839                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1840                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1841                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1842                         hash_keys.ports.src = fl4->fl4_sport;
1843                         hash_keys.ports.dst = fl4->fl4_dport;
1844                         hash_keys.basic.ip_proto = fl4->flowi4_proto;
1845                 }
1846                 break;
1847         }
1848         mhash = flow_hash_from_keys(&hash_keys);
1849
1850         return mhash >> 1;
1851 }
1852 EXPORT_SYMBOL_GPL(fib_multipath_hash);
1853 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1854
1855 static int ip_mkroute_input(struct sk_buff *skb,
1856                             struct fib_result *res,
1857                             struct in_device *in_dev,
1858                             __be32 daddr, __be32 saddr, u32 tos)
1859 {
1860 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1861         if (res->fi && res->fi->fib_nhs > 1) {
1862                 int h = fib_multipath_hash(res->fi, NULL, skb);
1863
1864                 fib_select_multipath(res, h);
1865         }
1866 #endif
1867
1868         /* create a routing cache entry */
1869         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1870 }
1871
1872 /*
1873  *      NOTE. We drop all the packets that has local source
1874  *      addresses, because every properly looped back packet
1875  *      must have correct destination already attached by output routine.
1876  *
1877  *      Such approach solves two big problems:
1878  *      1. Not simplex devices are handled properly.
1879  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1880  *      called with rcu_read_lock()
1881  */
1882
1883 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1884                                u8 tos, struct net_device *dev,
1885                                struct fib_result *res)
1886 {
1887         struct in_device *in_dev = __in_dev_get_rcu(dev);
1888         struct ip_tunnel_info *tun_info;
1889         struct flowi4   fl4;
1890         unsigned int    flags = 0;
1891         u32             itag = 0;
1892         struct rtable   *rth;
1893         int             err = -EINVAL;
1894         struct net    *net = dev_net(dev);
1895         bool do_cache;
1896
1897         /* IP on this device is disabled. */
1898
1899         if (!in_dev)
1900                 goto out;
1901
1902         /* Check for the most weird martians, which can be not detected
1903            by fib_lookup.
1904          */
1905
1906         tun_info = skb_tunnel_info(skb);
1907         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1908                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1909         else
1910                 fl4.flowi4_tun_key.tun_id = 0;
1911         skb_dst_drop(skb);
1912
1913         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1914                 goto martian_source;
1915
1916         res->fi = NULL;
1917         res->table = NULL;
1918         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1919                 goto brd_input;
1920
1921         /* Accept zero addresses only to limited broadcast;
1922          * I even do not know to fix it or not. Waiting for complains :-)
1923          */
1924         if (ipv4_is_zeronet(saddr))
1925                 goto martian_source;
1926
1927         if (ipv4_is_zeronet(daddr))
1928                 goto martian_destination;
1929
1930         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1931          * and call it once if daddr or/and saddr are loopback addresses
1932          */
1933         if (ipv4_is_loopback(daddr)) {
1934                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1935                         goto martian_destination;
1936         } else if (ipv4_is_loopback(saddr)) {
1937                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1938                         goto martian_source;
1939         }
1940
1941         /*
1942          *      Now we are ready to route packet.
1943          */
1944         fl4.flowi4_oif = 0;
1945         fl4.flowi4_iif = dev->ifindex;
1946         fl4.flowi4_mark = skb->mark;
1947         fl4.flowi4_tos = tos;
1948         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1949         fl4.flowi4_flags = 0;
1950         fl4.daddr = daddr;
1951         fl4.saddr = saddr;
1952         fl4.flowi4_uid = sock_net_uid(net, NULL);
1953         err = fib_lookup(net, &fl4, res, 0);
1954         if (err != 0) {
1955                 if (!IN_DEV_FORWARD(in_dev))
1956                         err = -EHOSTUNREACH;
1957                 goto no_route;
1958         }
1959
1960         if (res->type == RTN_BROADCAST)
1961                 goto brd_input;
1962
1963         if (res->type == RTN_LOCAL) {
1964                 err = fib_validate_source(skb, saddr, daddr, tos,
1965                                           0, dev, in_dev, &itag);
1966                 if (err < 0)
1967                         goto martian_source;
1968                 goto local_input;
1969         }
1970
1971         if (!IN_DEV_FORWARD(in_dev)) {
1972                 err = -EHOSTUNREACH;
1973                 goto no_route;
1974         }
1975         if (res->type != RTN_UNICAST)
1976                 goto martian_destination;
1977
1978         err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1979 out:    return err;
1980
1981 brd_input:
1982         if (skb->protocol != htons(ETH_P_IP))
1983                 goto e_inval;
1984
1985         if (!ipv4_is_zeronet(saddr)) {
1986                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1987                                           in_dev, &itag);
1988                 if (err < 0)
1989                         goto martian_source;
1990         }
1991         flags |= RTCF_BROADCAST;
1992         res->type = RTN_BROADCAST;
1993         RT_CACHE_STAT_INC(in_brd);
1994
1995 local_input:
1996         do_cache = false;
1997         if (res->fi) {
1998                 if (!itag) {
1999                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
2000                         if (rt_cache_valid(rth)) {
2001                                 skb_dst_set_noref(skb, &rth->dst);
2002                                 err = 0;
2003                                 goto out;
2004                         }
2005                         do_cache = true;
2006                 }
2007         }
2008
2009         rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2010                            flags | RTCF_LOCAL, res->type,
2011                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2012         if (!rth)
2013                 goto e_nobufs;
2014
2015         rth->dst.output= ip_rt_bug;
2016 #ifdef CONFIG_IP_ROUTE_CLASSID
2017         rth->dst.tclassid = itag;
2018 #endif
2019         rth->rt_is_input = 1;
2020         if (res->table)
2021                 rth->rt_table_id = res->table->tb_id;
2022
2023         RT_CACHE_STAT_INC(in_slow_tot);
2024         if (res->type == RTN_UNREACHABLE) {
2025                 rth->dst.input= ip_error;
2026                 rth->dst.error= -err;
2027                 rth->rt_flags   &= ~RTCF_LOCAL;
2028         }
2029
2030         if (do_cache) {
2031                 struct fib_nh *nh = &FIB_RES_NH(*res);
2032
2033                 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2034                 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2035                         WARN_ON(rth->dst.input == lwtunnel_input);
2036                         rth->dst.lwtstate->orig_input = rth->dst.input;
2037                         rth->dst.input = lwtunnel_input;
2038                 }
2039
2040                 if (unlikely(!rt_cache_route(nh, rth)))
2041                         rt_add_uncached_list(rth);
2042         }
2043         skb_dst_set(skb, &rth->dst);
2044         err = 0;
2045         goto out;
2046
2047 no_route:
2048         RT_CACHE_STAT_INC(in_no_route);
2049         res->type = RTN_UNREACHABLE;
2050         res->fi = NULL;
2051         res->table = NULL;
2052         goto local_input;
2053
2054         /*
2055          *      Do not cache martian addresses: they should be logged (RFC1812)
2056          */
2057 martian_destination:
2058         RT_CACHE_STAT_INC(in_martian_dst);
2059 #ifdef CONFIG_IP_ROUTE_VERBOSE
2060         if (IN_DEV_LOG_MARTIANS(in_dev))
2061                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2062                                      &daddr, &saddr, dev->name);
2063 #endif
2064
2065 e_inval:
2066         err = -EINVAL;
2067         goto out;
2068
2069 e_nobufs:
2070         err = -ENOBUFS;
2071         goto out;
2072
2073 martian_source:
2074         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2075         goto out;
2076 }
2077
2078 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2079                          u8 tos, struct net_device *dev)
2080 {
2081         struct fib_result res;
2082         int err;
2083
2084         tos &= IPTOS_RT_MASK;
2085         rcu_read_lock();
2086         err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2087         rcu_read_unlock();
2088
2089         return err;
2090 }
2091 EXPORT_SYMBOL(ip_route_input_noref);
2092
2093 /* called with rcu_read_lock held */
2094 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2095                        u8 tos, struct net_device *dev, struct fib_result *res)
2096 {
2097         /* Multicast recognition logic is moved from route cache to here.
2098            The problem was that too many Ethernet cards have broken/missing
2099            hardware multicast filters :-( As result the host on multicasting
2100            network acquires a lot of useless route cache entries, sort of
2101            SDR messages from all the world. Now we try to get rid of them.
2102            Really, provided software IP multicast filter is organized
2103            reasonably (at least, hashed), it does not result in a slowdown
2104            comparing with route cache reject entries.
2105            Note, that multicast routers are not affected, because
2106            route cache entry is created eventually.
2107          */
2108         if (ipv4_is_multicast(daddr)) {
2109                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2110                 int our = 0;
2111                 int err = -EINVAL;
2112
2113                 if (in_dev)
2114                         our = ip_check_mc_rcu(in_dev, daddr, saddr,
2115                                               ip_hdr(skb)->protocol);
2116
2117                 /* check l3 master if no match yet */
2118                 if ((!in_dev || !our) && netif_is_l3_slave(dev)) {
2119                         struct in_device *l3_in_dev;
2120
2121                         l3_in_dev = __in_dev_get_rcu(skb->dev);
2122                         if (l3_in_dev)
2123                                 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2124                                                       ip_hdr(skb)->protocol);
2125                 }
2126
2127                 if (our
2128 #ifdef CONFIG_IP_MROUTE
2129                         ||
2130                     (!ipv4_is_local_multicast(daddr) &&
2131                      IN_DEV_MFORWARD(in_dev))
2132 #endif
2133                    ) {
2134                         err = ip_route_input_mc(skb, daddr, saddr,
2135                                                 tos, dev, our);
2136                 }
2137                 return err;
2138         }
2139
2140         return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2141 }
2142
2143 /* called with rcu_read_lock() */
2144 static struct rtable *__mkroute_output(const struct fib_result *res,
2145                                        const struct flowi4 *fl4, int orig_oif,
2146                                        struct net_device *dev_out,
2147                                        unsigned int flags)
2148 {
2149         struct fib_info *fi = res->fi;
2150         struct fib_nh_exception *fnhe;
2151         struct in_device *in_dev;
2152         u16 type = res->type;
2153         struct rtable *rth;
2154         bool do_cache;
2155
2156         in_dev = __in_dev_get_rcu(dev_out);
2157         if (!in_dev)
2158                 return ERR_PTR(-EINVAL);
2159
2160         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2161                 if (ipv4_is_loopback(fl4->saddr) &&
2162                     !(dev_out->flags & IFF_LOOPBACK) &&
2163                     !netif_is_l3_master(dev_out))
2164                         return ERR_PTR(-EINVAL);
2165
2166         if (ipv4_is_lbcast(fl4->daddr))
2167                 type = RTN_BROADCAST;
2168         else if (ipv4_is_multicast(fl4->daddr))
2169                 type = RTN_MULTICAST;
2170         else if (ipv4_is_zeronet(fl4->daddr))
2171                 return ERR_PTR(-EINVAL);
2172
2173         if (dev_out->flags & IFF_LOOPBACK)
2174                 flags |= RTCF_LOCAL;
2175
2176         do_cache = true;
2177         if (type == RTN_BROADCAST) {
2178                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2179                 fi = NULL;
2180         } else if (type == RTN_MULTICAST) {
2181                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2182                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2183                                      fl4->flowi4_proto))
2184                         flags &= ~RTCF_LOCAL;
2185                 else
2186                         do_cache = false;
2187                 /* If multicast route do not exist use
2188                  * default one, but do not gateway in this case.
2189                  * Yes, it is hack.
2190                  */
2191                 if (fi && res->prefixlen < 4)
2192                         fi = NULL;
2193         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2194                    (orig_oif != dev_out->ifindex)) {
2195                 /* For local routes that require a particular output interface
2196                  * we do not want to cache the result.  Caching the result
2197                  * causes incorrect behaviour when there are multiple source
2198                  * addresses on the interface, the end result being that if the
2199                  * intended recipient is waiting on that interface for the
2200                  * packet he won't receive it because it will be delivered on
2201                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2202                  * be set to the loopback interface as well.
2203                  */
2204                 fi = NULL;
2205         }
2206
2207         fnhe = NULL;
2208         do_cache &= fi != NULL;
2209         if (do_cache) {
2210                 struct rtable __rcu **prth;
2211                 struct fib_nh *nh = &FIB_RES_NH(*res);
2212
2213                 fnhe = find_exception(nh, fl4->daddr);
2214                 if (fnhe) {
2215                         prth = &fnhe->fnhe_rth_output;
2216                         rth = rcu_dereference(*prth);
2217                         if (rth && rth->dst.expires &&
2218                             time_after(jiffies, rth->dst.expires)) {
2219                                 ip_del_fnhe(nh, fl4->daddr);
2220                                 fnhe = NULL;
2221                         } else {
2222                                 goto rt_cache;
2223                         }
2224                 }
2225
2226                 if (unlikely(fl4->flowi4_flags &
2227                              FLOWI_FLAG_KNOWN_NH &&
2228                              !(nh->nh_gw &&
2229                                nh->nh_scope == RT_SCOPE_LINK))) {
2230                         do_cache = false;
2231                         goto add;
2232                 }
2233                 prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2234                 rth = rcu_dereference(*prth);
2235
2236 rt_cache:
2237                 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2238                         return rth;
2239         }
2240
2241 add:
2242         rth = rt_dst_alloc(dev_out, flags, type,
2243                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2244                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2245                            do_cache);
2246         if (!rth)
2247                 return ERR_PTR(-ENOBUFS);
2248
2249         rth->rt_iif = orig_oif;
2250         if (res->table)
2251                 rth->rt_table_id = res->table->tb_id;
2252
2253         RT_CACHE_STAT_INC(out_slow_tot);
2254
2255         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2256                 if (flags & RTCF_LOCAL &&
2257                     !(dev_out->flags & IFF_LOOPBACK)) {
2258                         rth->dst.output = ip_mc_output;
2259                         RT_CACHE_STAT_INC(out_slow_mc);
2260                 }
2261 #ifdef CONFIG_IP_MROUTE
2262                 if (type == RTN_MULTICAST) {
2263                         if (IN_DEV_MFORWARD(in_dev) &&
2264                             !ipv4_is_local_multicast(fl4->daddr)) {
2265                                 rth->dst.input = ip_mr_input;
2266                                 rth->dst.output = ip_mc_output;
2267                         }
2268                 }
2269 #endif
2270         }
2271
2272         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2273         set_lwt_redirect(rth);
2274
2275         return rth;
2276 }
2277
2278 /*
2279  * Major route resolver routine.
2280  */
2281
2282 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2283                                         const struct sk_buff *skb)
2284 {
2285         __u8 tos = RT_FL_TOS(fl4);
2286         struct fib_result res;
2287         struct rtable *rth;
2288
2289         res.tclassid    = 0;
2290         res.fi          = NULL;
2291         res.table       = NULL;
2292
2293         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2294         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2295         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2296                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2297
2298         rcu_read_lock();
2299         rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2300         rcu_read_unlock();
2301
2302         return rth;
2303 }
2304 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2305
2306 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2307                                             struct fib_result *res,
2308                                             const struct sk_buff *skb)
2309 {
2310         struct net_device *dev_out = NULL;
2311         int orig_oif = fl4->flowi4_oif;
2312         unsigned int flags = 0;
2313         struct rtable *rth;
2314         int err = -ENETUNREACH;
2315
2316         if (fl4->saddr) {
2317                 rth = ERR_PTR(-EINVAL);
2318                 if (ipv4_is_multicast(fl4->saddr) ||
2319                     ipv4_is_lbcast(fl4->saddr) ||
2320                     ipv4_is_zeronet(fl4->saddr))
2321                         goto out;
2322
2323                 /* I removed check for oif == dev_out->oif here.
2324                    It was wrong for two reasons:
2325                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2326                       is assigned to multiple interfaces.
2327                    2. Moreover, we are allowed to send packets with saddr
2328                       of another iface. --ANK
2329                  */
2330
2331                 if (fl4->flowi4_oif == 0 &&
2332                     (ipv4_is_multicast(fl4->daddr) ||
2333                      ipv4_is_lbcast(fl4->daddr))) {
2334                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2335                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2336                         if (!dev_out)
2337                                 goto out;
2338
2339                         /* Special hack: user can direct multicasts
2340                            and limited broadcast via necessary interface
2341                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2342                            This hack is not just for fun, it allows
2343                            vic,vat and friends to work.
2344                            They bind socket to loopback, set ttl to zero
2345                            and expect that it will work.
2346                            From the viewpoint of routing cache they are broken,
2347                            because we are not allowed to build multicast path
2348                            with loopback source addr (look, routing cache
2349                            cannot know, that ttl is zero, so that packet
2350                            will not leave this host and route is valid).
2351                            Luckily, this hack is good workaround.
2352                          */
2353
2354                         fl4->flowi4_oif = dev_out->ifindex;
2355                         goto make_route;
2356                 }
2357
2358                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2359                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2360                         if (!__ip_dev_find(net, fl4->saddr, false))
2361                                 goto out;
2362                 }
2363         }
2364
2365
2366         if (fl4->flowi4_oif) {
2367                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2368                 rth = ERR_PTR(-ENODEV);
2369                 if (!dev_out)
2370                         goto out;
2371
2372                 /* RACE: Check return value of inet_select_addr instead. */
2373                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2374                         rth = ERR_PTR(-ENETUNREACH);
2375                         goto out;
2376                 }
2377                 if (ipv4_is_local_multicast(fl4->daddr) ||
2378                     ipv4_is_lbcast(fl4->daddr) ||
2379                     fl4->flowi4_proto == IPPROTO_IGMP) {
2380                         if (!fl4->saddr)
2381                                 fl4->saddr = inet_select_addr(dev_out, 0,
2382                                                               RT_SCOPE_LINK);
2383                         goto make_route;
2384                 }
2385                 if (!fl4->saddr) {
2386                         if (ipv4_is_multicast(fl4->daddr))
2387                                 fl4->saddr = inet_select_addr(dev_out, 0,
2388                                                               fl4->flowi4_scope);
2389                         else if (!fl4->daddr)
2390                                 fl4->saddr = inet_select_addr(dev_out, 0,
2391                                                               RT_SCOPE_HOST);
2392                 }
2393         }
2394
2395         if (!fl4->daddr) {
2396                 fl4->daddr = fl4->saddr;
2397                 if (!fl4->daddr)
2398                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2399                 dev_out = net->loopback_dev;
2400                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2401                 res->type = RTN_LOCAL;
2402                 flags |= RTCF_LOCAL;
2403                 goto make_route;
2404         }
2405
2406         err = fib_lookup(net, fl4, res, 0);
2407         if (err) {
2408                 res->fi = NULL;
2409                 res->table = NULL;
2410                 if (fl4->flowi4_oif &&
2411                     (ipv4_is_multicast(fl4->daddr) ||
2412                     !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2413                         /* Apparently, routing tables are wrong. Assume,
2414                            that the destination is on link.
2415
2416                            WHY? DW.
2417                            Because we are allowed to send to iface
2418                            even if it has NO routes and NO assigned
2419                            addresses. When oif is specified, routing
2420                            tables are looked up with only one purpose:
2421                            to catch if destination is gatewayed, rather than
2422                            direct. Moreover, if MSG_DONTROUTE is set,
2423                            we send packet, ignoring both routing tables
2424                            and ifaddr state. --ANK
2425
2426
2427                            We could make it even if oif is unknown,
2428                            likely IPv6, but we do not.
2429                          */
2430
2431                         if (fl4->saddr == 0)
2432                                 fl4->saddr = inet_select_addr(dev_out, 0,
2433                                                               RT_SCOPE_LINK);
2434                         res->type = RTN_UNICAST;
2435                         goto make_route;
2436                 }
2437                 rth = ERR_PTR(err);
2438                 goto out;
2439         }
2440
2441         if (res->type == RTN_LOCAL) {
2442                 if (!fl4->saddr) {
2443                         if (res->fi->fib_prefsrc)
2444                                 fl4->saddr = res->fi->fib_prefsrc;
2445                         else
2446                                 fl4->saddr = fl4->daddr;
2447                 }
2448
2449                 /* L3 master device is the loopback for that domain */
2450                 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2451                         net->loopback_dev;
2452
2453                 /* make sure orig_oif points to fib result device even
2454                  * though packet rx/tx happens over loopback or l3mdev
2455                  */
2456                 orig_oif = FIB_RES_OIF(*res);
2457
2458                 fl4->flowi4_oif = dev_out->ifindex;
2459                 flags |= RTCF_LOCAL;
2460                 goto make_route;
2461         }
2462
2463         fib_select_path(net, res, fl4, skb);
2464
2465         dev_out = FIB_RES_DEV(*res);
2466         fl4->flowi4_oif = dev_out->ifindex;
2467
2468
2469 make_route:
2470         rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2471
2472 out:
2473         return rth;
2474 }
2475
2476 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2477 {
2478         return NULL;
2479 }
2480
2481 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2482 {
2483         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2484
2485         return mtu ? : dst->dev->mtu;
2486 }
2487
2488 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2489                                           struct sk_buff *skb, u32 mtu)
2490 {
2491 }
2492
2493 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2494                                        struct sk_buff *skb)
2495 {
2496 }
2497
2498 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2499                                           unsigned long old)
2500 {
2501         return NULL;
2502 }
2503
2504 static struct dst_ops ipv4_dst_blackhole_ops = {
2505         .family                 =       AF_INET,
2506         .check                  =       ipv4_blackhole_dst_check,
2507         .mtu                    =       ipv4_blackhole_mtu,
2508         .default_advmss         =       ipv4_default_advmss,
2509         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2510         .redirect               =       ipv4_rt_blackhole_redirect,
2511         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2512         .neigh_lookup           =       ipv4_neigh_lookup,
2513 };
2514
2515 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2516 {
2517         struct rtable *ort = (struct rtable *) dst_orig;
2518         struct rtable *rt;
2519
2520         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2521         if (rt) {
2522                 struct dst_entry *new = &rt->dst;
2523
2524                 new->__use = 1;
2525                 new->input = dst_discard;
2526                 new->output = dst_discard_out;
2527
2528                 new->dev = net->loopback_dev;
2529                 if (new->dev)
2530                         dev_hold(new->dev);
2531
2532                 rt->rt_is_input = ort->rt_is_input;
2533                 rt->rt_iif = ort->rt_iif;
2534                 rt->rt_pmtu = ort->rt_pmtu;
2535
2536                 rt->rt_genid = rt_genid_ipv4(net);
2537                 rt->rt_flags = ort->rt_flags;
2538                 rt->rt_type = ort->rt_type;
2539                 rt->rt_gateway = ort->rt_gateway;
2540                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2541
2542                 INIT_LIST_HEAD(&rt->rt_uncached);
2543         }
2544
2545         dst_release(dst_orig);
2546
2547         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2548 }
2549
2550 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2551                                     const struct sock *sk)
2552 {
2553         struct rtable *rt = __ip_route_output_key(net, flp4);
2554
2555         if (IS_ERR(rt))
2556                 return rt;
2557
2558         if (flp4->flowi4_proto)
2559                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2560                                                         flowi4_to_flowi(flp4),
2561                                                         sk, 0);
2562
2563         return rt;
2564 }
2565 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2566
2567 /* called with rcu_read_lock held */
2568 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src, u32 table_id,
2569                         struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2570                         u32 seq)
2571 {
2572         struct rtable *rt = skb_rtable(skb);
2573         struct rtmsg *r;
2574         struct nlmsghdr *nlh;
2575         unsigned long expires = 0;
2576         u32 error;
2577         u32 metrics[RTAX_MAX];
2578
2579         nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2580         if (!nlh)
2581                 return -EMSGSIZE;
2582
2583         r = nlmsg_data(nlh);
2584         r->rtm_family    = AF_INET;
2585         r->rtm_dst_len  = 32;
2586         r->rtm_src_len  = 0;
2587         r->rtm_tos      = fl4->flowi4_tos;
2588         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2589         if (nla_put_u32(skb, RTA_TABLE, table_id))
2590                 goto nla_put_failure;
2591         r->rtm_type     = rt->rt_type;
2592         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2593         r->rtm_protocol = RTPROT_UNSPEC;
2594         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2595         if (rt->rt_flags & RTCF_NOTIFY)
2596                 r->rtm_flags |= RTM_F_NOTIFY;
2597         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2598                 r->rtm_flags |= RTCF_DOREDIRECT;
2599
2600         if (nla_put_in_addr(skb, RTA_DST, dst))
2601                 goto nla_put_failure;
2602         if (src) {
2603                 r->rtm_src_len = 32;
2604                 if (nla_put_in_addr(skb, RTA_SRC, src))
2605                         goto nla_put_failure;
2606         }
2607         if (rt->dst.dev &&
2608             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2609                 goto nla_put_failure;
2610 #ifdef CONFIG_IP_ROUTE_CLASSID
2611         if (rt->dst.tclassid &&
2612             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2613                 goto nla_put_failure;
2614 #endif
2615         if (!rt_is_input_route(rt) &&
2616             fl4->saddr != src) {
2617                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2618                         goto nla_put_failure;
2619         }
2620         if (rt->rt_uses_gateway &&
2621             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2622                 goto nla_put_failure;
2623
2624         expires = rt->dst.expires;
2625         if (expires) {
2626                 unsigned long now = jiffies;
2627
2628                 if (time_before(now, expires))
2629                         expires -= now;
2630                 else
2631                         expires = 0;
2632         }
2633
2634         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2635         if (rt->rt_pmtu && expires)
2636                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2637         if (rtnetlink_put_metrics(skb, metrics) < 0)
2638                 goto nla_put_failure;
2639
2640         if (fl4->flowi4_mark &&
2641             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2642                 goto nla_put_failure;
2643
2644         if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2645             nla_put_u32(skb, RTA_UID,
2646                         from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2647                 goto nla_put_failure;
2648
2649         error = rt->dst.error;
2650
2651         if (rt_is_input_route(rt)) {
2652 #ifdef CONFIG_IP_MROUTE
2653                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2654                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2655                         int err = ipmr_get_route(net, skb,
2656                                                  fl4->saddr, fl4->daddr,
2657                                                  r, portid);
2658
2659                         if (err <= 0) {
2660                                 if (err == 0)
2661                                         return 0;
2662                                 goto nla_put_failure;
2663                         }
2664                 } else
2665 #endif
2666                         if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2667                                 goto nla_put_failure;
2668         }
2669
2670         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2671                 goto nla_put_failure;
2672
2673         nlmsg_end(skb, nlh);
2674         return 0;
2675
2676 nla_put_failure:
2677         nlmsg_cancel(skb, nlh);
2678         return -EMSGSIZE;
2679 }
2680
2681 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2682                              struct netlink_ext_ack *extack)
2683 {
2684         struct net *net = sock_net(in_skb->sk);
2685         struct rtmsg *rtm;
2686         struct nlattr *tb[RTA_MAX+1];
2687         struct fib_result res = {};
2688         struct rtable *rt = NULL;
2689         struct flowi4 fl4;
2690         __be32 dst = 0;
2691         __be32 src = 0;
2692         u32 iif;
2693         int err;
2694         int mark;
2695         struct sk_buff *skb;
2696         u32 table_id = RT_TABLE_MAIN;
2697         kuid_t uid;
2698
2699         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy,
2700                           extack);
2701         if (err < 0)
2702                 goto errout;
2703
2704         rtm = nlmsg_data(nlh);
2705
2706         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2707         if (!skb) {
2708                 err = -ENOBUFS;
2709                 goto errout;
2710         }
2711
2712         /* Reserve room for dummy headers, this skb can pass
2713            through good chunk of routing engine.
2714          */
2715         skb_reset_mac_header(skb);
2716         skb_reset_network_header(skb);
2717
2718         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2719         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2720         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2721         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2722         if (tb[RTA_UID])
2723                 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2724         else
2725                 uid = (iif ? INVALID_UID : current_uid());
2726
2727         /* Bugfix: need to give ip_route_input enough of an IP header to
2728          * not gag.
2729          */
2730         ip_hdr(skb)->protocol = IPPROTO_UDP;
2731         ip_hdr(skb)->saddr = src;
2732         ip_hdr(skb)->daddr = dst;
2733
2734         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2735
2736         memset(&fl4, 0, sizeof(fl4));
2737         fl4.daddr = dst;
2738         fl4.saddr = src;
2739         fl4.flowi4_tos = rtm->rtm_tos;
2740         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2741         fl4.flowi4_mark = mark;
2742         fl4.flowi4_uid = uid;
2743
2744         rcu_read_lock();
2745
2746         if (iif) {
2747                 struct net_device *dev;
2748
2749                 dev = dev_get_by_index_rcu(net, iif);
2750                 if (!dev) {
2751                         err = -ENODEV;
2752                         goto errout_free;
2753                 }
2754
2755                 skb->protocol   = htons(ETH_P_IP);
2756                 skb->dev        = dev;
2757                 skb->mark       = mark;
2758                 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2759                                          dev, &res);
2760
2761                 rt = skb_rtable(skb);
2762                 if (err == 0 && rt->dst.error)
2763                         err = -rt->dst.error;
2764         } else {
2765                 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2766                 err = 0;
2767                 if (IS_ERR(rt))
2768                         err = PTR_ERR(rt);
2769                 else
2770                         skb_dst_set(skb, &rt->dst);
2771         }
2772
2773         if (err)
2774                 goto errout_free;
2775
2776         if (rtm->rtm_flags & RTM_F_NOTIFY)
2777                 rt->rt_flags |= RTCF_NOTIFY;
2778
2779         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2780                 table_id = rt->rt_table_id;
2781
2782         if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2783                 if (!res.fi) {
2784                         err = fib_props[res.type].error;
2785                         if (!err)
2786                                 err = -EHOSTUNREACH;
2787                         goto errout_free;
2788                 }
2789                 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2790                                     nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2791                                     rt->rt_type, res.prefix, res.prefixlen,
2792                                     fl4.flowi4_tos, res.fi, 0);
2793         } else {
2794                 err = rt_fill_info(net, dst, src, table_id, &fl4, skb,
2795                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2796         }
2797         if (err < 0)
2798                 goto errout_free;
2799
2800         rcu_read_unlock();
2801
2802         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2803 errout:
2804         return err;
2805
2806 errout_free:
2807         rcu_read_unlock();
2808         kfree_skb(skb);
2809         goto errout;
2810 }
2811
2812 void ip_rt_multicast_event(struct in_device *in_dev)
2813 {
2814         rt_cache_flush(dev_net(in_dev->dev));
2815 }
2816
2817 #ifdef CONFIG_SYSCTL
2818 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2819 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2820 static int ip_rt_gc_elasticity __read_mostly    = 8;
2821
2822 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2823                                         void __user *buffer,
2824                                         size_t *lenp, loff_t *ppos)
2825 {
2826         struct net *net = (struct net *)__ctl->extra1;
2827
2828         if (write) {
2829                 rt_cache_flush(net);
2830                 fnhe_genid_bump(net);
2831                 return 0;
2832         }
2833
2834         return -EINVAL;
2835 }
2836
2837 static struct ctl_table ipv4_route_table[] = {
2838         {
2839                 .procname       = "gc_thresh",
2840                 .data           = &ipv4_dst_ops.gc_thresh,
2841                 .maxlen         = sizeof(int),
2842                 .mode           = 0644,
2843                 .proc_handler   = proc_dointvec,
2844         },
2845         {
2846                 .procname       = "max_size",
2847                 .data           = &ip_rt_max_size,
2848                 .maxlen         = sizeof(int),
2849                 .mode           = 0644,
2850                 .proc_handler   = proc_dointvec,
2851         },
2852         {
2853                 /*  Deprecated. Use gc_min_interval_ms */
2854
2855                 .procname       = "gc_min_interval",
2856                 .data           = &ip_rt_gc_min_interval,
2857                 .maxlen         = sizeof(int),
2858                 .mode           = 0644,
2859                 .proc_handler   = proc_dointvec_jiffies,
2860         },
2861         {
2862                 .procname       = "gc_min_interval_ms",
2863                 .data           = &ip_rt_gc_min_interval,
2864                 .maxlen         = sizeof(int),
2865                 .mode           = 0644,
2866                 .proc_handler   = proc_dointvec_ms_jiffies,
2867         },
2868         {
2869                 .procname       = "gc_timeout",
2870                 .data           = &ip_rt_gc_timeout,
2871                 .maxlen         = sizeof(int),
2872                 .mode           = 0644,
2873                 .proc_handler   = proc_dointvec_jiffies,
2874         },
2875         {
2876                 .procname       = "gc_interval",
2877                 .data           = &ip_rt_gc_interval,
2878                 .maxlen         = sizeof(int),
2879                 .mode           = 0644,
2880                 .proc_handler   = proc_dointvec_jiffies,
2881         },
2882         {
2883                 .procname       = "redirect_load",
2884                 .data           = &ip_rt_redirect_load,
2885                 .maxlen         = sizeof(int),
2886                 .mode           = 0644,
2887                 .proc_handler   = proc_dointvec,
2888         },
2889         {
2890                 .procname       = "redirect_number",
2891                 .data           = &ip_rt_redirect_number,
2892                 .maxlen         = sizeof(int),
2893                 .mode           = 0644,
2894                 .proc_handler   = proc_dointvec,
2895         },
2896         {
2897                 .procname       = "redirect_silence",
2898                 .data           = &ip_rt_redirect_silence,
2899                 .maxlen         = sizeof(int),
2900                 .mode           = 0644,
2901                 .proc_handler   = proc_dointvec,
2902         },
2903         {
2904                 .procname       = "error_cost",
2905                 .data           = &ip_rt_error_cost,
2906                 .maxlen         = sizeof(int),
2907                 .mode           = 0644,
2908                 .proc_handler   = proc_dointvec,
2909         },
2910         {
2911                 .procname       = "error_burst",
2912                 .data           = &ip_rt_error_burst,
2913                 .maxlen         = sizeof(int),
2914                 .mode           = 0644,
2915                 .proc_handler   = proc_dointvec,
2916         },
2917         {
2918                 .procname       = "gc_elasticity",
2919                 .data           = &ip_rt_gc_elasticity,
2920                 .maxlen         = sizeof(int),
2921                 .mode           = 0644,
2922                 .proc_handler   = proc_dointvec,
2923         },
2924         {
2925                 .procname       = "mtu_expires",
2926                 .data           = &ip_rt_mtu_expires,
2927                 .maxlen         = sizeof(int),
2928                 .mode           = 0644,
2929                 .proc_handler   = proc_dointvec_jiffies,
2930         },
2931         {
2932                 .procname       = "min_pmtu",
2933                 .data           = &ip_rt_min_pmtu,
2934                 .maxlen         = sizeof(int),
2935                 .mode           = 0644,
2936                 .proc_handler   = proc_dointvec,
2937         },
2938         {
2939                 .procname       = "min_adv_mss",
2940                 .data           = &ip_rt_min_advmss,
2941                 .maxlen         = sizeof(int),
2942                 .mode           = 0644,
2943                 .proc_handler   = proc_dointvec,
2944         },
2945         { }
2946 };
2947
2948 static struct ctl_table ipv4_route_flush_table[] = {
2949         {
2950                 .procname       = "flush",
2951                 .maxlen         = sizeof(int),
2952                 .mode           = 0200,
2953                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2954         },
2955         { },
2956 };
2957
2958 static __net_init int sysctl_route_net_init(struct net *net)
2959 {
2960         struct ctl_table *tbl;
2961
2962         tbl = ipv4_route_flush_table;
2963         if (!net_eq(net, &init_net)) {
2964                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2965                 if (!tbl)
2966                         goto err_dup;
2967
2968                 /* Don't export sysctls to unprivileged users */
2969                 if (net->user_ns != &init_user_ns)
2970                         tbl[0].procname = NULL;
2971         }
2972         tbl[0].extra1 = net;
2973
2974         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2975         if (!net->ipv4.route_hdr)
2976                 goto err_reg;
2977         return 0;
2978
2979 err_reg:
2980         if (tbl != ipv4_route_flush_table)
2981                 kfree(tbl);
2982 err_dup:
2983         return -ENOMEM;
2984 }
2985
2986 static __net_exit void sysctl_route_net_exit(struct net *net)
2987 {
2988         struct ctl_table *tbl;
2989
2990         tbl = net->ipv4.route_hdr->ctl_table_arg;
2991         unregister_net_sysctl_table(net->ipv4.route_hdr);
2992         BUG_ON(tbl == ipv4_route_flush_table);
2993         kfree(tbl);
2994 }
2995
2996 static __net_initdata struct pernet_operations sysctl_route_ops = {
2997         .init = sysctl_route_net_init,
2998         .exit = sysctl_route_net_exit,
2999 };
3000 #endif
3001
3002 static __net_init int rt_genid_init(struct net *net)
3003 {
3004         atomic_set(&net->ipv4.rt_genid, 0);
3005         atomic_set(&net->fnhe_genid, 0);
3006         atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3007         return 0;
3008 }
3009
3010 static __net_initdata struct pernet_operations rt_genid_ops = {
3011         .init = rt_genid_init,
3012 };
3013
3014 static int __net_init ipv4_inetpeer_init(struct net *net)
3015 {
3016         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3017
3018         if (!bp)
3019                 return -ENOMEM;
3020         inet_peer_base_init(bp);
3021         net->ipv4.peers = bp;
3022         return 0;
3023 }
3024
3025 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3026 {
3027         struct inet_peer_base *bp = net->ipv4.peers;
3028
3029         net->ipv4.peers = NULL;
3030         inetpeer_invalidate_tree(bp);
3031         kfree(bp);
3032 }
3033
3034 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3035         .init   =       ipv4_inetpeer_init,
3036         .exit   =       ipv4_inetpeer_exit,
3037 };
3038
3039 #ifdef CONFIG_IP_ROUTE_CLASSID
3040 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3041 #endif /* CONFIG_IP_ROUTE_CLASSID */
3042
3043 int __init ip_rt_init(void)
3044 {
3045         int cpu;
3046
3047         ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
3048         if (!ip_idents)
3049                 panic("IP: failed to allocate ip_idents\n");
3050
3051         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3052
3053         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3054         if (!ip_tstamps)
3055                 panic("IP: failed to allocate ip_tstamps\n");
3056
3057         for_each_possible_cpu(cpu) {
3058                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3059
3060                 INIT_LIST_HEAD(&ul->head);
3061                 spin_lock_init(&ul->lock);
3062         }
3063 #ifdef CONFIG_IP_ROUTE_CLASSID
3064         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3065         if (!ip_rt_acct)
3066                 panic("IP: failed to allocate ip_rt_acct\n");
3067 #endif
3068
3069         ipv4_dst_ops.kmem_cachep =
3070                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3071                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3072
3073         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3074
3075         if (dst_entries_init(&ipv4_dst_ops) < 0)
3076                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3077
3078         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3079                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3080
3081         ipv4_dst_ops.gc_thresh = ~0;
3082         ip_rt_max_size = INT_MAX;
3083
3084         devinet_init();
3085         ip_fib_init();
3086
3087         if (ip_rt_proc_init())
3088                 pr_err("Unable to create route proc files\n");
3089 #ifdef CONFIG_XFRM
3090         xfrm_init();
3091         xfrm4_init();
3092 #endif
3093         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3094                       RTNL_FLAG_DOIT_UNLOCKED);
3095
3096 #ifdef CONFIG_SYSCTL
3097         register_pernet_subsys(&sysctl_route_ops);
3098 #endif
3099         register_pernet_subsys(&rt_genid_ops);
3100         register_pernet_subsys(&ipv4_inetpeer_ops);
3101         return 0;
3102 }
3103
3104 #ifdef CONFIG_SYSCTL
3105 /*
3106  * We really need to sanitize the damn ipv4 init order, then all
3107  * this nonsense will go away.
3108  */
3109 void __init ip_static_sysctl_init(void)
3110 {
3111         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3112 }
3113 #endif