net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #include <linux/kmemleak.h>
 112 #endif
 113 #include <net/secure_seq.h>
 114 #include <net/ip_tunnels.h>
 115 #include <net/l3mdev.h>
 116
 117 #define RT_FL_TOS(oldflp4) \
 118         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 119
 120 #define RT_GC_TIMEOUT (300*HZ)
 121
 122 static int ip_rt_max_size;
 123 static int ip_rt_redirect_number __read_mostly  = 9;
 124 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 125 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 126 static int ip_rt_error_cost __read_mostly       = HZ;
 127 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 128 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 129 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 130 static int ip_rt_min_advmss __read_mostly       = 256;
 131
 132 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 133 /*
 134  *      Interface to generic destination cache.
 135  */
 136
 137 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 138 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 139 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 140 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 141 static void              ipv4_link_failure(struct sk_buff *skb);
 142 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 143                                            struct sk_buff *skb, u32 mtu);
 144 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 145                                         struct sk_buff *skb);
 146 static void             ipv4_dst_destroy(struct dst_entry *dst);
 147
 148 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 149 {
 150         WARN_ON(1);
 151         return NULL;
 152 }
 153
 154 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 155                                            struct sk_buff *skb,
 156                                            const void *daddr);
 157
 158 static struct dst_ops ipv4_dst_ops = {
 159         .family =               AF_INET,
 160         .check =                ipv4_dst_check,
 161         .default_advmss =       ipv4_default_advmss,
 162         .mtu =                  ipv4_mtu,
 163         .cow_metrics =          ipv4_cow_metrics,
 164         .destroy =              ipv4_dst_destroy,
 165         .negative_advice =      ipv4_negative_advice,
 166         .link_failure =         ipv4_link_failure,
 167         .update_pmtu =          ip_rt_update_pmtu,
 168         .redirect =             ip_do_redirect,
 169         .local_out =            __ip_local_out,
 170         .neigh_lookup =         ipv4_neigh_lookup,
 171 };
 172
 173 #define ECN_OR_COST(class)      TC_PRIO_##class
 174
 175 const __u8 ip_tos2prio[16] = {
 176         TC_PRIO_BESTEFFORT,
 177         ECN_OR_COST(BESTEFFORT),
 178         TC_PRIO_BESTEFFORT,
 179         ECN_OR_COST(BESTEFFORT),
 180         TC_PRIO_BULK,
 181         ECN_OR_COST(BULK),
 182         TC_PRIO_BULK,
 183         ECN_OR_COST(BULK),
 184         TC_PRIO_INTERACTIVE,
 185         ECN_OR_COST(INTERACTIVE),
 186         TC_PRIO_INTERACTIVE,
 187         ECN_OR_COST(INTERACTIVE),
 188         TC_PRIO_INTERACTIVE_BULK,
 189         ECN_OR_COST(INTERACTIVE_BULK),
 190         TC_PRIO_INTERACTIVE_BULK,
 191         ECN_OR_COST(INTERACTIVE_BULK)
 192 };
 193 EXPORT_SYMBOL(ip_tos2prio);
 194
 195 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 196 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 197
 198 #ifdef CONFIG_PROC_FS
 199 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 200 {
 201         if (*pos)
 202                 return NULL;
 203         return SEQ_START_TOKEN;
 204 }
 205
 206 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 207 {
 208         ++*pos;
 209         return NULL;
 210 }
 211
 212 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 213 {
 214 }
 215
 216 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 217 {
 218         if (v == SEQ_START_TOKEN)
 219                 seq_printf(seq, "%-127s\n",
 220                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 221                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 222                            "HHUptod\tSpecDst");
 223         return 0;
 224 }
 225
 226 static const struct seq_operations rt_cache_seq_ops = {
 227         .start  = rt_cache_seq_start,
 228         .next   = rt_cache_seq_next,
 229         .stop   = rt_cache_seq_stop,
 230         .show   = rt_cache_seq_show,
 231 };
 232
 233 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 234 {
 235         return seq_open(file, &rt_cache_seq_ops);
 236 }
 237
 238 static const struct file_operations rt_cache_seq_fops = {
 239         .owner   = THIS_MODULE,
 240         .open    = rt_cache_seq_open,
 241         .read    = seq_read,
 242         .llseek  = seq_lseek,
 243         .release = seq_release,
 244 };
 245
 246
 247 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 248 {
 249         int cpu;
 250
 251         if (*pos == 0)
 252                 return SEQ_START_TOKEN;
 253
 254         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 255                 if (!cpu_possible(cpu))
 256                         continue;
 257                 *pos = cpu+1;
 258                 return &per_cpu(rt_cache_stat, cpu);
 259         }
 260         return NULL;
 261 }
 262
 263 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 264 {
 265         int cpu;
 266
 267         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 268                 if (!cpu_possible(cpu))
 269                         continue;
 270                 *pos = cpu+1;
 271                 return &per_cpu(rt_cache_stat, cpu);
 272         }
 273         return NULL;
 274
 275 }
 276
 277 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 278 {
 279
 280 }
 281
 282 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 283 {
 284         struct rt_cache_stat *st = v;
 285
 286         if (v == SEQ_START_TOKEN) {
 287                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 288                 return 0;
 289         }
 290
 291         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 292                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 293                    dst_entries_get_slow(&ipv4_dst_ops),
 294                    0, /* st->in_hit */
 295                    st->in_slow_tot,
 296                    st->in_slow_mc,
 297                    st->in_no_route,
 298                    st->in_brd,
 299                    st->in_martian_dst,
 300                    st->in_martian_src,
 301
 302                    0, /* st->out_hit */
 303                    st->out_slow_tot,
 304                    st->out_slow_mc,
 305
 306                    0, /* st->gc_total */
 307                    0, /* st->gc_ignored */
 308                    0, /* st->gc_goal_miss */
 309                    0, /* st->gc_dst_overflow */
 310                    0, /* st->in_hlist_search */
 311                    0  /* st->out_hlist_search */
 312                 );
 313         return 0;
 314 }
 315
 316 static const struct seq_operations rt_cpu_seq_ops = {
 317         .start  = rt_cpu_seq_start,
 318         .next   = rt_cpu_seq_next,
 319         .stop   = rt_cpu_seq_stop,
 320         .show   = rt_cpu_seq_show,
 321 };
 322
 323
 324 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 325 {
 326         return seq_open(file, &rt_cpu_seq_ops);
 327 }
 328
 329 static const struct file_operations rt_cpu_seq_fops = {
 330         .owner   = THIS_MODULE,
 331         .open    = rt_cpu_seq_open,
 332         .read    = seq_read,
 333         .llseek  = seq_lseek,
 334         .release = seq_release,
 335 };
 336
 337 #ifdef CONFIG_IP_ROUTE_CLASSID
 338 static int rt_acct_proc_show(struct seq_file *m, void *v)
 339 {
 340         struct ip_rt_acct *dst, *src;
 341         unsigned int i, j;
 342
 343         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 344         if (!dst)
 345                 return -ENOMEM;
 346
 347         for_each_possible_cpu(i) {
 348                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 349                 for (j = 0; j < 256; j++) {
 350                         dst[j].o_bytes   += src[j].o_bytes;
 351                         dst[j].o_packets += src[j].o_packets;
 352                         dst[j].i_bytes   += src[j].i_bytes;
 353                         dst[j].i_packets += src[j].i_packets;
 354                 }
 355         }
 356
 357         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 358         kfree(dst);
 359         return 0;
 360 }
 361
 362 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 363 {
 364         return single_open(file, rt_acct_proc_show, NULL);
 365 }
 366
 367 static const struct file_operations rt_acct_proc_fops = {
 368         .owner          = THIS_MODULE,
 369         .open           = rt_acct_proc_open,
 370         .read           = seq_read,
 371         .llseek         = seq_lseek,
 372         .release        = single_release,
 373 };
 374 #endif
 375
 376 static int __net_init ip_rt_do_proc_init(struct net *net)
 377 {
 378         struct proc_dir_entry *pde;
 379
 380         pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
 381                           &rt_cache_seq_fops);
 382         if (!pde)
 383                 goto err1;
 384
 385         pde = proc_create("rt_cache", S_IRUGO,
 386                           net->proc_net_stat, &rt_cpu_seq_fops);
 387         if (!pde)
 388                 goto err2;
 389
 390 #ifdef CONFIG_IP_ROUTE_CLASSID
 391         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 392         if (!pde)
 393                 goto err3;
 394 #endif
 395         return 0;
 396
 397 #ifdef CONFIG_IP_ROUTE_CLASSID
 398 err3:
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400 #endif
 401 err2:
 402         remove_proc_entry("rt_cache", net->proc_net);
 403 err1:
 404         return -ENOMEM;
 405 }
 406
 407 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 408 {
 409         remove_proc_entry("rt_cache", net->proc_net_stat);
 410         remove_proc_entry("rt_cache", net->proc_net);
 411 #ifdef CONFIG_IP_ROUTE_CLASSID
 412         remove_proc_entry("rt_acct", net->proc_net);
 413 #endif
 414 }
 415
 416 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 417         .init = ip_rt_do_proc_init,
 418         .exit = ip_rt_do_proc_exit,
 419 };
 420
 421 static int __init ip_rt_proc_init(void)
 422 {
 423         return register_pernet_subsys(&ip_rt_proc_ops);
 424 }
 425
 426 #else
 427 static inline int ip_rt_proc_init(void)
 428 {
 429         return 0;
 430 }
 431 #endif /* CONFIG_PROC_FS */
 432
 433 static inline bool rt_is_expired(const struct rtable *rth)
 434 {
 435         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 436 }
 437
 438 void rt_cache_flush(struct net *net)
 439 {
 440         rt_genid_bump_ipv4(net);
 441 }
 442
 443 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 444                                            struct sk_buff *skb,
 445                                            const void *daddr)
 446 {
 447         struct net_device *dev = dst->dev;
 448         const __be32 *pkey = daddr;
 449         const struct rtable *rt;
 450         struct neighbour *n;
 451
 452         rt = (const struct rtable *) dst;
 453         if (rt->rt_gateway)
 454                 pkey = (const __be32 *) &rt->rt_gateway;
 455         else if (skb)
 456                 pkey = &ip_hdr(skb)->daddr;
 457
 458         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 459         if (n)
 460                 return n;
 461         return neigh_create(&arp_tbl, pkey, dev);
 462 }
 463
 464 #define IP_IDENTS_SZ 2048u
 465
 466 static atomic_t *ip_idents __read_mostly;
 467 static u32 *ip_tstamps __read_mostly;
 468
 469 /* In order to protect privacy, we add a perturbation to identifiers
 470  * if one generator is seldom used. This makes hard for an attacker
 471  * to infer how many packets were sent between two points in time.
 472  */
 473 u32 ip_idents_reserve(u32 hash, int segs)
 474 {
 475         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 476         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 477         u32 old = ACCESS_ONCE(*p_tstamp);
 478         u32 now = (u32)jiffies;
 479         u32 delta = 0;
 480
 481         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 482                 delta = prandom_u32_max(now - old);
 483
 484         return atomic_add_return(segs + delta, p_id) - segs;
 485 }
 486 EXPORT_SYMBOL(ip_idents_reserve);
 487
 488 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 489 {
 490         static u32 ip_idents_hashrnd __read_mostly;
 491         u32 hash, id;
 492
 493         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 494
 495         hash = jhash_3words((__force u32)iph->daddr,
 496                             (__force u32)iph->saddr,
 497                             iph->protocol ^ net_hash_mix(net),
 498                             ip_idents_hashrnd);
 499         id = ip_idents_reserve(hash, segs);
 500         iph->id = htons(id);
 501 }
 502 EXPORT_SYMBOL(__ip_select_ident);
 503
 504 static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 505                              const struct iphdr *iph,
 506                              int oif, u8 tos,
 507                              u8 prot, u32 mark, int flow_flags)
 508 {
 509         if (sk) {
 510                 const struct inet_sock *inet = inet_sk(sk);
 511
 512                 oif = sk->sk_bound_dev_if;
 513                 mark = sk->sk_mark;
 514                 tos = RT_CONN_FLAGS(sk);
 515                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 516         }
 517         flowi4_init_output(fl4, oif, mark, tos,
 518                            RT_SCOPE_UNIVERSE, prot,
 519                            flow_flags,
 520                            iph->daddr, iph->saddr, 0, 0);
 521 }
 522
 523 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 524                                const struct sock *sk)
 525 {
 526         const struct iphdr *iph = ip_hdr(skb);
 527         int oif = skb->dev->ifindex;
 528         u8 tos = RT_TOS(iph->tos);
 529         u8 prot = iph->protocol;
 530         u32 mark = skb->mark;
 531
 532         __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
 533 }
 534
 535 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 536 {
 537         const struct inet_sock *inet = inet_sk(sk);
 538         const struct ip_options_rcu *inet_opt;
 539         __be32 daddr = inet->inet_daddr;
 540
 541         rcu_read_lock();
 542         inet_opt = rcu_dereference(inet->inet_opt);
 543         if (inet_opt && inet_opt->opt.srr)
 544                 daddr = inet_opt->opt.faddr;
 545         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 546                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 547                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 548                            inet_sk_flowi_flags(sk),
 549                            daddr, inet->inet_saddr, 0, 0);
 550         rcu_read_unlock();
 551 }
 552
 553 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 554                                  const struct sk_buff *skb)
 555 {
 556         if (skb)
 557                 build_skb_flow_key(fl4, skb, sk);
 558         else
 559                 build_sk_flow_key(fl4, sk);
 560 }
 561
 562 static inline void rt_free(struct rtable *rt)
 563 {
 564         call_rcu(&rt->dst.rcu_head, dst_rcu_free);
 565 }
 566
 567 static DEFINE_SPINLOCK(fnhe_lock);
 568
 569 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 570 {
 571         struct rtable *rt;
 572
 573         rt = rcu_dereference(fnhe->fnhe_rth_input);
 574         if (rt) {
 575                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 576                 rt_free(rt);
 577         }
 578         rt = rcu_dereference(fnhe->fnhe_rth_output);
 579         if (rt) {
 580                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 581                 rt_free(rt);
 582         }
 583 }
 584
 585 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 586 {
 587         struct fib_nh_exception *fnhe, *oldest;
 588
 589         oldest = rcu_dereference(hash->chain);
 590         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 591              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 592                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 593                         oldest = fnhe;
 594         }
 595         fnhe_flush_routes(oldest);
 596         return oldest;
 597 }
 598
 599 static inline u32 fnhe_hashfun(__be32 daddr)
 600 {
 601         static u32 fnhe_hashrnd __read_mostly;
 602         u32 hval;
 603
 604         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 605         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 606         return hash_32(hval, FNHE_HASH_SHIFT);
 607 }
 608
 609 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 610 {
 611         rt->rt_pmtu = fnhe->fnhe_pmtu;
 612         rt->dst.expires = fnhe->fnhe_expires;
 613
 614         if (fnhe->fnhe_gw) {
 615                 rt->rt_flags |= RTCF_REDIRECTED;
 616                 rt->rt_gateway = fnhe->fnhe_gw;
 617                 rt->rt_uses_gateway = 1;
 618         }
 619 }
 620
 621 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 622                                   u32 pmtu, unsigned long expires)
 623 {
 624         struct fnhe_hash_bucket *hash;
 625         struct fib_nh_exception *fnhe;
 626         struct rtable *rt;
 627         unsigned int i;
 628         int depth;
 629         u32 hval = fnhe_hashfun(daddr);
 630
 631         spin_lock_bh(&fnhe_lock);
 632
 633         hash = rcu_dereference(nh->nh_exceptions);
 634         if (!hash) {
 635                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 636                 if (!hash)
 637                         goto out_unlock;
 638                 rcu_assign_pointer(nh->nh_exceptions, hash);
 639         }
 640
 641         hash += hval;
 642
 643         depth = 0;
 644         for (fnhe = rcu_dereference(hash->chain); fnhe;
 645              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 646                 if (fnhe->fnhe_daddr == daddr)
 647                         break;
 648                 depth++;
 649         }
 650
 651         if (fnhe) {
 652                 if (gw)
 653                         fnhe->fnhe_gw = gw;
 654                 if (pmtu) {
 655                         fnhe->fnhe_pmtu = pmtu;
 656                         fnhe->fnhe_expires = max(1UL, expires);
 657                 }
 658                 /* Update all cached dsts too */
 659                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 660                 if (rt)
 661                         fill_route_from_fnhe(rt, fnhe);
 662                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 663                 if (rt)
 664                         fill_route_from_fnhe(rt, fnhe);
 665         } else {
 666                 if (depth > FNHE_RECLAIM_DEPTH)
 667                         fnhe = fnhe_oldest(hash);
 668                 else {
 669                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 670                         if (!fnhe)
 671                                 goto out_unlock;
 672
 673                         fnhe->fnhe_next = hash->chain;
 674                         rcu_assign_pointer(hash->chain, fnhe);
 675                 }
 676                 fnhe->fnhe_genid = fnhe_genid(dev_net(nh->nh_dev));
 677                 fnhe->fnhe_daddr = daddr;
 678                 fnhe->fnhe_gw = gw;
 679                 fnhe->fnhe_pmtu = pmtu;
 680                 fnhe->fnhe_expires = expires;
 681
 682                 /* Exception created; mark the cached routes for the nexthop
 683                  * stale, so anyone caching it rechecks if this exception
 684                  * applies to them.
 685                  */
 686                 rt = rcu_dereference(nh->nh_rth_input);
 687                 if (rt)
 688                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 689
 690                 for_each_possible_cpu(i) {
 691                         struct rtable __rcu **prt;
 692                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 693                         rt = rcu_dereference(*prt);
 694                         if (rt)
 695                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 696                 }
 697         }
 698
 699         fnhe->fnhe_stamp = jiffies;
 700
 701 out_unlock:
 702         spin_unlock_bh(&fnhe_lock);
 703 }
 704
 705 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 706                              bool kill_route)
 707 {
 708         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 709         __be32 old_gw = ip_hdr(skb)->saddr;
 710         struct net_device *dev = skb->dev;
 711         struct in_device *in_dev;
 712         struct fib_result res;
 713         struct neighbour *n;
 714         struct net *net;
 715
 716         switch (icmp_hdr(skb)->code & 7) {
 717         case ICMP_REDIR_NET:
 718         case ICMP_REDIR_NETTOS:
 719         case ICMP_REDIR_HOST:
 720         case ICMP_REDIR_HOSTTOS:
 721                 break;
 722
 723         default:
 724                 return;
 725         }
 726
 727         if (rt->rt_gateway != old_gw)
 728                 return;
 729
 730         in_dev = __in_dev_get_rcu(dev);
 731         if (!in_dev)
 732                 return;
 733
 734         net = dev_net(dev);
 735         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 736             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 737             ipv4_is_zeronet(new_gw))
 738                 goto reject_redirect;
 739
 740         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 741                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 742                         goto reject_redirect;
 743                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 744                         goto reject_redirect;
 745         } else {
 746                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 747                         goto reject_redirect;
 748         }
 749
 750         n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
 751         if (!IS_ERR(n)) {
 752                 if (!(n->nud_state & NUD_VALID)) {
 753                         neigh_event_send(n, NULL);
 754                 } else {
 755                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 756                                 struct fib_nh *nh = &FIB_RES_NH(res);
 757
 758                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 759                                                 0, jiffies + ip_rt_gc_timeout);
 760                         }
 761                         if (kill_route)
 762                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 763                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 764                 }
 765                 neigh_release(n);
 766         }
 767         return;
 768
 769 reject_redirect:
 770 #ifdef CONFIG_IP_ROUTE_VERBOSE
 771         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 772                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 773                 __be32 daddr = iph->daddr;
 774                 __be32 saddr = iph->saddr;
 775
 776                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 777                                      "  Advised path = %pI4 -> %pI4\n",
 778                                      &old_gw, dev->name, &new_gw,
 779                                      &saddr, &daddr);
 780         }
 781 #endif
 782         ;
 783 }
 784
 785 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 786 {
 787         struct rtable *rt;
 788         struct flowi4 fl4;
 789         const struct iphdr *iph = (const struct iphdr *) skb->data;
 790         int oif = skb->dev->ifindex;
 791         u8 tos = RT_TOS(iph->tos);
 792         u8 prot = iph->protocol;
 793         u32 mark = skb->mark;
 794
 795         rt = (struct rtable *) dst;
 796
 797         __build_flow_key(&fl4, sk, iph, oif, tos, prot, mark, 0);
 798         __ip_do_redirect(rt, skb, &fl4, true);
 799 }
 800
 801 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 802 {
 803         struct rtable *rt = (struct rtable *)dst;
 804         struct dst_entry *ret = dst;
 805
 806         if (rt) {
 807                 if (dst->obsolete > 0) {
 808                         ip_rt_put(rt);
 809                         ret = NULL;
 810                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 811                            rt->dst.expires) {
 812                         ip_rt_put(rt);
 813                         ret = NULL;
 814                 }
 815         }
 816         return ret;
 817 }
 818
 819 /*
 820  * Algorithm:
 821  *      1. The first ip_rt_redirect_number redirects are sent
 822  *         with exponential backoff, then we stop sending them at all,
 823  *         assuming that the host ignores our redirects.
 824  *      2. If we did not see packets requiring redirects
 825  *         during ip_rt_redirect_silence, we assume that the host
 826  *         forgot redirected route and start to send redirects again.
 827  *
 828  * This algorithm is much cheaper and more intelligent than dumb load limiting
 829  * in icmp.c.
 830  *
 831  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 832  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 833  */
 834
 835 void ip_rt_send_redirect(struct sk_buff *skb)
 836 {
 837         struct rtable *rt = skb_rtable(skb);
 838         struct in_device *in_dev;
 839         struct inet_peer *peer;
 840         struct net *net;
 841         int log_martians;
 842         int vif;
 843
 844         rcu_read_lock();
 845         in_dev = __in_dev_get_rcu(rt->dst.dev);
 846         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 847                 rcu_read_unlock();
 848                 return;
 849         }
 850         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 851         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 852         rcu_read_unlock();
 853
 854         net = dev_net(rt->dst.dev);
 855         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 856         if (!peer) {
 857                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 858                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 859                 return;
 860         }
 861
 862         /* No redirected packets during ip_rt_redirect_silence;
 863          * reset the algorithm.
 864          */
 865         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 866                 peer->rate_tokens = 0;
 867
 868         /* Too many ignored redirects; do not send anything
 869          * set dst.rate_last to the last seen redirected packet.
 870          */
 871         if (peer->rate_tokens >= ip_rt_redirect_number) {
 872                 peer->rate_last = jiffies;
 873                 goto out_put_peer;
 874         }
 875
 876         /* Check for load limit; set rate_last to the latest sent
 877          * redirect.
 878          */
 879         if (peer->rate_tokens == 0 ||
 880             time_after(jiffies,
 881                        (peer->rate_last +
 882                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 883                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 884
 885                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 886                 peer->rate_last = jiffies;
 887                 ++peer->rate_tokens;
 888 #ifdef CONFIG_IP_ROUTE_VERBOSE
 889                 if (log_martians &&
 890                     peer->rate_tokens == ip_rt_redirect_number)
 891                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 892                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 893                                              &ip_hdr(skb)->daddr, &gw);
 894 #endif
 895         }
 896 out_put_peer:
 897         inet_putpeer(peer);
 898 }
 899
 900 static int ip_error(struct sk_buff *skb)
 901 {
 902         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 903         struct rtable *rt = skb_rtable(skb);
 904         struct inet_peer *peer;
 905         unsigned long now;
 906         struct net *net;
 907         bool send;
 908         int code;
 909
 910         /* IP on this device is disabled. */
 911         if (!in_dev)
 912                 goto out;
 913
 914         net = dev_net(rt->dst.dev);
 915         if (!IN_DEV_FORWARD(in_dev)) {
 916                 switch (rt->dst.error) {
 917                 case EHOSTUNREACH:
 918                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 919                         break;
 920
 921                 case ENETUNREACH:
 922                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 923                         break;
 924                 }
 925                 goto out;
 926         }
 927
 928         switch (rt->dst.error) {
 929         case EINVAL:
 930         default:
 931                 goto out;
 932         case EHOSTUNREACH:
 933                 code = ICMP_HOST_UNREACH;
 934                 break;
 935         case ENETUNREACH:
 936                 code = ICMP_NET_UNREACH;
 937                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 938                 break;
 939         case EACCES:
 940                 code = ICMP_PKT_FILTERED;
 941                 break;
 942         }
 943
 944         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 945                                l3mdev_master_ifindex(skb->dev), 1);
 946
 947         send = true;
 948         if (peer) {
 949                 now = jiffies;
 950                 peer->rate_tokens += now - peer->rate_last;
 951                 if (peer->rate_tokens > ip_rt_error_burst)
 952                         peer->rate_tokens = ip_rt_error_burst;
 953                 peer->rate_last = now;
 954                 if (peer->rate_tokens >= ip_rt_error_cost)
 955                         peer->rate_tokens -= ip_rt_error_cost;
 956                 else
 957                         send = false;
 958                 inet_putpeer(peer);
 959         }
 960         if (send)
 961                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 962
 963 out:    kfree_skb(skb);
 964         return 0;
 965 }
 966
 967 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 968 {
 969         struct dst_entry *dst = &rt->dst;
 970         struct fib_result res;
 971
 972         if (dst_metric_locked(dst, RTAX_MTU))
 973                 return;
 974
 975         if (ipv4_mtu(dst) < mtu)
 976                 return;
 977
 978         if (mtu < ip_rt_min_pmtu)
 979                 mtu = ip_rt_min_pmtu;
 980
 981         if (rt->rt_pmtu == mtu &&
 982             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
 983                 return;
 984
 985         rcu_read_lock();
 986         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
 987                 struct fib_nh *nh = &FIB_RES_NH(res);
 988
 989                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
 990                                       jiffies + ip_rt_mtu_expires);
 991         }
 992         rcu_read_unlock();
 993 }
 994
 995 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 996                               struct sk_buff *skb, u32 mtu)
 997 {
 998         struct rtable *rt = (struct rtable *) dst;
 999         struct flowi4 fl4;
1000
1001         ip_rt_build_flow_key(&fl4, sk, skb);
1002         __ip_rt_update_pmtu(rt, &fl4, mtu);
1003 }
1004
1005 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1006                       int oif, u32 mark, u8 protocol, int flow_flags)
1007 {
1008         const struct iphdr *iph = (const struct iphdr *) skb->data;
1009         struct flowi4 fl4;
1010         struct rtable *rt;
1011
1012         if (!mark)
1013                 mark = IP4_REPLY_MARK(net, skb->mark);
1014
1015         __build_flow_key(&fl4, NULL, iph, oif,
1016                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1017         rt = __ip_route_output_key(net, &fl4);
1018         if (!IS_ERR(rt)) {
1019                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1020                 ip_rt_put(rt);
1021         }
1022 }
1023 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1024
1025 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1026 {
1027         const struct iphdr *iph = (const struct iphdr *) skb->data;
1028         struct flowi4 fl4;
1029         struct rtable *rt;
1030
1031         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1032
1033         if (!fl4.flowi4_mark)
1034                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1035
1036         rt = __ip_route_output_key(sock_net(sk), &fl4);
1037         if (!IS_ERR(rt)) {
1038                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1039                 ip_rt_put(rt);
1040         }
1041 }
1042
1043 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1044 {
1045         const struct iphdr *iph = (const struct iphdr *) skb->data;
1046         struct flowi4 fl4;
1047         struct rtable *rt;
1048         struct dst_entry *odst = NULL;
1049         bool new = false;
1050
1051         bh_lock_sock(sk);
1052
1053         if (!ip_sk_accept_pmtu(sk))
1054                 goto out;
1055
1056         odst = sk_dst_get(sk);
1057
1058         if (sock_owned_by_user(sk) || !odst) {
1059                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1060                 goto out;
1061         }
1062
1063         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1064
1065         rt = (struct rtable *)odst;
1066         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1067                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1068                 if (IS_ERR(rt))
1069                         goto out;
1070
1071                 new = true;
1072         }
1073
1074         __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
1075
1076         if (!dst_check(&rt->dst, 0)) {
1077                 if (new)
1078                         dst_release(&rt->dst);
1079
1080                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1081                 if (IS_ERR(rt))
1082                         goto out;
1083
1084                 new = true;
1085         }
1086
1087         if (new)
1088                 sk_dst_set(sk, &rt->dst);
1089
1090 out:
1091         bh_unlock_sock(sk);
1092         dst_release(odst);
1093 }
1094 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1095
1096 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1097                    int oif, u32 mark, u8 protocol, int flow_flags)
1098 {
1099         const struct iphdr *iph = (const struct iphdr *) skb->data;
1100         struct flowi4 fl4;
1101         struct rtable *rt;
1102
1103         __build_flow_key(&fl4, NULL, iph, oif,
1104                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1105         rt = __ip_route_output_key(net, &fl4);
1106         if (!IS_ERR(rt)) {
1107                 __ip_do_redirect(rt, skb, &fl4, false);
1108                 ip_rt_put(rt);
1109         }
1110 }
1111 EXPORT_SYMBOL_GPL(ipv4_redirect);
1112
1113 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1114 {
1115         const struct iphdr *iph = (const struct iphdr *) skb->data;
1116         struct flowi4 fl4;
1117         struct rtable *rt;
1118
1119         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1120         rt = __ip_route_output_key(sock_net(sk), &fl4);
1121         if (!IS_ERR(rt)) {
1122                 __ip_do_redirect(rt, skb, &fl4, false);
1123                 ip_rt_put(rt);
1124         }
1125 }
1126 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1127
1128 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1129 {
1130         struct rtable *rt = (struct rtable *) dst;
1131
1132         /* All IPV4 dsts are created with ->obsolete set to the value
1133          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1134          * into this function always.
1135          *
1136          * When a PMTU/redirect information update invalidates a route,
1137          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1138          * DST_OBSOLETE_DEAD by dst_free().
1139          */
1140         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1141                 return NULL;
1142         return dst;
1143 }
1144
1145 static void ipv4_link_failure(struct sk_buff *skb)
1146 {
1147         struct rtable *rt;
1148
1149         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1150
1151         rt = skb_rtable(skb);
1152         if (rt)
1153                 dst_set_expires(&rt->dst, 0);
1154 }
1155
1156 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1157 {
1158         pr_debug("%s: %pI4 -> %pI4, %s\n",
1159                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1160                  skb->dev ? skb->dev->name : "?");
1161         kfree_skb(skb);
1162         WARN_ON(1);
1163         return 0;
1164 }
1165
1166 /*
1167    We do not cache source address of outgoing interface,
1168    because it is used only by IP RR, TS and SRR options,
1169    so that it out of fast path.
1170
1171    BTW remember: "addr" is allowed to be not aligned
1172    in IP options!
1173  */
1174
1175 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1176 {
1177         __be32 src;
1178
1179         if (rt_is_output_route(rt))
1180                 src = ip_hdr(skb)->saddr;
1181         else {
1182                 struct fib_result res;
1183                 struct flowi4 fl4;
1184                 struct iphdr *iph;
1185
1186                 iph = ip_hdr(skb);
1187
1188                 memset(&fl4, 0, sizeof(fl4));
1189                 fl4.daddr = iph->daddr;
1190                 fl4.saddr = iph->saddr;
1191                 fl4.flowi4_tos = RT_TOS(iph->tos);
1192                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1193                 fl4.flowi4_iif = skb->dev->ifindex;
1194                 fl4.flowi4_mark = skb->mark;
1195
1196                 rcu_read_lock();
1197                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1198                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1199                 else
1200                         src = inet_select_addr(rt->dst.dev,
1201                                                rt_nexthop(rt, iph->daddr),
1202                                                RT_SCOPE_UNIVERSE);
1203                 rcu_read_unlock();
1204         }
1205         memcpy(addr, &src, 4);
1206 }
1207
1208 #ifdef CONFIG_IP_ROUTE_CLASSID
1209 static void set_class_tag(struct rtable *rt, u32 tag)
1210 {
1211         if (!(rt->dst.tclassid & 0xFFFF))
1212                 rt->dst.tclassid |= tag & 0xFFFF;
1213         if (!(rt->dst.tclassid & 0xFFFF0000))
1214                 rt->dst.tclassid |= tag & 0xFFFF0000;
1215 }
1216 #endif
1217
1218 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1219 {
1220         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1221
1222         if (advmss == 0) {
1223                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1224                                ip_rt_min_advmss);
1225                 if (advmss > 65535 - 40)
1226                         advmss = 65535 - 40;
1227         }
1228         return advmss;
1229 }
1230
1231 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1232 {
1233         const struct rtable *rt = (const struct rtable *) dst;
1234         unsigned int mtu = rt->rt_pmtu;
1235
1236         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1237                 mtu = dst_metric_raw(dst, RTAX_MTU);
1238
1239         if (mtu)
1240                 return mtu;
1241
1242         mtu = dst->dev->mtu;
1243
1244         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1245                 if (rt->rt_uses_gateway && mtu > 576)
1246                         mtu = 576;
1247         }
1248
1249         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1250
1251         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1252 }
1253
1254 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1255 {
1256         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1257         struct fib_nh_exception *fnhe;
1258         u32 hval;
1259
1260         if (!hash)
1261                 return NULL;
1262
1263         hval = fnhe_hashfun(daddr);
1264
1265         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1266              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1267                 if (fnhe->fnhe_daddr == daddr)
1268                         return fnhe;
1269         }
1270         return NULL;
1271 }
1272
1273 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1274                               __be32 daddr)
1275 {
1276         bool ret = false;
1277
1278         spin_lock_bh(&fnhe_lock);
1279
1280         if (daddr == fnhe->fnhe_daddr) {
1281                 struct rtable __rcu **porig;
1282                 struct rtable *orig;
1283                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1284
1285                 if (rt_is_input_route(rt))
1286                         porig = &fnhe->fnhe_rth_input;
1287                 else
1288                         porig = &fnhe->fnhe_rth_output;
1289                 orig = rcu_dereference(*porig);
1290
1291                 if (fnhe->fnhe_genid != genid) {
1292                         fnhe->fnhe_genid = genid;
1293                         fnhe->fnhe_gw = 0;
1294                         fnhe->fnhe_pmtu = 0;
1295                         fnhe->fnhe_expires = 0;
1296                         fnhe_flush_routes(fnhe);
1297                         orig = NULL;
1298                 }
1299                 fill_route_from_fnhe(rt, fnhe);
1300                 if (!rt->rt_gateway)
1301                         rt->rt_gateway = daddr;
1302
1303                 if (!(rt->dst.flags & DST_NOCACHE)) {
1304                         rcu_assign_pointer(*porig, rt);
1305                         if (orig)
1306                                 rt_free(orig);
1307                         ret = true;
1308                 }
1309
1310                 fnhe->fnhe_stamp = jiffies;
1311         }
1312         spin_unlock_bh(&fnhe_lock);
1313
1314         return ret;
1315 }
1316
1317 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1318 {
1319         struct rtable *orig, *prev, **p;
1320         bool ret = true;
1321
1322         if (rt_is_input_route(rt)) {
1323                 p = (struct rtable **)&nh->nh_rth_input;
1324         } else {
1325                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1326         }
1327         orig = *p;
1328
1329         prev = cmpxchg(p, orig, rt);
1330         if (prev == orig) {
1331                 if (orig)
1332                         rt_free(orig);
1333         } else
1334                 ret = false;
1335
1336         return ret;
1337 }
1338
1339 struct uncached_list {
1340         spinlock_t              lock;
1341         struct list_head        head;
1342 };
1343
1344 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1345
1346 static void rt_add_uncached_list(struct rtable *rt)
1347 {
1348         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1349
1350         rt->rt_uncached_list = ul;
1351
1352         spin_lock_bh(&ul->lock);
1353         list_add_tail(&rt->rt_uncached, &ul->head);
1354         spin_unlock_bh(&ul->lock);
1355 }
1356
1357 static void ipv4_dst_destroy(struct dst_entry *dst)
1358 {
1359         struct rtable *rt = (struct rtable *) dst;
1360
1361         if (!list_empty(&rt->rt_uncached)) {
1362                 struct uncached_list *ul = rt->rt_uncached_list;
1363
1364                 spin_lock_bh(&ul->lock);
1365                 list_del(&rt->rt_uncached);
1366                 spin_unlock_bh(&ul->lock);
1367         }
1368 }
1369
1370 void rt_flush_dev(struct net_device *dev)
1371 {
1372         struct net *net = dev_net(dev);
1373         struct rtable *rt;
1374         int cpu;
1375
1376         for_each_possible_cpu(cpu) {
1377                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1378
1379                 spin_lock_bh(&ul->lock);
1380                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1381                         if (rt->dst.dev != dev)
1382                                 continue;
1383                         rt->dst.dev = net->loopback_dev;
1384                         dev_hold(rt->dst.dev);
1385                         dev_put(dev);
1386                 }
1387                 spin_unlock_bh(&ul->lock);
1388         }
1389 }
1390
1391 static bool rt_cache_valid(const struct rtable *rt)
1392 {
1393         return  rt &&
1394                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1395                 !rt_is_expired(rt);
1396 }
1397
1398 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1399                            const struct fib_result *res,
1400                            struct fib_nh_exception *fnhe,
1401                            struct fib_info *fi, u16 type, u32 itag)
1402 {
1403         bool cached = false;
1404
1405         if (fi) {
1406                 struct fib_nh *nh = &FIB_RES_NH(*res);
1407
1408                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1409                         rt->rt_gateway = nh->nh_gw;
1410                         rt->rt_uses_gateway = 1;
1411                 }
1412                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1413 #ifdef CONFIG_IP_ROUTE_CLASSID
1414                 rt->dst.tclassid = nh->nh_tclassid;
1415 #endif
1416                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1417                 if (unlikely(fnhe))
1418                         cached = rt_bind_exception(rt, fnhe, daddr);
1419                 else if (!(rt->dst.flags & DST_NOCACHE))
1420                         cached = rt_cache_route(nh, rt);
1421                 if (unlikely(!cached)) {
1422                         /* Routes we intend to cache in nexthop exception or
1423                          * FIB nexthop have the DST_NOCACHE bit clear.
1424                          * However, if we are unsuccessful at storing this
1425                          * route into the cache we really need to set it.
1426                          */
1427                         rt->dst.flags |= DST_NOCACHE;
1428                         if (!rt->rt_gateway)
1429                                 rt->rt_gateway = daddr;
1430                         rt_add_uncached_list(rt);
1431                 }
1432         } else
1433                 rt_add_uncached_list(rt);
1434
1435 #ifdef CONFIG_IP_ROUTE_CLASSID
1436 #ifdef CONFIG_IP_MULTIPLE_TABLES
1437         set_class_tag(rt, res->tclassid);
1438 #endif
1439         set_class_tag(rt, itag);
1440 #endif
1441 }
1442
1443 struct rtable *rt_dst_alloc(struct net_device *dev,
1444                             unsigned int flags, u16 type,
1445                             bool nopolicy, bool noxfrm, bool will_cache)
1446 {
1447         struct rtable *rt;
1448
1449         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1450                        (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
1451                        (nopolicy ? DST_NOPOLICY : 0) |
1452                        (noxfrm ? DST_NOXFRM : 0));
1453
1454         if (rt) {
1455                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1456                 rt->rt_flags = flags;
1457                 rt->rt_type = type;
1458                 rt->rt_is_input = 0;
1459                 rt->rt_iif = 0;
1460                 rt->rt_pmtu = 0;
1461                 rt->rt_gateway = 0;
1462                 rt->rt_uses_gateway = 0;
1463                 rt->rt_table_id = 0;
1464                 INIT_LIST_HEAD(&rt->rt_uncached);
1465
1466                 rt->dst.output = ip_output;
1467                 if (flags & RTCF_LOCAL)
1468                         rt->dst.input = ip_local_deliver;
1469         }
1470
1471         return rt;
1472 }
1473 EXPORT_SYMBOL(rt_dst_alloc);
1474
1475 /* called in rcu_read_lock() section */
1476 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1477                                 u8 tos, struct net_device *dev, int our)
1478 {
1479         struct rtable *rth;
1480         struct in_device *in_dev = __in_dev_get_rcu(dev);
1481         unsigned int flags = RTCF_MULTICAST;
1482         u32 itag = 0;
1483         int err;
1484
1485         /* Primary sanity checks. */
1486
1487         if (!in_dev)
1488                 return -EINVAL;
1489
1490         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1491             skb->protocol != htons(ETH_P_IP))
1492                 goto e_inval;
1493
1494         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1495                 goto e_inval;
1496
1497         if (ipv4_is_zeronet(saddr)) {
1498                 if (!ipv4_is_local_multicast(daddr))
1499                         goto e_inval;
1500         } else {
1501                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1502                                           in_dev, &itag);
1503                 if (err < 0)
1504                         goto e_err;
1505         }
1506         if (our)
1507                 flags |= RTCF_LOCAL;
1508
1509         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1510                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1511         if (!rth)
1512                 goto e_nobufs;
1513
1514 #ifdef CONFIG_IP_ROUTE_CLASSID
1515         rth->dst.tclassid = itag;
1516 #endif
1517         rth->dst.output = ip_rt_bug;
1518         rth->rt_is_input= 1;
1519
1520 #ifdef CONFIG_IP_MROUTE
1521         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1522                 rth->dst.input = ip_mr_input;
1523 #endif
1524         RT_CACHE_STAT_INC(in_slow_mc);
1525
1526         skb_dst_set(skb, &rth->dst);
1527         return 0;
1528
1529 e_nobufs:
1530         return -ENOBUFS;
1531 e_inval:
1532         return -EINVAL;
1533 e_err:
1534         return err;
1535 }
1536
1537
1538 static void ip_handle_martian_source(struct net_device *dev,
1539                                      struct in_device *in_dev,
1540                                      struct sk_buff *skb,
1541                                      __be32 daddr,
1542                                      __be32 saddr)
1543 {
1544         RT_CACHE_STAT_INC(in_martian_src);
1545 #ifdef CONFIG_IP_ROUTE_VERBOSE
1546         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1547                 /*
1548                  *      RFC1812 recommendation, if source is martian,
1549                  *      the only hint is MAC header.
1550                  */
1551                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1552                         &daddr, &saddr, dev->name);
1553                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1554                         print_hex_dump(KERN_WARNING, "ll header: ",
1555                                        DUMP_PREFIX_OFFSET, 16, 1,
1556                                        skb_mac_header(skb),
1557                                        dev->hard_header_len, true);
1558                 }
1559         }
1560 #endif
1561 }
1562
1563 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1564 {
1565         struct fnhe_hash_bucket *hash;
1566         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1567         u32 hval = fnhe_hashfun(daddr);
1568
1569         spin_lock_bh(&fnhe_lock);
1570
1571         hash = rcu_dereference_protected(nh->nh_exceptions,
1572                                          lockdep_is_held(&fnhe_lock));
1573         hash += hval;
1574
1575         fnhe_p = &hash->chain;
1576         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1577         while (fnhe) {
1578                 if (fnhe->fnhe_daddr == daddr) {
1579                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1580                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1581                         fnhe_flush_routes(fnhe);
1582                         kfree_rcu(fnhe, rcu);
1583                         break;
1584                 }
1585                 fnhe_p = &fnhe->fnhe_next;
1586                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1587                                                  lockdep_is_held(&fnhe_lock));
1588         }
1589
1590         spin_unlock_bh(&fnhe_lock);
1591 }
1592
1593 /* called in rcu_read_lock() section */
1594 static int __mkroute_input(struct sk_buff *skb,
1595                            const struct fib_result *res,
1596                            struct in_device *in_dev,
1597                            __be32 daddr, __be32 saddr, u32 tos)
1598 {
1599         struct fib_nh_exception *fnhe;
1600         struct rtable *rth;
1601         int err;
1602         struct in_device *out_dev;
1603         bool do_cache;
1604         u32 itag = 0;
1605
1606         /* get a working reference to the output device */
1607         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1608         if (!out_dev) {
1609                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1610                 return -EINVAL;
1611         }
1612
1613         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1614                                   in_dev->dev, in_dev, &itag);
1615         if (err < 0) {
1616                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1617                                          saddr);
1618
1619                 goto cleanup;
1620         }
1621
1622         do_cache = res->fi && !itag;
1623         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1624             skb->protocol == htons(ETH_P_IP) &&
1625             (IN_DEV_SHARED_MEDIA(out_dev) ||
1626              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1627                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1628
1629         if (skb->protocol != htons(ETH_P_IP)) {
1630                 /* Not IP (i.e. ARP). Do not create route, if it is
1631                  * invalid for proxy arp. DNAT routes are always valid.
1632                  *
1633                  * Proxy arp feature have been extended to allow, ARP
1634                  * replies back to the same interface, to support
1635                  * Private VLAN switch technologies. See arp.c.
1636                  */
1637                 if (out_dev == in_dev &&
1638                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1639                         err = -EINVAL;
1640                         goto cleanup;
1641                 }
1642         }
1643
1644         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1645         if (do_cache) {
1646                 if (fnhe) {
1647                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1648                         if (rth && rth->dst.expires &&
1649                             time_after(jiffies, rth->dst.expires)) {
1650                                 ip_del_fnhe(&FIB_RES_NH(*res), daddr);
1651                                 fnhe = NULL;
1652                         } else {
1653                                 goto rt_cache;
1654                         }
1655                 }
1656
1657                 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1658
1659 rt_cache:
1660                 if (rt_cache_valid(rth)) {
1661                         skb_dst_set_noref(skb, &rth->dst);
1662                         goto out;
1663                 }
1664         }
1665
1666         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1667                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1668                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1669         if (!rth) {
1670                 err = -ENOBUFS;
1671                 goto cleanup;
1672         }
1673
1674         rth->rt_is_input = 1;
1675         if (res->table)
1676                 rth->rt_table_id = res->table->tb_id;
1677         RT_CACHE_STAT_INC(in_slow_tot);
1678
1679         rth->dst.input = ip_forward;
1680
1681         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag);
1682         if (lwtunnel_output_redirect(rth->dst.lwtstate)) {
1683                 rth->dst.lwtstate->orig_output = rth->dst.output;
1684                 rth->dst.output = lwtunnel_output;
1685         }
1686         if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
1687                 rth->dst.lwtstate->orig_input = rth->dst.input;
1688                 rth->dst.input = lwtunnel_input;
1689         }
1690         skb_dst_set(skb, &rth->dst);
1691 out:
1692         err = 0;
1693  cleanup:
1694         return err;
1695 }
1696
1697 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1698
1699 /* To make ICMP packets follow the right flow, the multipath hash is
1700  * calculated from the inner IP addresses in reverse order.
1701  */
1702 static int ip_multipath_icmp_hash(struct sk_buff *skb)
1703 {
1704         const struct iphdr *outer_iph = ip_hdr(skb);
1705         struct icmphdr _icmph;
1706         const struct icmphdr *icmph;
1707         struct iphdr _inner_iph;
1708         const struct iphdr *inner_iph;
1709
1710         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1711                 goto standard_hash;
1712
1713         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1714                                    &_icmph);
1715         if (!icmph)
1716                 goto standard_hash;
1717
1718         if (icmph->type != ICMP_DEST_UNREACH &&
1719             icmph->type != ICMP_REDIRECT &&
1720             icmph->type != ICMP_TIME_EXCEEDED &&
1721             icmph->type != ICMP_PARAMETERPROB) {
1722                 goto standard_hash;
1723         }
1724
1725         inner_iph = skb_header_pointer(skb,
1726                                        outer_iph->ihl * 4 + sizeof(_icmph),
1727                                        sizeof(_inner_iph), &_inner_iph);
1728         if (!inner_iph)
1729                 goto standard_hash;
1730
1731         return fib_multipath_hash(inner_iph->daddr, inner_iph->saddr);
1732
1733 standard_hash:
1734         return fib_multipath_hash(outer_iph->saddr, outer_iph->daddr);
1735 }
1736
1737 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1738
1739 static int ip_mkroute_input(struct sk_buff *skb,
1740                             struct fib_result *res,
1741                             const struct flowi4 *fl4,
1742                             struct in_device *in_dev,
1743                             __be32 daddr, __be32 saddr, u32 tos)
1744 {
1745 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1746         if (res->fi && res->fi->fib_nhs > 1) {
1747                 int h;
1748
1749                 if (unlikely(ip_hdr(skb)->protocol == IPPROTO_ICMP))
1750                         h = ip_multipath_icmp_hash(skb);
1751                 else
1752                         h = fib_multipath_hash(saddr, daddr);
1753                 fib_select_multipath(res, h);
1754         }
1755 #endif
1756
1757         /* create a routing cache entry */
1758         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1759 }
1760
1761 /*
1762  *      NOTE. We drop all the packets that has local source
1763  *      addresses, because every properly looped back packet
1764  *      must have correct destination already attached by output routine.
1765  *
1766  *      Such approach solves two big problems:
1767  *      1. Not simplex devices are handled properly.
1768  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1769  *      called with rcu_read_lock()
1770  */
1771
1772 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1773                                u8 tos, struct net_device *dev)
1774 {
1775         struct fib_result res;
1776         struct in_device *in_dev = __in_dev_get_rcu(dev);
1777         struct ip_tunnel_info *tun_info;
1778         struct flowi4   fl4;
1779         unsigned int    flags = 0;
1780         u32             itag = 0;
1781         struct rtable   *rth;
1782         int             err = -EINVAL;
1783         struct net    *net = dev_net(dev);
1784         bool do_cache;
1785
1786         /* IP on this device is disabled. */
1787
1788         if (!in_dev)
1789                 goto out;
1790
1791         /* Check for the most weird martians, which can be not detected
1792            by fib_lookup.
1793          */
1794
1795         tun_info = skb_tunnel_info(skb);
1796         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1797                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1798         else
1799                 fl4.flowi4_tun_key.tun_id = 0;
1800         skb_dst_drop(skb);
1801
1802         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1803                 goto martian_source;
1804
1805         res.fi = NULL;
1806         res.table = NULL;
1807         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1808                 goto brd_input;
1809
1810         /* Accept zero addresses only to limited broadcast;
1811          * I even do not know to fix it or not. Waiting for complains :-)
1812          */
1813         if (ipv4_is_zeronet(saddr))
1814                 goto martian_source;
1815
1816         if (ipv4_is_zeronet(daddr))
1817                 goto martian_destination;
1818
1819         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1820          * and call it once if daddr or/and saddr are loopback addresses
1821          */
1822         if (ipv4_is_loopback(daddr)) {
1823                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1824                         goto martian_destination;
1825         } else if (ipv4_is_loopback(saddr)) {
1826                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1827                         goto martian_source;
1828         }
1829
1830         /*
1831          *      Now we are ready to route packet.
1832          */
1833         fl4.flowi4_oif = 0;
1834         fl4.flowi4_iif = l3mdev_fib_oif_rcu(dev);
1835         fl4.flowi4_mark = skb->mark;
1836         fl4.flowi4_tos = tos;
1837         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1838         fl4.flowi4_flags = 0;
1839         fl4.daddr = daddr;
1840         fl4.saddr = saddr;
1841         err = fib_lookup(net, &fl4, &res, 0);
1842         if (err != 0) {
1843                 if (!IN_DEV_FORWARD(in_dev))
1844                         err = -EHOSTUNREACH;
1845                 goto no_route;
1846         }
1847
1848         if (res.type == RTN_BROADCAST)
1849                 goto brd_input;
1850
1851         if (res.type == RTN_LOCAL) {
1852                 err = fib_validate_source(skb, saddr, daddr, tos,
1853                                           0, dev, in_dev, &itag);
1854                 if (err < 0)
1855                         goto martian_source;
1856                 goto local_input;
1857         }
1858
1859         if (!IN_DEV_FORWARD(in_dev)) {
1860                 err = -EHOSTUNREACH;
1861                 goto no_route;
1862         }
1863         if (res.type != RTN_UNICAST)
1864                 goto martian_destination;
1865
1866         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1867 out:    return err;
1868
1869 brd_input:
1870         if (skb->protocol != htons(ETH_P_IP))
1871                 goto e_inval;
1872
1873         if (!ipv4_is_zeronet(saddr)) {
1874                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1875                                           in_dev, &itag);
1876                 if (err < 0)
1877                         goto martian_source;
1878         }
1879         flags |= RTCF_BROADCAST;
1880         res.type = RTN_BROADCAST;
1881         RT_CACHE_STAT_INC(in_brd);
1882
1883 local_input:
1884         do_cache = false;
1885         if (res.fi) {
1886                 if (!itag) {
1887                         rth = rcu_dereference(FIB_RES_NH(res).nh_rth_input);
1888                         if (rt_cache_valid(rth)) {
1889                                 skb_dst_set_noref(skb, &rth->dst);
1890                                 err = 0;
1891                                 goto out;
1892                         }
1893                         do_cache = true;
1894                 }
1895         }
1896
1897         rth = rt_dst_alloc(net->loopback_dev, flags | RTCF_LOCAL, res.type,
1898                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
1899         if (!rth)
1900                 goto e_nobufs;
1901
1902         rth->dst.output= ip_rt_bug;
1903 #ifdef CONFIG_IP_ROUTE_CLASSID
1904         rth->dst.tclassid = itag;
1905 #endif
1906         rth->rt_is_input = 1;
1907         if (res.table)
1908                 rth->rt_table_id = res.table->tb_id;
1909
1910         RT_CACHE_STAT_INC(in_slow_tot);
1911         if (res.type == RTN_UNREACHABLE) {
1912                 rth->dst.input= ip_error;
1913                 rth->dst.error= -err;
1914                 rth->rt_flags   &= ~RTCF_LOCAL;
1915         }
1916         if (do_cache) {
1917                 if (unlikely(!rt_cache_route(&FIB_RES_NH(res), rth))) {
1918                         rth->dst.flags |= DST_NOCACHE;
1919                         rt_add_uncached_list(rth);
1920                 }
1921         }
1922         skb_dst_set(skb, &rth->dst);
1923         err = 0;
1924         goto out;
1925
1926 no_route:
1927         RT_CACHE_STAT_INC(in_no_route);
1928         res.type = RTN_UNREACHABLE;
1929         res.fi = NULL;
1930         res.table = NULL;
1931         goto local_input;
1932
1933         /*
1934          *      Do not cache martian addresses: they should be logged (RFC1812)
1935          */
1936 martian_destination:
1937         RT_CACHE_STAT_INC(in_martian_dst);
1938 #ifdef CONFIG_IP_ROUTE_VERBOSE
1939         if (IN_DEV_LOG_MARTIANS(in_dev))
1940                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
1941                                      &daddr, &saddr, dev->name);
1942 #endif
1943
1944 e_inval:
1945         err = -EINVAL;
1946         goto out;
1947
1948 e_nobufs:
1949         err = -ENOBUFS;
1950         goto out;
1951
1952 martian_source:
1953         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
1954         goto out;
1955 }
1956
1957 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1958                          u8 tos, struct net_device *dev)
1959 {
1960         int res;
1961
1962         rcu_read_lock();
1963
1964         /* Multicast recognition logic is moved from route cache to here.
1965            The problem was that too many Ethernet cards have broken/missing
1966            hardware multicast filters :-( As result the host on multicasting
1967            network acquires a lot of useless route cache entries, sort of
1968            SDR messages from all the world. Now we try to get rid of them.
1969            Really, provided software IP multicast filter is organized
1970            reasonably (at least, hashed), it does not result in a slowdown
1971            comparing with route cache reject entries.
1972            Note, that multicast routers are not affected, because
1973            route cache entry is created eventually.
1974          */
1975         if (ipv4_is_multicast(daddr)) {
1976                 struct in_device *in_dev = __in_dev_get_rcu(dev);
1977
1978                 if (in_dev) {
1979                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
1980                                                   ip_hdr(skb)->protocol);
1981                         if (our
1982 #ifdef CONFIG_IP_MROUTE
1983                                 ||
1984                             (!ipv4_is_local_multicast(daddr) &&
1985                              IN_DEV_MFORWARD(in_dev))
1986 #endif
1987                            ) {
1988                                 int res = ip_route_input_mc(skb, daddr, saddr,
1989                                                             tos, dev, our);
1990                                 rcu_read_unlock();
1991                                 return res;
1992                         }
1993                 }
1994                 rcu_read_unlock();
1995                 return -EINVAL;
1996         }
1997         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
1998         rcu_read_unlock();
1999         return res;
2000 }
2001 EXPORT_SYMBOL(ip_route_input_noref);
2002
2003 /* called with rcu_read_lock() */
2004 static struct rtable *__mkroute_output(const struct fib_result *res,
2005                                        const struct flowi4 *fl4, int orig_oif,
2006                                        struct net_device *dev_out,
2007                                        unsigned int flags)
2008 {
2009         struct fib_info *fi = res->fi;
2010         struct fib_nh_exception *fnhe;
2011         struct in_device *in_dev;
2012         u16 type = res->type;
2013         struct rtable *rth;
2014         bool do_cache;
2015
2016         in_dev = __in_dev_get_rcu(dev_out);
2017         if (!in_dev)
2018                 return ERR_PTR(-EINVAL);
2019
2020         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2021                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2022                         return ERR_PTR(-EINVAL);
2023
2024         if (ipv4_is_lbcast(fl4->daddr))
2025                 type = RTN_BROADCAST;
2026         else if (ipv4_is_multicast(fl4->daddr))
2027                 type = RTN_MULTICAST;
2028         else if (ipv4_is_zeronet(fl4->daddr))
2029                 return ERR_PTR(-EINVAL);
2030
2031         if (dev_out->flags & IFF_LOOPBACK)
2032                 flags |= RTCF_LOCAL;
2033
2034         do_cache = true;
2035         if (type == RTN_BROADCAST) {
2036                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2037                 fi = NULL;
2038         } else if (type == RTN_MULTICAST) {
2039                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2040                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2041                                      fl4->flowi4_proto))
2042                         flags &= ~RTCF_LOCAL;
2043                 else
2044                         do_cache = false;
2045                 /* If multicast route do not exist use
2046                  * default one, but do not gateway in this case.
2047                  * Yes, it is hack.
2048                  */
2049                 if (fi && res->prefixlen < 4)
2050                         fi = NULL;
2051         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2052                    (orig_oif != dev_out->ifindex)) {
2053                 /* For local routes that require a particular output interface
2054                  * we do not want to cache the result.  Caching the result
2055                  * causes incorrect behaviour when there are multiple source
2056                  * addresses on the interface, the end result being that if the
2057                  * intended recipient is waiting on that interface for the
2058                  * packet he won't receive it because it will be delivered on
2059                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2060                  * be set to the loopback interface as well.
2061                  */
2062                 fi = NULL;
2063         }
2064
2065         fnhe = NULL;
2066         do_cache &= fi != NULL;
2067         if (do_cache) {
2068                 struct rtable __rcu **prth;
2069                 struct fib_nh *nh = &FIB_RES_NH(*res);
2070
2071                 fnhe = find_exception(nh, fl4->daddr);
2072                 if (fnhe) {
2073                         prth = &fnhe->fnhe_rth_output;
2074                         rth = rcu_dereference(*prth);
2075                         if (rth && rth->dst.expires &&
2076                             time_after(jiffies, rth->dst.expires)) {
2077                                 ip_del_fnhe(nh, fl4->daddr);
2078                                 fnhe = NULL;
2079                         } else {
2080                                 goto rt_cache;
2081                         }
2082                 }
2083
2084                 if (unlikely(fl4->flowi4_flags &
2085                              FLOWI_FLAG_KNOWN_NH &&
2086                              !(nh->nh_gw &&
2087                                nh->nh_scope == RT_SCOPE_LINK))) {
2088                         do_cache = false;
2089                         goto add;
2090                 }
2091                 prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2092                 rth = rcu_dereference(*prth);
2093
2094 rt_cache:
2095                 if (rt_cache_valid(rth)) {
2096                         dst_hold(&rth->dst);
2097                         return rth;
2098                 }
2099         }
2100
2101 add:
2102         rth = rt_dst_alloc(dev_out, flags, type,
2103                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2104                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2105                            do_cache);
2106         if (!rth)
2107                 return ERR_PTR(-ENOBUFS);
2108
2109         rth->rt_iif     = orig_oif ? : 0;
2110         if (res->table)
2111                 rth->rt_table_id = res->table->tb_id;
2112
2113         RT_CACHE_STAT_INC(out_slow_tot);
2114
2115         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2116                 if (flags & RTCF_LOCAL &&
2117                     !(dev_out->flags & IFF_LOOPBACK)) {
2118                         rth->dst.output = ip_mc_output;
2119                         RT_CACHE_STAT_INC(out_slow_mc);
2120                 }
2121 #ifdef CONFIG_IP_MROUTE
2122                 if (type == RTN_MULTICAST) {
2123                         if (IN_DEV_MFORWARD(in_dev) &&
2124                             !ipv4_is_local_multicast(fl4->daddr)) {
2125                                 rth->dst.input = ip_mr_input;
2126                                 rth->dst.output = ip_mc_output;
2127                         }
2128                 }
2129 #endif
2130         }
2131
2132         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
2133         if (lwtunnel_output_redirect(rth->dst.lwtstate))
2134                 rth->dst.output = lwtunnel_output;
2135
2136         return rth;
2137 }
2138
2139 /*
2140  * Major route resolver routine.
2141  */
2142
2143 struct rtable *__ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2144                                           int mp_hash)
2145 {
2146         struct net_device *dev_out = NULL;
2147         __u8 tos = RT_FL_TOS(fl4);
2148         unsigned int flags = 0;
2149         struct fib_result res;
2150         struct rtable *rth;
2151         int master_idx;
2152         int orig_oif;
2153         int err = -ENETUNREACH;
2154
2155         res.tclassid    = 0;
2156         res.fi          = NULL;
2157         res.table       = NULL;
2158
2159         orig_oif = fl4->flowi4_oif;
2160
2161         master_idx = l3mdev_master_ifindex_by_index(net, fl4->flowi4_oif);
2162         if (master_idx)
2163                 fl4->flowi4_oif = master_idx;
2164         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2165         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2166         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2167                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2168
2169         rcu_read_lock();
2170         if (fl4->saddr) {
2171                 rth = ERR_PTR(-EINVAL);
2172                 if (ipv4_is_multicast(fl4->saddr) ||
2173                     ipv4_is_lbcast(fl4->saddr) ||
2174                     ipv4_is_zeronet(fl4->saddr))
2175                         goto out;
2176
2177                 /* I removed check for oif == dev_out->oif here.
2178                    It was wrong for two reasons:
2179                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2180                       is assigned to multiple interfaces.
2181                    2. Moreover, we are allowed to send packets with saddr
2182                       of another iface. --ANK
2183                  */
2184
2185                 if (fl4->flowi4_oif == 0 &&
2186                     (ipv4_is_multicast(fl4->daddr) ||
2187                      ipv4_is_lbcast(fl4->daddr))) {
2188                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2189                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2190                         if (!dev_out)
2191                                 goto out;
2192
2193                         /* Special hack: user can direct multicasts
2194                            and limited broadcast via necessary interface
2195                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2196                            This hack is not just for fun, it allows
2197                            vic,vat and friends to work.
2198                            They bind socket to loopback, set ttl to zero
2199                            and expect that it will work.
2200                            From the viewpoint of routing cache they are broken,
2201                            because we are not allowed to build multicast path
2202                            with loopback source addr (look, routing cache
2203                            cannot know, that ttl is zero, so that packet
2204                            will not leave this host and route is valid).
2205                            Luckily, this hack is good workaround.
2206                          */
2207
2208                         fl4->flowi4_oif = dev_out->ifindex;
2209                         goto make_route;
2210                 }
2211
2212                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2213                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2214                         if (!__ip_dev_find(net, fl4->saddr, false))
2215                                 goto out;
2216                 }
2217         }
2218
2219
2220         if (fl4->flowi4_oif) {
2221                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2222                 rth = ERR_PTR(-ENODEV);
2223                 if (!dev_out)
2224                         goto out;
2225
2226                 /* RACE: Check return value of inet_select_addr instead. */
2227                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2228                         rth = ERR_PTR(-ENETUNREACH);
2229                         goto out;
2230                 }
2231                 if (ipv4_is_local_multicast(fl4->daddr) ||
2232                     ipv4_is_lbcast(fl4->daddr) ||
2233                     fl4->flowi4_proto == IPPROTO_IGMP) {
2234                         if (!fl4->saddr)
2235                                 fl4->saddr = inet_select_addr(dev_out, 0,
2236                                                               RT_SCOPE_LINK);
2237                         goto make_route;
2238                 }
2239                 if (!fl4->saddr) {
2240                         if (ipv4_is_multicast(fl4->daddr))
2241                                 fl4->saddr = inet_select_addr(dev_out, 0,
2242                                                               fl4->flowi4_scope);
2243                         else if (!fl4->daddr)
2244                                 fl4->saddr = inet_select_addr(dev_out, 0,
2245                                                               RT_SCOPE_HOST);
2246                 }
2247
2248                 rth = l3mdev_get_rtable(dev_out, fl4);
2249                 if (rth)
2250                         goto out;
2251         }
2252
2253         if (!fl4->daddr) {
2254                 fl4->daddr = fl4->saddr;
2255                 if (!fl4->daddr)
2256                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2257                 dev_out = net->loopback_dev;
2258                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2259                 res.type = RTN_LOCAL;
2260                 flags |= RTCF_LOCAL;
2261                 goto make_route;
2262         }
2263
2264         err = fib_lookup(net, fl4, &res, 0);
2265         if (err) {
2266                 res.fi = NULL;
2267                 res.table = NULL;
2268                 if (fl4->flowi4_oif &&
2269                     !netif_index_is_l3_master(net, fl4->flowi4_oif)) {
2270                         /* Apparently, routing tables are wrong. Assume,
2271                            that the destination is on link.
2272
2273                            WHY? DW.
2274                            Because we are allowed to send to iface
2275                            even if it has NO routes and NO assigned
2276                            addresses. When oif is specified, routing
2277                            tables are looked up with only one purpose:
2278                            to catch if destination is gatewayed, rather than
2279                            direct. Moreover, if MSG_DONTROUTE is set,
2280                            we send packet, ignoring both routing tables
2281                            and ifaddr state. --ANK
2282
2283
2284                            We could make it even if oif is unknown,
2285                            likely IPv6, but we do not.
2286                          */
2287
2288                         if (fl4->saddr == 0)
2289                                 fl4->saddr = inet_select_addr(dev_out, 0,
2290                                                               RT_SCOPE_LINK);
2291                         res.type = RTN_UNICAST;
2292                         goto make_route;
2293                 }
2294                 rth = ERR_PTR(err);
2295                 goto out;
2296         }
2297
2298         if (res.type == RTN_LOCAL) {
2299                 if (!fl4->saddr) {
2300                         if (res.fi->fib_prefsrc)
2301                                 fl4->saddr = res.fi->fib_prefsrc;
2302                         else
2303                                 fl4->saddr = fl4->daddr;
2304                 }
2305                 dev_out = net->loopback_dev;
2306                 fl4->flowi4_oif = dev_out->ifindex;
2307                 flags |= RTCF_LOCAL;
2308                 goto make_route;
2309         }
2310
2311         fib_select_path(net, &res, fl4, mp_hash);
2312
2313         dev_out = FIB_RES_DEV(res);
2314         fl4->flowi4_oif = dev_out->ifindex;
2315
2316
2317 make_route:
2318         rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
2319
2320 out:
2321         rcu_read_unlock();
2322         return rth;
2323 }
2324 EXPORT_SYMBOL_GPL(__ip_route_output_key_hash);
2325
2326 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2327 {
2328         return NULL;
2329 }
2330
2331 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2332 {
2333         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2334
2335         return mtu ? : dst->dev->mtu;
2336 }
2337
2338 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2339                                           struct sk_buff *skb, u32 mtu)
2340 {
2341 }
2342
2343 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2344                                        struct sk_buff *skb)
2345 {
2346 }
2347
2348 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2349                                           unsigned long old)
2350 {
2351         return NULL;
2352 }
2353
2354 static struct dst_ops ipv4_dst_blackhole_ops = {
2355         .family                 =       AF_INET,
2356         .check                  =       ipv4_blackhole_dst_check,
2357         .mtu                    =       ipv4_blackhole_mtu,
2358         .default_advmss         =       ipv4_default_advmss,
2359         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2360         .redirect               =       ipv4_rt_blackhole_redirect,
2361         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2362         .neigh_lookup           =       ipv4_neigh_lookup,
2363 };
2364
2365 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2366 {
2367         struct rtable *ort = (struct rtable *) dst_orig;
2368         struct rtable *rt;
2369
2370         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2371         if (rt) {
2372                 struct dst_entry *new = &rt->dst;
2373
2374                 new->__use = 1;
2375                 new->input = dst_discard;
2376                 new->output = dst_discard_out;
2377
2378                 new->dev = ort->dst.dev;
2379                 if (new->dev)
2380                         dev_hold(new->dev);
2381
2382                 rt->rt_is_input = ort->rt_is_input;
2383                 rt->rt_iif = ort->rt_iif;
2384                 rt->rt_pmtu = ort->rt_pmtu;
2385
2386                 rt->rt_genid = rt_genid_ipv4(net);
2387                 rt->rt_flags = ort->rt_flags;
2388                 rt->rt_type = ort->rt_type;
2389                 rt->rt_gateway = ort->rt_gateway;
2390                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2391
2392                 INIT_LIST_HEAD(&rt->rt_uncached);
2393                 dst_free(new);
2394         }
2395
2396         dst_release(dst_orig);
2397
2398         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2399 }
2400
2401 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2402                                     const struct sock *sk)
2403 {
2404         struct rtable *rt = __ip_route_output_key(net, flp4);
2405
2406         if (IS_ERR(rt))
2407                 return rt;
2408
2409         if (flp4->flowi4_proto)
2410                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2411                                                         flowi4_to_flowi(flp4),
2412                                                         sk, 0);
2413
2414         return rt;
2415 }
2416 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2417
2418 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src, u32 table_id,
2419                         struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2420                         u32 seq, int event, int nowait, unsigned int flags)
2421 {
2422         struct rtable *rt = skb_rtable(skb);
2423         struct rtmsg *r;
2424         struct nlmsghdr *nlh;
2425         unsigned long expires = 0;
2426         u32 error;
2427         u32 metrics[RTAX_MAX];
2428
2429         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
2430         if (!nlh)
2431                 return -EMSGSIZE;
2432
2433         r = nlmsg_data(nlh);
2434         r->rtm_family    = AF_INET;
2435         r->rtm_dst_len  = 32;
2436         r->rtm_src_len  = 0;
2437         r->rtm_tos      = fl4->flowi4_tos;
2438         r->rtm_table    = table_id;
2439         if (nla_put_u32(skb, RTA_TABLE, table_id))
2440                 goto nla_put_failure;
2441         r->rtm_type     = rt->rt_type;
2442         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2443         r->rtm_protocol = RTPROT_UNSPEC;
2444         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2445         if (rt->rt_flags & RTCF_NOTIFY)
2446                 r->rtm_flags |= RTM_F_NOTIFY;
2447         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2448                 r->rtm_flags |= RTCF_DOREDIRECT;
2449
2450         if (nla_put_in_addr(skb, RTA_DST, dst))
2451                 goto nla_put_failure;
2452         if (src) {
2453                 r->rtm_src_len = 32;
2454                 if (nla_put_in_addr(skb, RTA_SRC, src))
2455                         goto nla_put_failure;
2456         }
2457         if (rt->dst.dev &&
2458             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2459                 goto nla_put_failure;
2460 #ifdef CONFIG_IP_ROUTE_CLASSID
2461         if (rt->dst.tclassid &&
2462             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2463                 goto nla_put_failure;
2464 #endif
2465         if (!rt_is_input_route(rt) &&
2466             fl4->saddr != src) {
2467                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2468                         goto nla_put_failure;
2469         }
2470         if (rt->rt_uses_gateway &&
2471             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2472                 goto nla_put_failure;
2473
2474         expires = rt->dst.expires;
2475         if (expires) {
2476                 unsigned long now = jiffies;
2477
2478                 if (time_before(now, expires))
2479                         expires -= now;
2480                 else
2481                         expires = 0;
2482         }
2483
2484         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2485         if (rt->rt_pmtu && expires)
2486                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2487         if (rtnetlink_put_metrics(skb, metrics) < 0)
2488                 goto nla_put_failure;
2489
2490         if (fl4->flowi4_mark &&
2491             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2492                 goto nla_put_failure;
2493
2494         error = rt->dst.error;
2495
2496         if (rt_is_input_route(rt)) {
2497 #ifdef CONFIG_IP_MROUTE
2498                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2499                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2500                         int err = ipmr_get_route(net, skb,
2501                                                  fl4->saddr, fl4->daddr,
2502                                                  r, nowait);
2503                         if (err <= 0) {
2504                                 if (!nowait) {
2505                                         if (err == 0)
2506                                                 return 0;
2507                                         goto nla_put_failure;
2508                                 } else {
2509                                         if (err == -EMSGSIZE)
2510                                                 goto nla_put_failure;
2511                                         error = err;
2512                                 }
2513                         }
2514                 } else
2515 #endif
2516                         if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2517                                 goto nla_put_failure;
2518         }
2519
2520         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2521                 goto nla_put_failure;
2522
2523         nlmsg_end(skb, nlh);
2524         return 0;
2525
2526 nla_put_failure:
2527         nlmsg_cancel(skb, nlh);
2528         return -EMSGSIZE;
2529 }
2530
2531 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
2532 {
2533         struct net *net = sock_net(in_skb->sk);
2534         struct rtmsg *rtm;
2535         struct nlattr *tb[RTA_MAX+1];
2536         struct rtable *rt = NULL;
2537         struct flowi4 fl4;
2538         __be32 dst = 0;
2539         __be32 src = 0;
2540         u32 iif;
2541         int err;
2542         int mark;
2543         struct sk_buff *skb;
2544         u32 table_id = RT_TABLE_MAIN;
2545
2546         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2547         if (err < 0)
2548                 goto errout;
2549
2550         rtm = nlmsg_data(nlh);
2551
2552         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2553         if (!skb) {
2554                 err = -ENOBUFS;
2555                 goto errout;
2556         }
2557
2558         /* Reserve room for dummy headers, this skb can pass
2559            through good chunk of routing engine.
2560          */
2561         skb_reset_mac_header(skb);
2562         skb_reset_network_header(skb);
2563
2564         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2565         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2566         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2567
2568         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2569         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2570         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2571         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2572
2573         memset(&fl4, 0, sizeof(fl4));
2574         fl4.daddr = dst;
2575         fl4.saddr = src;
2576         fl4.flowi4_tos = rtm->rtm_tos;
2577         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2578         fl4.flowi4_mark = mark;
2579
2580         if (netif_index_is_l3_master(net, fl4.flowi4_oif))
2581                 fl4.flowi4_flags = FLOWI_FLAG_L3MDEV_SRC | FLOWI_FLAG_SKIP_NH_OIF;
2582
2583         if (iif) {
2584                 struct net_device *dev;
2585
2586                 dev = __dev_get_by_index(net, iif);
2587                 if (!dev) {
2588                         err = -ENODEV;
2589                         goto errout_free;
2590                 }
2591
2592                 skb->protocol   = htons(ETH_P_IP);
2593                 skb->dev        = dev;
2594                 skb->mark       = mark;
2595                 local_bh_disable();
2596                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2597                 local_bh_enable();
2598
2599                 rt = skb_rtable(skb);
2600                 if (err == 0 && rt->dst.error)
2601                         err = -rt->dst.error;
2602         } else {
2603                 rt = ip_route_output_key(net, &fl4);
2604
2605                 err = 0;
2606                 if (IS_ERR(rt))
2607                         err = PTR_ERR(rt);
2608         }
2609
2610         if (err)
2611                 goto errout_free;
2612
2613         skb_dst_set(skb, &rt->dst);
2614         if (rtm->rtm_flags & RTM_F_NOTIFY)
2615                 rt->rt_flags |= RTCF_NOTIFY;
2616
2617         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2618                 table_id = rt->rt_table_id;
2619
2620         err = rt_fill_info(net, dst, src, table_id, &fl4, skb,
2621                            NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2622                            RTM_NEWROUTE, 0, 0);
2623         if (err < 0)
2624                 goto errout_free;
2625
2626         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2627 errout:
2628         return err;
2629
2630 errout_free:
2631         kfree_skb(skb);
2632         goto errout;
2633 }
2634
2635 void ip_rt_multicast_event(struct in_device *in_dev)
2636 {
2637         rt_cache_flush(dev_net(in_dev->dev));
2638 }
2639
2640 #ifdef CONFIG_SYSCTL
2641 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2642 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2643 static int ip_rt_gc_elasticity __read_mostly    = 8;
2644
2645 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2646                                         void __user *buffer,
2647                                         size_t *lenp, loff_t *ppos)
2648 {
2649         struct net *net = (struct net *)__ctl->extra1;
2650
2651         if (write) {
2652                 rt_cache_flush(net);
2653                 fnhe_genid_bump(net);
2654                 return 0;
2655         }
2656
2657         return -EINVAL;
2658 }
2659
2660 static struct ctl_table ipv4_route_table[] = {
2661         {
2662                 .procname       = "gc_thresh",
2663                 .data           = &ipv4_dst_ops.gc_thresh,
2664                 .maxlen         = sizeof(int),
2665                 .mode           = 0644,
2666                 .proc_handler   = proc_dointvec,
2667         },
2668         {
2669                 .procname       = "max_size",
2670                 .data           = &ip_rt_max_size,
2671                 .maxlen         = sizeof(int),
2672                 .mode           = 0644,
2673                 .proc_handler   = proc_dointvec,
2674         },
2675         {
2676                 /*  Deprecated. Use gc_min_interval_ms */
2677
2678                 .procname       = "gc_min_interval",
2679                 .data           = &ip_rt_gc_min_interval,
2680                 .maxlen         = sizeof(int),
2681                 .mode           = 0644,
2682                 .proc_handler   = proc_dointvec_jiffies,
2683         },
2684         {
2685                 .procname       = "gc_min_interval_ms",
2686                 .data           = &ip_rt_gc_min_interval,
2687                 .maxlen         = sizeof(int),
2688                 .mode           = 0644,
2689                 .proc_handler   = proc_dointvec_ms_jiffies,
2690         },
2691         {
2692                 .procname       = "gc_timeout",
2693                 .data           = &ip_rt_gc_timeout,
2694                 .maxlen         = sizeof(int),
2695                 .mode           = 0644,
2696                 .proc_handler   = proc_dointvec_jiffies,
2697         },
2698         {
2699                 .procname       = "gc_interval",
2700                 .data           = &ip_rt_gc_interval,
2701                 .maxlen         = sizeof(int),
2702                 .mode           = 0644,
2703                 .proc_handler   = proc_dointvec_jiffies,
2704         },
2705         {
2706                 .procname       = "redirect_load",
2707                 .data           = &ip_rt_redirect_load,
2708                 .maxlen         = sizeof(int),
2709                 .mode           = 0644,
2710                 .proc_handler   = proc_dointvec,
2711         },
2712         {
2713                 .procname       = "redirect_number",
2714                 .data           = &ip_rt_redirect_number,
2715                 .maxlen         = sizeof(int),
2716                 .mode           = 0644,
2717                 .proc_handler   = proc_dointvec,
2718         },
2719         {
2720                 .procname       = "redirect_silence",
2721                 .data           = &ip_rt_redirect_silence,
2722                 .maxlen         = sizeof(int),
2723                 .mode           = 0644,
2724                 .proc_handler   = proc_dointvec,
2725         },
2726         {
2727                 .procname       = "error_cost",
2728                 .data           = &ip_rt_error_cost,
2729                 .maxlen         = sizeof(int),
2730                 .mode           = 0644,
2731                 .proc_handler   = proc_dointvec,
2732         },
2733         {
2734                 .procname       = "error_burst",
2735                 .data           = &ip_rt_error_burst,
2736                 .maxlen         = sizeof(int),
2737                 .mode           = 0644,
2738                 .proc_handler   = proc_dointvec,
2739         },
2740         {
2741                 .procname       = "gc_elasticity",
2742                 .data           = &ip_rt_gc_elasticity,
2743                 .maxlen         = sizeof(int),
2744                 .mode           = 0644,
2745                 .proc_handler   = proc_dointvec,
2746         },
2747         {
2748                 .procname       = "mtu_expires",
2749                 .data           = &ip_rt_mtu_expires,
2750                 .maxlen         = sizeof(int),
2751                 .mode           = 0644,
2752                 .proc_handler   = proc_dointvec_jiffies,
2753         },
2754         {
2755                 .procname       = "min_pmtu",
2756                 .data           = &ip_rt_min_pmtu,
2757                 .maxlen         = sizeof(int),
2758                 .mode           = 0644,
2759                 .proc_handler   = proc_dointvec,
2760         },
2761         {
2762                 .procname       = "min_adv_mss",
2763                 .data           = &ip_rt_min_advmss,
2764                 .maxlen         = sizeof(int),
2765                 .mode           = 0644,
2766                 .proc_handler   = proc_dointvec,
2767         },
2768         { }
2769 };
2770
2771 static struct ctl_table ipv4_route_flush_table[] = {
2772         {
2773                 .procname       = "flush",
2774                 .maxlen         = sizeof(int),
2775                 .mode           = 0200,
2776                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2777         },
2778         { },
2779 };
2780
2781 static __net_init int sysctl_route_net_init(struct net *net)
2782 {
2783         struct ctl_table *tbl;
2784
2785         tbl = ipv4_route_flush_table;
2786         if (!net_eq(net, &init_net)) {
2787                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2788                 if (!tbl)
2789                         goto err_dup;
2790
2791                 /* Don't export sysctls to unprivileged users */
2792                 if (net->user_ns != &init_user_ns)
2793                         tbl[0].procname = NULL;
2794         }
2795         tbl[0].extra1 = net;
2796
2797         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2798         if (!net->ipv4.route_hdr)
2799                 goto err_reg;
2800         return 0;
2801
2802 err_reg:
2803         if (tbl != ipv4_route_flush_table)
2804                 kfree(tbl);
2805 err_dup:
2806         return -ENOMEM;
2807 }
2808
2809 static __net_exit void sysctl_route_net_exit(struct net *net)
2810 {
2811         struct ctl_table *tbl;
2812
2813         tbl = net->ipv4.route_hdr->ctl_table_arg;
2814         unregister_net_sysctl_table(net->ipv4.route_hdr);
2815         BUG_ON(tbl == ipv4_route_flush_table);
2816         kfree(tbl);
2817 }
2818
2819 static __net_initdata struct pernet_operations sysctl_route_ops = {
2820         .init = sysctl_route_net_init,
2821         .exit = sysctl_route_net_exit,
2822 };
2823 #endif
2824
2825 static __net_init int rt_genid_init(struct net *net)
2826 {
2827         atomic_set(&net->ipv4.rt_genid, 0);
2828         atomic_set(&net->fnhe_genid, 0);
2829         get_random_bytes(&net->ipv4.dev_addr_genid,
2830                          sizeof(net->ipv4.dev_addr_genid));
2831         return 0;
2832 }
2833
2834 static __net_initdata struct pernet_operations rt_genid_ops = {
2835         .init = rt_genid_init,
2836 };
2837
2838 static int __net_init ipv4_inetpeer_init(struct net *net)
2839 {
2840         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
2841
2842         if (!bp)
2843                 return -ENOMEM;
2844         inet_peer_base_init(bp);
2845         net->ipv4.peers = bp;
2846         return 0;
2847 }
2848
2849 static void __net_exit ipv4_inetpeer_exit(struct net *net)
2850 {
2851         struct inet_peer_base *bp = net->ipv4.peers;
2852
2853         net->ipv4.peers = NULL;
2854         inetpeer_invalidate_tree(bp);
2855         kfree(bp);
2856 }
2857
2858 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
2859         .init   =       ipv4_inetpeer_init,
2860         .exit   =       ipv4_inetpeer_exit,
2861 };
2862
2863 #ifdef CONFIG_IP_ROUTE_CLASSID
2864 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2865 #endif /* CONFIG_IP_ROUTE_CLASSID */
2866
2867 int __init ip_rt_init(void)
2868 {
2869         int rc = 0;
2870         int cpu;
2871
2872         ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
2873         if (!ip_idents)
2874                 panic("IP: failed to allocate ip_idents\n");
2875
2876         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
2877
2878         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
2879         if (!ip_tstamps)
2880                 panic("IP: failed to allocate ip_tstamps\n");
2881
2882         for_each_possible_cpu(cpu) {
2883                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
2884
2885                 INIT_LIST_HEAD(&ul->head);
2886                 spin_lock_init(&ul->lock);
2887         }
2888 #ifdef CONFIG_IP_ROUTE_CLASSID
2889         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
2890         if (!ip_rt_acct)
2891                 panic("IP: failed to allocate ip_rt_acct\n");
2892 #endif
2893
2894         ipv4_dst_ops.kmem_cachep =
2895                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2896                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2897
2898         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
2899
2900         if (dst_entries_init(&ipv4_dst_ops) < 0)
2901                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
2902
2903         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
2904                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
2905
2906         ipv4_dst_ops.gc_thresh = ~0;
2907         ip_rt_max_size = INT_MAX;
2908
2909         devinet_init();
2910         ip_fib_init();
2911
2912         if (ip_rt_proc_init())
2913                 pr_err("Unable to create route proc files\n");
2914 #ifdef CONFIG_XFRM
2915         xfrm_init();
2916         xfrm4_init();
2917 #endif
2918         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2919
2920 #ifdef CONFIG_SYSCTL
2921         register_pernet_subsys(&sysctl_route_ops);
2922 #endif
2923         register_pernet_subsys(&rt_genid_ops);
2924         register_pernet_subsys(&ipv4_inetpeer_ops);
2925         return rc;
2926 }
2927
2928 #ifdef CONFIG_SYSCTL
2929 /*
2930  * We really need to sanitize the damn ipv4 init order, then all
2931  * this nonsense will go away.
2932  */
2933 void __init ip_static_sysctl_init(void)
2934 {
2935         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
2936 }
2937 #endif