net/core/dev.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      NET3    Protocol independent device support routines.
   4  *
   5  *      Derived from the non IP parts of dev.c 1.0.19
   6  *              Authors:        Ross Biro
   7  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
   8  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
   9  *
  10  *      Additional Authors:
  11  *              Florian la Roche <rzsfl@rz.uni-sb.de>
  12  *              Alan Cox <gw4pts@gw4pts.ampr.org>
  13  *              David Hinds <dahinds@users.sourceforge.net>
  14  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  15  *              Adam Sulmicki <adam@cfar.umd.edu>
  16  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
  17  *
  18  *      Changes:
  19  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
  20  *                                      to 2 if register_netdev gets called
  21  *                                      before net_dev_init & also removed a
  22  *                                      few lines of code in the process.
  23  *              Alan Cox        :       device private ioctl copies fields back.
  24  *              Alan Cox        :       Transmit queue code does relevant
  25  *                                      stunts to keep the queue safe.
  26  *              Alan Cox        :       Fixed double lock.
  27  *              Alan Cox        :       Fixed promisc NULL pointer trap
  28  *              ????????        :       Support the full private ioctl range
  29  *              Alan Cox        :       Moved ioctl permission check into
  30  *                                      drivers
  31  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
  32  *              Alan Cox        :       100 backlog just doesn't cut it when
  33  *                                      you start doing multicast video 8)
  34  *              Alan Cox        :       Rewrote net_bh and list manager.
  35  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
  36  *              Alan Cox        :       Took out transmit every packet pass
  37  *                                      Saved a few bytes in the ioctl handler
  38  *              Alan Cox        :       Network driver sets packet type before
  39  *                                      calling netif_rx. Saves a function
  40  *                                      call a packet.
  41  *              Alan Cox        :       Hashed net_bh()
  42  *              Richard Kooijman:       Timestamp fixes.
  43  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
  44  *              Alan Cox        :       Device lock protection.
  45  *              Alan Cox        :       Fixed nasty side effect of device close
  46  *                                      changes.
  47  *              Rudi Cilibrasi  :       Pass the right thing to
  48  *                                      set_mac_address()
  49  *              Dave Miller     :       32bit quantity for the device lock to
  50  *                                      make it work out on a Sparc.
  51  *              Bjorn Ekwall    :       Added KERNELD hack.
  52  *              Alan Cox        :       Cleaned up the backlog initialise.
  53  *              Craig Metz      :       SIOCGIFCONF fix if space for under
  54  *                                      1 device.
  55  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
  56  *                                      is no device open function.
  57  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
  58  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
  59  *              Cyrus Durgin    :       Cleaned for KMOD
  60  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
  61  *                                      A network device unload needs to purge
  62  *                                      the backlog queue.
  63  *      Paul Rusty Russell      :       SIOCSIFNAME
  64  *              Pekka Riikonen  :       Netdev boot-time settings code
  65  *              Andrew Morton   :       Make unregister_netdevice wait
  66  *                                      indefinitely on dev->refcnt
  67  *              J Hadi Salim    :       - Backlog queue sampling
  68  *                                      - netif_rx() feedback
  69  */
  70
  71 #include <linux/uaccess.h>
  72 #include <linux/bitops.h>
  73 #include <linux/capability.h>
  74 #include <linux/cpu.h>
  75 #include <linux/types.h>
  76 #include <linux/kernel.h>
  77 #include <linux/hash.h>
  78 #include <linux/slab.h>
  79 #include <linux/sched.h>
  80 #include <linux/sched/mm.h>
  81 #include <linux/mutex.h>
  82 #include <linux/rwsem.h>
  83 #include <linux/string.h>
  84 #include <linux/mm.h>
  85 #include <linux/socket.h>
  86 #include <linux/sockios.h>
  87 #include <linux/errno.h>
  88 #include <linux/interrupt.h>
  89 #include <linux/if_ether.h>
  90 #include <linux/netdevice.h>
  91 #include <linux/etherdevice.h>
  92 #include <linux/ethtool.h>
  93 #include <linux/skbuff.h>
  94 #include <linux/kthread.h>
  95 #include <linux/bpf.h>
  96 #include <linux/bpf_trace.h>
  97 #include <net/net_namespace.h>
  98 #include <net/sock.h>
  99 #include <net/busy_poll.h>
 100 #include <linux/rtnetlink.h>
 101 #include <linux/stat.h>
 102 #include <net/dsa.h>
 103 #include <net/dst.h>
 104 #include <net/dst_metadata.h>
 105 #include <net/gro.h>
 106 #include <net/pkt_sched.h>
 107 #include <net/pkt_cls.h>
 108 #include <net/checksum.h>
 109 #include <net/xfrm.h>
 110 #include <linux/highmem.h>
 111 #include <linux/init.h>
 112 #include <linux/module.h>
 113 #include <linux/netpoll.h>
 114 #include <linux/rcupdate.h>
 115 #include <linux/delay.h>
 116 #include <net/iw_handler.h>
 117 #include <asm/current.h>
 118 #include <linux/audit.h>
 119 #include <linux/dmaengine.h>
 120 #include <linux/err.h>
 121 #include <linux/ctype.h>
 122 #include <linux/if_arp.h>
 123 #include <linux/if_vlan.h>
 124 #include <linux/ip.h>
 125 #include <net/ip.h>
 126 #include <net/mpls.h>
 127 #include <linux/ipv6.h>
 128 #include <linux/in.h>
 129 #include <linux/jhash.h>
 130 #include <linux/random.h>
 131 #include <trace/events/napi.h>
 132 #include <trace/events/net.h>
 133 #include <trace/events/skb.h>
 134 #include <trace/events/qdisc.h>
 135 #include <linux/inetdevice.h>
 136 #include <linux/cpu_rmap.h>
 137 #include <linux/static_key.h>
 138 #include <linux/hashtable.h>
 139 #include <linux/vmalloc.h>
 140 #include <linux/if_macvlan.h>
 141 #include <linux/errqueue.h>
 142 #include <linux/hrtimer.h>
 143 #include <linux/netfilter_netdev.h>
 144 #include <linux/crash_dump.h>
 145 #include <linux/sctp.h>
 146 #include <net/udp_tunnel.h>
 147 #include <linux/net_namespace.h>
 148 #include <linux/indirect_call_wrapper.h>
 149 #include <net/devlink.h>
 150 #include <linux/pm_runtime.h>
 151 #include <linux/prandom.h>
 152 #include <linux/once_lite.h>
 153
 154 #include "dev.h"
 155 #include "net-sysfs.h"
 156
 157
 158 static DEFINE_SPINLOCK(ptype_lock);
 159 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 160 struct list_head ptype_all __read_mostly;       /* Taps */
 161
 162 static int netif_rx_internal(struct sk_buff *skb);
 163 static int call_netdevice_notifiers_extack(unsigned long val,
 164                                            struct net_device *dev,
 165                                            struct netlink_ext_ack *extack);
 166 static struct napi_struct *napi_by_id(unsigned int napi_id);
 167
 168 /*
 169  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
 170  * semaphore.
 171  *
 172  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
 173  *
 174  * Writers must hold the rtnl semaphore while they loop through the
 175  * dev_base_head list, and hold dev_base_lock for writing when they do the
 176  * actual updates.  This allows pure readers to access the list even
 177  * while a writer is preparing to update it.
 178  *
 179  * To put it another way, dev_base_lock is held for writing only to
 180  * protect against pure readers; the rtnl semaphore provides the
 181  * protection against other writers.
 182  *
 183  * See, for example usages, register_netdevice() and
 184  * unregister_netdevice(), which must be called with the rtnl
 185  * semaphore held.
 186  */
 187 DEFINE_RWLOCK(dev_base_lock);
 188 EXPORT_SYMBOL(dev_base_lock);
 189
 190 static DEFINE_MUTEX(ifalias_mutex);
 191
 192 /* protects napi_hash addition/deletion and napi_gen_id */
 193 static DEFINE_SPINLOCK(napi_hash_lock);
 194
 195 static unsigned int napi_gen_id = NR_CPUS;
 196 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
 197
 198 static DECLARE_RWSEM(devnet_rename_sem);
 199
 200 static inline void dev_base_seq_inc(struct net *net)
 201 {
 202         while (++net->dev_base_seq == 0)
 203                 ;
 204 }
 205
 206 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
 207 {
 208         unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
 209
 210         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
 211 }
 212
 213 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
 214 {
 215         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
 216 }
 217
 218 static inline void rps_lock_irqsave(struct softnet_data *sd,
 219                                     unsigned long *flags)
 220 {
 221         if (IS_ENABLED(CONFIG_RPS))
 222                 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
 223         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 224                 local_irq_save(*flags);
 225 }
 226
 227 static inline void rps_lock_irq_disable(struct softnet_data *sd)
 228 {
 229         if (IS_ENABLED(CONFIG_RPS))
 230                 spin_lock_irq(&sd->input_pkt_queue.lock);
 231         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 232                 local_irq_disable();
 233 }
 234
 235 static inline void rps_unlock_irq_restore(struct softnet_data *sd,
 236                                           unsigned long *flags)
 237 {
 238         if (IS_ENABLED(CONFIG_RPS))
 239                 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
 240         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 241                 local_irq_restore(*flags);
 242 }
 243
 244 static inline void rps_unlock_irq_enable(struct softnet_data *sd)
 245 {
 246         if (IS_ENABLED(CONFIG_RPS))
 247                 spin_unlock_irq(&sd->input_pkt_queue.lock);
 248         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 249                 local_irq_enable();
 250 }
 251
 252 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
 253                                                        const char *name)
 254 {
 255         struct netdev_name_node *name_node;
 256
 257         name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
 258         if (!name_node)
 259                 return NULL;
 260         INIT_HLIST_NODE(&name_node->hlist);
 261         name_node->dev = dev;
 262         name_node->name = name;
 263         return name_node;
 264 }
 265
 266 static struct netdev_name_node *
 267 netdev_name_node_head_alloc(struct net_device *dev)
 268 {
 269         struct netdev_name_node *name_node;
 270
 271         name_node = netdev_name_node_alloc(dev, dev->name);
 272         if (!name_node)
 273                 return NULL;
 274         INIT_LIST_HEAD(&name_node->list);
 275         return name_node;
 276 }
 277
 278 static void netdev_name_node_free(struct netdev_name_node *name_node)
 279 {
 280         kfree(name_node);
 281 }
 282
 283 static void netdev_name_node_add(struct net *net,
 284                                  struct netdev_name_node *name_node)
 285 {
 286         hlist_add_head_rcu(&name_node->hlist,
 287                            dev_name_hash(net, name_node->name));
 288 }
 289
 290 static void netdev_name_node_del(struct netdev_name_node *name_node)
 291 {
 292         hlist_del_rcu(&name_node->hlist);
 293 }
 294
 295 static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
 296                                                         const char *name)
 297 {
 298         struct hlist_head *head = dev_name_hash(net, name);
 299         struct netdev_name_node *name_node;
 300
 301         hlist_for_each_entry(name_node, head, hlist)
 302                 if (!strcmp(name_node->name, name))
 303                         return name_node;
 304         return NULL;
 305 }
 306
 307 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
 308                                                             const char *name)
 309 {
 310         struct hlist_head *head = dev_name_hash(net, name);
 311         struct netdev_name_node *name_node;
 312
 313         hlist_for_each_entry_rcu(name_node, head, hlist)
 314                 if (!strcmp(name_node->name, name))
 315                         return name_node;
 316         return NULL;
 317 }
 318
 319 bool netdev_name_in_use(struct net *net, const char *name)
 320 {
 321         return netdev_name_node_lookup(net, name);
 322 }
 323 EXPORT_SYMBOL(netdev_name_in_use);
 324
 325 int netdev_name_node_alt_create(struct net_device *dev, const char *name)
 326 {
 327         struct netdev_name_node *name_node;
 328         struct net *net = dev_net(dev);
 329
 330         name_node = netdev_name_node_lookup(net, name);
 331         if (name_node)
 332                 return -EEXIST;
 333         name_node = netdev_name_node_alloc(dev, name);
 334         if (!name_node)
 335                 return -ENOMEM;
 336         netdev_name_node_add(net, name_node);
 337         /* The node that holds dev->name acts as a head of per-device list. */
 338         list_add_tail(&name_node->list, &dev->name_node->list);
 339
 340         return 0;
 341 }
 342
 343 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
 344 {
 345         list_del(&name_node->list);
 346         netdev_name_node_del(name_node);
 347         kfree(name_node->name);
 348         netdev_name_node_free(name_node);
 349 }
 350
 351 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
 352 {
 353         struct netdev_name_node *name_node;
 354         struct net *net = dev_net(dev);
 355
 356         name_node = netdev_name_node_lookup(net, name);
 357         if (!name_node)
 358                 return -ENOENT;
 359         /* lookup might have found our primary name or a name belonging
 360          * to another device.
 361          */
 362         if (name_node == dev->name_node || name_node->dev != dev)
 363                 return -EINVAL;
 364
 365         __netdev_name_node_alt_destroy(name_node);
 366
 367         return 0;
 368 }
 369
 370 static void netdev_name_node_alt_flush(struct net_device *dev)
 371 {
 372         struct netdev_name_node *name_node, *tmp;
 373
 374         list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
 375                 __netdev_name_node_alt_destroy(name_node);
 376 }
 377
 378 /* Device list insertion */
 379 static void list_netdevice(struct net_device *dev)
 380 {
 381         struct net *net = dev_net(dev);
 382
 383         ASSERT_RTNL();
 384
 385         write_lock(&dev_base_lock);
 386         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
 387         netdev_name_node_add(net, dev->name_node);
 388         hlist_add_head_rcu(&dev->index_hlist,
 389                            dev_index_hash(net, dev->ifindex));
 390         write_unlock(&dev_base_lock);
 391
 392         dev_base_seq_inc(net);
 393 }
 394
 395 /* Device list removal
 396  * caller must respect a RCU grace period before freeing/reusing dev
 397  */
 398 static void unlist_netdevice(struct net_device *dev, bool lock)
 399 {
 400         ASSERT_RTNL();
 401
 402         /* Unlink dev from the device chain */
 403         if (lock)
 404                 write_lock(&dev_base_lock);
 405         list_del_rcu(&dev->dev_list);
 406         netdev_name_node_del(dev->name_node);
 407         hlist_del_rcu(&dev->index_hlist);
 408         if (lock)
 409                 write_unlock(&dev_base_lock);
 410
 411         dev_base_seq_inc(dev_net(dev));
 412 }
 413
 414 /*
 415  *      Our notifier list
 416  */
 417
 418 static RAW_NOTIFIER_HEAD(netdev_chain);
 419
 420 /*
 421  *      Device drivers call our routines to queue packets here. We empty the
 422  *      queue in the local softnet handler.
 423  */
 424
 425 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 426 EXPORT_PER_CPU_SYMBOL(softnet_data);
 427
 428 #ifdef CONFIG_LOCKDEP
 429 /*
 430  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
 431  * according to dev->type
 432  */
 433 static const unsigned short netdev_lock_type[] = {
 434          ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
 435          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
 436          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
 437          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
 438          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
 439          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
 440          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
 441          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
 442          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
 443          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
 444          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
 445          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
 446          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
 447          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
 448          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
 449
 450 static const char *const netdev_lock_name[] = {
 451         "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
 452         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
 453         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
 454         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
 455         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
 456         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
 457         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
 458         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
 459         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
 460         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
 461         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
 462         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
 463         "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
 464         "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
 465         "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
 466
 467 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
 468 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
 469
 470 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
 471 {
 472         int i;
 473
 474         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
 475                 if (netdev_lock_type[i] == dev_type)
 476                         return i;
 477         /* the last key is used by default */
 478         return ARRAY_SIZE(netdev_lock_type) - 1;
 479 }
 480
 481 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 482                                                  unsigned short dev_type)
 483 {
 484         int i;
 485
 486         i = netdev_lock_pos(dev_type);
 487         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
 488                                    netdev_lock_name[i]);
 489 }
 490
 491 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 492 {
 493         int i;
 494
 495         i = netdev_lock_pos(dev->type);
 496         lockdep_set_class_and_name(&dev->addr_list_lock,
 497                                    &netdev_addr_lock_key[i],
 498                                    netdev_lock_name[i]);
 499 }
 500 #else
 501 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 502                                                  unsigned short dev_type)
 503 {
 504 }
 505
 506 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 507 {
 508 }
 509 #endif
 510
 511 /*******************************************************************************
 512  *
 513  *              Protocol management and registration routines
 514  *
 515  *******************************************************************************/
 516
 517
 518 /*
 519  *      Add a protocol ID to the list. Now that the input handler is
 520  *      smarter we can dispense with all the messy stuff that used to be
 521  *      here.
 522  *
 523  *      BEWARE!!! Protocol handlers, mangling input packets,
 524  *      MUST BE last in hash buckets and checking protocol handlers
 525  *      MUST start from promiscuous ptype_all chain in net_bh.
 526  *      It is true now, do not change it.
 527  *      Explanation follows: if protocol handler, mangling packet, will
 528  *      be the first on list, it is not able to sense, that packet
 529  *      is cloned and should be copied-on-write, so that it will
 530  *      change it and subsequent readers will get broken packet.
 531  *                                                      --ANK (980803)
 532  */
 533
 534 static inline struct list_head *ptype_head(const struct packet_type *pt)
 535 {
 536         if (pt->type == htons(ETH_P_ALL))
 537                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
 538         else
 539                 return pt->dev ? &pt->dev->ptype_specific :
 540                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
 541 }
 542
 543 /**
 544  *      dev_add_pack - add packet handler
 545  *      @pt: packet type declaration
 546  *
 547  *      Add a protocol handler to the networking stack. The passed &packet_type
 548  *      is linked into kernel lists and may not be freed until it has been
 549  *      removed from the kernel lists.
 550  *
 551  *      This call does not sleep therefore it can not
 552  *      guarantee all CPU's that are in middle of receiving packets
 553  *      will see the new packet type (until the next received packet).
 554  */
 555
 556 void dev_add_pack(struct packet_type *pt)
 557 {
 558         struct list_head *head = ptype_head(pt);
 559
 560         spin_lock(&ptype_lock);
 561         list_add_rcu(&pt->list, head);
 562         spin_unlock(&ptype_lock);
 563 }
 564 EXPORT_SYMBOL(dev_add_pack);
 565
 566 /**
 567  *      __dev_remove_pack        - remove packet handler
 568  *      @pt: packet type declaration
 569  *
 570  *      Remove a protocol handler that was previously added to the kernel
 571  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 572  *      from the kernel lists and can be freed or reused once this function
 573  *      returns.
 574  *
 575  *      The packet type might still be in use by receivers
 576  *      and must not be freed until after all the CPU's have gone
 577  *      through a quiescent state.
 578  */
 579 void __dev_remove_pack(struct packet_type *pt)
 580 {
 581         struct list_head *head = ptype_head(pt);
 582         struct packet_type *pt1;
 583
 584         spin_lock(&ptype_lock);
 585
 586         list_for_each_entry(pt1, head, list) {
 587                 if (pt == pt1) {
 588                         list_del_rcu(&pt->list);
 589                         goto out;
 590                 }
 591         }
 592
 593         pr_warn("dev_remove_pack: %p not found\n", pt);
 594 out:
 595         spin_unlock(&ptype_lock);
 596 }
 597 EXPORT_SYMBOL(__dev_remove_pack);
 598
 599 /**
 600  *      dev_remove_pack  - remove packet handler
 601  *      @pt: packet type declaration
 602  *
 603  *      Remove a protocol handler that was previously added to the kernel
 604  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 605  *      from the kernel lists and can be freed or reused once this function
 606  *      returns.
 607  *
 608  *      This call sleeps to guarantee that no CPU is looking at the packet
 609  *      type after return.
 610  */
 611 void dev_remove_pack(struct packet_type *pt)
 612 {
 613         __dev_remove_pack(pt);
 614
 615         synchronize_net();
 616 }
 617 EXPORT_SYMBOL(dev_remove_pack);
 618
 619
 620 /*******************************************************************************
 621  *
 622  *                          Device Interface Subroutines
 623  *
 624  *******************************************************************************/
 625
 626 /**
 627  *      dev_get_iflink  - get 'iflink' value of a interface
 628  *      @dev: targeted interface
 629  *
 630  *      Indicates the ifindex the interface is linked to.
 631  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
 632  */
 633
 634 int dev_get_iflink(const struct net_device *dev)
 635 {
 636         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
 637                 return dev->netdev_ops->ndo_get_iflink(dev);
 638
 639         return dev->ifindex;
 640 }
 641 EXPORT_SYMBOL(dev_get_iflink);
 642
 643 /**
 644  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
 645  *      @dev: targeted interface
 646  *      @skb: The packet.
 647  *
 648  *      For better visibility of tunnel traffic OVS needs to retrieve
 649  *      egress tunnel information for a packet. Following API allows
 650  *      user to get this info.
 651  */
 652 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 653 {
 654         struct ip_tunnel_info *info;
 655
 656         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
 657                 return -EINVAL;
 658
 659         info = skb_tunnel_info_unclone(skb);
 660         if (!info)
 661                 return -ENOMEM;
 662         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
 663                 return -EINVAL;
 664
 665         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
 666 }
 667 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
 668
 669 static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
 670 {
 671         int k = stack->num_paths++;
 672
 673         if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
 674                 return NULL;
 675
 676         return &stack->path[k];
 677 }
 678
 679 int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
 680                           struct net_device_path_stack *stack)
 681 {
 682         const struct net_device *last_dev;
 683         struct net_device_path_ctx ctx = {
 684                 .dev    = dev,
 685         };
 686         struct net_device_path *path;
 687         int ret = 0;
 688
 689         memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
 690         stack->num_paths = 0;
 691         while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
 692                 last_dev = ctx.dev;
 693                 path = dev_fwd_path(stack);
 694                 if (!path)
 695                         return -1;
 696
 697                 memset(path, 0, sizeof(struct net_device_path));
 698                 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
 699                 if (ret < 0)
 700                         return -1;
 701
 702                 if (WARN_ON_ONCE(last_dev == ctx.dev))
 703                         return -1;
 704         }
 705
 706         if (!ctx.dev)
 707                 return ret;
 708
 709         path = dev_fwd_path(stack);
 710         if (!path)
 711                 return -1;
 712         path->type = DEV_PATH_ETHERNET;
 713         path->dev = ctx.dev;
 714
 715         return ret;
 716 }
 717 EXPORT_SYMBOL_GPL(dev_fill_forward_path);
 718
 719 /**
 720  *      __dev_get_by_name       - find a device by its name
 721  *      @net: the applicable net namespace
 722  *      @name: name to find
 723  *
 724  *      Find an interface by name. Must be called under RTNL semaphore
 725  *      or @dev_base_lock. If the name is found a pointer to the device
 726  *      is returned. If the name is not found then %NULL is returned. The
 727  *      reference counters are not incremented so the caller must be
 728  *      careful with locks.
 729  */
 730
 731 struct net_device *__dev_get_by_name(struct net *net, const char *name)
 732 {
 733         struct netdev_name_node *node_name;
 734
 735         node_name = netdev_name_node_lookup(net, name);
 736         return node_name ? node_name->dev : NULL;
 737 }
 738 EXPORT_SYMBOL(__dev_get_by_name);
 739
 740 /**
 741  * dev_get_by_name_rcu  - find a device by its name
 742  * @net: the applicable net namespace
 743  * @name: name to find
 744  *
 745  * Find an interface by name.
 746  * If the name is found a pointer to the device is returned.
 747  * If the name is not found then %NULL is returned.
 748  * The reference counters are not incremented so the caller must be
 749  * careful with locks. The caller must hold RCU lock.
 750  */
 751
 752 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
 753 {
 754         struct netdev_name_node *node_name;
 755
 756         node_name = netdev_name_node_lookup_rcu(net, name);
 757         return node_name ? node_name->dev : NULL;
 758 }
 759 EXPORT_SYMBOL(dev_get_by_name_rcu);
 760
 761 /**
 762  *      dev_get_by_name         - find a device by its name
 763  *      @net: the applicable net namespace
 764  *      @name: name to find
 765  *
 766  *      Find an interface by name. This can be called from any
 767  *      context and does its own locking. The returned handle has
 768  *      the usage count incremented and the caller must use dev_put() to
 769  *      release it when it is no longer needed. %NULL is returned if no
 770  *      matching device is found.
 771  */
 772
 773 struct net_device *dev_get_by_name(struct net *net, const char *name)
 774 {
 775         struct net_device *dev;
 776
 777         rcu_read_lock();
 778         dev = dev_get_by_name_rcu(net, name);
 779         dev_hold(dev);
 780         rcu_read_unlock();
 781         return dev;
 782 }
 783 EXPORT_SYMBOL(dev_get_by_name);
 784
 785 /**
 786  *      __dev_get_by_index - find a device by its ifindex
 787  *      @net: the applicable net namespace
 788  *      @ifindex: index of device
 789  *
 790  *      Search for an interface by index. Returns %NULL if the device
 791  *      is not found or a pointer to the device. The device has not
 792  *      had its reference counter increased so the caller must be careful
 793  *      about locking. The caller must hold either the RTNL semaphore
 794  *      or @dev_base_lock.
 795  */
 796
 797 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
 798 {
 799         struct net_device *dev;
 800         struct hlist_head *head = dev_index_hash(net, ifindex);
 801
 802         hlist_for_each_entry(dev, head, index_hlist)
 803                 if (dev->ifindex == ifindex)
 804                         return dev;
 805
 806         return NULL;
 807 }
 808 EXPORT_SYMBOL(__dev_get_by_index);
 809
 810 /**
 811  *      dev_get_by_index_rcu - find a device by its ifindex
 812  *      @net: the applicable net namespace
 813  *      @ifindex: index of device
 814  *
 815  *      Search for an interface by index. Returns %NULL if the device
 816  *      is not found or a pointer to the device. The device has not
 817  *      had its reference counter increased so the caller must be careful
 818  *      about locking. The caller must hold RCU lock.
 819  */
 820
 821 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
 822 {
 823         struct net_device *dev;
 824         struct hlist_head *head = dev_index_hash(net, ifindex);
 825
 826         hlist_for_each_entry_rcu(dev, head, index_hlist)
 827                 if (dev->ifindex == ifindex)
 828                         return dev;
 829
 830         return NULL;
 831 }
 832 EXPORT_SYMBOL(dev_get_by_index_rcu);
 833
 834
 835 /**
 836  *      dev_get_by_index - find a device by its ifindex
 837  *      @net: the applicable net namespace
 838  *      @ifindex: index of device
 839  *
 840  *      Search for an interface by index. Returns NULL if the device
 841  *      is not found or a pointer to the device. The device returned has
 842  *      had a reference added and the pointer is safe until the user calls
 843  *      dev_put to indicate they have finished with it.
 844  */
 845
 846 struct net_device *dev_get_by_index(struct net *net, int ifindex)
 847 {
 848         struct net_device *dev;
 849
 850         rcu_read_lock();
 851         dev = dev_get_by_index_rcu(net, ifindex);
 852         dev_hold(dev);
 853         rcu_read_unlock();
 854         return dev;
 855 }
 856 EXPORT_SYMBOL(dev_get_by_index);
 857
 858 /**
 859  *      dev_get_by_napi_id - find a device by napi_id
 860  *      @napi_id: ID of the NAPI struct
 861  *
 862  *      Search for an interface by NAPI ID. Returns %NULL if the device
 863  *      is not found or a pointer to the device. The device has not had
 864  *      its reference counter increased so the caller must be careful
 865  *      about locking. The caller must hold RCU lock.
 866  */
 867
 868 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
 869 {
 870         struct napi_struct *napi;
 871
 872         WARN_ON_ONCE(!rcu_read_lock_held());
 873
 874         if (napi_id < MIN_NAPI_ID)
 875                 return NULL;
 876
 877         napi = napi_by_id(napi_id);
 878
 879         return napi ? napi->dev : NULL;
 880 }
 881 EXPORT_SYMBOL(dev_get_by_napi_id);
 882
 883 /**
 884  *      netdev_get_name - get a netdevice name, knowing its ifindex.
 885  *      @net: network namespace
 886  *      @name: a pointer to the buffer where the name will be stored.
 887  *      @ifindex: the ifindex of the interface to get the name from.
 888  */
 889 int netdev_get_name(struct net *net, char *name, int ifindex)
 890 {
 891         struct net_device *dev;
 892         int ret;
 893
 894         down_read(&devnet_rename_sem);
 895         rcu_read_lock();
 896
 897         dev = dev_get_by_index_rcu(net, ifindex);
 898         if (!dev) {
 899                 ret = -ENODEV;
 900                 goto out;
 901         }
 902
 903         strcpy(name, dev->name);
 904
 905         ret = 0;
 906 out:
 907         rcu_read_unlock();
 908         up_read(&devnet_rename_sem);
 909         return ret;
 910 }
 911
 912 /**
 913  *      dev_getbyhwaddr_rcu - find a device by its hardware address
 914  *      @net: the applicable net namespace
 915  *      @type: media type of device
 916  *      @ha: hardware address
 917  *
 918  *      Search for an interface by MAC address. Returns NULL if the device
 919  *      is not found or a pointer to the device.
 920  *      The caller must hold RCU or RTNL.
 921  *      The returned device has not had its ref count increased
 922  *      and the caller must therefore be careful about locking
 923  *
 924  */
 925
 926 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
 927                                        const char *ha)
 928 {
 929         struct net_device *dev;
 930
 931         for_each_netdev_rcu(net, dev)
 932                 if (dev->type == type &&
 933                     !memcmp(dev->dev_addr, ha, dev->addr_len))
 934                         return dev;
 935
 936         return NULL;
 937 }
 938 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
 939
 940 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
 941 {
 942         struct net_device *dev, *ret = NULL;
 943
 944         rcu_read_lock();
 945         for_each_netdev_rcu(net, dev)
 946                 if (dev->type == type) {
 947                         dev_hold(dev);
 948                         ret = dev;
 949                         break;
 950                 }
 951         rcu_read_unlock();
 952         return ret;
 953 }
 954 EXPORT_SYMBOL(dev_getfirstbyhwtype);
 955
 956 /**
 957  *      __dev_get_by_flags - find any device with given flags
 958  *      @net: the applicable net namespace
 959  *      @if_flags: IFF_* values
 960  *      @mask: bitmask of bits in if_flags to check
 961  *
 962  *      Search for any interface with the given flags. Returns NULL if a device
 963  *      is not found or a pointer to the device. Must be called inside
 964  *      rtnl_lock(), and result refcount is unchanged.
 965  */
 966
 967 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
 968                                       unsigned short mask)
 969 {
 970         struct net_device *dev, *ret;
 971
 972         ASSERT_RTNL();
 973
 974         ret = NULL;
 975         for_each_netdev(net, dev) {
 976                 if (((dev->flags ^ if_flags) & mask) == 0) {
 977                         ret = dev;
 978                         break;
 979                 }
 980         }
 981         return ret;
 982 }
 983 EXPORT_SYMBOL(__dev_get_by_flags);
 984
 985 /**
 986  *      dev_valid_name - check if name is okay for network device
 987  *      @name: name string
 988  *
 989  *      Network device names need to be valid file names to
 990  *      allow sysfs to work.  We also disallow any kind of
 991  *      whitespace.
 992  */
 993 bool dev_valid_name(const char *name)
 994 {
 995         if (*name == '\0')
 996                 return false;
 997         if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
 998                 return false;
 999         if (!strcmp(name, ".") || !strcmp(name, ".."))
1000                 return false;
1001
1002         while (*name) {
1003                 if (*name == '/' || *name == ':' || isspace(*name))
1004                         return false;
1005                 name++;
1006         }
1007         return true;
1008 }
1009 EXPORT_SYMBOL(dev_valid_name);
1010
1011 /**
1012  *      __dev_alloc_name - allocate a name for a device
1013  *      @net: network namespace to allocate the device name in
1014  *      @name: name format string
1015  *      @buf:  scratch buffer and result name string
1016  *
1017  *      Passed a format string - eg "lt%d" it will try and find a suitable
1018  *      id. It scans list of devices to build up a free map, then chooses
1019  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1020  *      while allocating the name and adding the device in order to avoid
1021  *      duplicates.
1022  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1023  *      Returns the number of the unit assigned or a negative errno code.
1024  */
1025
1026 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1027 {
1028         int i = 0;
1029         const char *p;
1030         const int max_netdevices = 8*PAGE_SIZE;
1031         unsigned long *inuse;
1032         struct net_device *d;
1033
1034         if (!dev_valid_name(name))
1035                 return -EINVAL;
1036
1037         p = strchr(name, '%');
1038         if (p) {
1039                 /*
1040                  * Verify the string as this thing may have come from
1041                  * the user.  There must be either one "%d" and no other "%"
1042                  * characters.
1043                  */
1044                 if (p[1] != 'd' || strchr(p + 2, '%'))
1045                         return -EINVAL;
1046
1047                 /* Use one page as a bit array of possible slots */
1048                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1049                 if (!inuse)
1050                         return -ENOMEM;
1051
1052                 for_each_netdev(net, d) {
1053                         struct netdev_name_node *name_node;
1054                         list_for_each_entry(name_node, &d->name_node->list, list) {
1055                                 if (!sscanf(name_node->name, name, &i))
1056                                         continue;
1057                                 if (i < 0 || i >= max_netdevices)
1058                                         continue;
1059
1060                                 /*  avoid cases where sscanf is not exact inverse of printf */
1061                                 snprintf(buf, IFNAMSIZ, name, i);
1062                                 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1063                                         __set_bit(i, inuse);
1064                         }
1065                         if (!sscanf(d->name, name, &i))
1066                                 continue;
1067                         if (i < 0 || i >= max_netdevices)
1068                                 continue;
1069
1070                         /*  avoid cases where sscanf is not exact inverse of printf */
1071                         snprintf(buf, IFNAMSIZ, name, i);
1072                         if (!strncmp(buf, d->name, IFNAMSIZ))
1073                                 __set_bit(i, inuse);
1074                 }
1075
1076                 i = find_first_zero_bit(inuse, max_netdevices);
1077                 free_page((unsigned long) inuse);
1078         }
1079
1080         snprintf(buf, IFNAMSIZ, name, i);
1081         if (!netdev_name_in_use(net, buf))
1082                 return i;
1083
1084         /* It is possible to run out of possible slots
1085          * when the name is long and there isn't enough space left
1086          * for the digits, or if all bits are used.
1087          */
1088         return -ENFILE;
1089 }
1090
1091 static int dev_alloc_name_ns(struct net *net,
1092                              struct net_device *dev,
1093                              const char *name)
1094 {
1095         char buf[IFNAMSIZ];
1096         int ret;
1097
1098         BUG_ON(!net);
1099         ret = __dev_alloc_name(net, name, buf);
1100         if (ret >= 0)
1101                 strscpy(dev->name, buf, IFNAMSIZ);
1102         return ret;
1103 }
1104
1105 /**
1106  *      dev_alloc_name - allocate a name for a device
1107  *      @dev: device
1108  *      @name: name format string
1109  *
1110  *      Passed a format string - eg "lt%d" it will try and find a suitable
1111  *      id. It scans list of devices to build up a free map, then chooses
1112  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1113  *      while allocating the name and adding the device in order to avoid
1114  *      duplicates.
1115  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1116  *      Returns the number of the unit assigned or a negative errno code.
1117  */
1118
1119 int dev_alloc_name(struct net_device *dev, const char *name)
1120 {
1121         return dev_alloc_name_ns(dev_net(dev), dev, name);
1122 }
1123 EXPORT_SYMBOL(dev_alloc_name);
1124
1125 static int dev_get_valid_name(struct net *net, struct net_device *dev,
1126                               const char *name)
1127 {
1128         BUG_ON(!net);
1129
1130         if (!dev_valid_name(name))
1131                 return -EINVAL;
1132
1133         if (strchr(name, '%'))
1134                 return dev_alloc_name_ns(net, dev, name);
1135         else if (netdev_name_in_use(net, name))
1136                 return -EEXIST;
1137         else if (dev->name != name)
1138                 strscpy(dev->name, name, IFNAMSIZ);
1139
1140         return 0;
1141 }
1142
1143 /**
1144  *      dev_change_name - change name of a device
1145  *      @dev: device
1146  *      @newname: name (or format string) must be at least IFNAMSIZ
1147  *
1148  *      Change name of a device, can pass format strings "eth%d".
1149  *      for wildcarding.
1150  */
1151 int dev_change_name(struct net_device *dev, const char *newname)
1152 {
1153         unsigned char old_assign_type;
1154         char oldname[IFNAMSIZ];
1155         int err = 0;
1156         int ret;
1157         struct net *net;
1158
1159         ASSERT_RTNL();
1160         BUG_ON(!dev_net(dev));
1161
1162         net = dev_net(dev);
1163
1164         down_write(&devnet_rename_sem);
1165
1166         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1167                 up_write(&devnet_rename_sem);
1168                 return 0;
1169         }
1170
1171         memcpy(oldname, dev->name, IFNAMSIZ);
1172
1173         err = dev_get_valid_name(net, dev, newname);
1174         if (err < 0) {
1175                 up_write(&devnet_rename_sem);
1176                 return err;
1177         }
1178
1179         if (oldname[0] && !strchr(oldname, '%'))
1180                 netdev_info(dev, "renamed from %s%s\n", oldname,
1181                             dev->flags & IFF_UP ? " (while UP)" : "");
1182
1183         old_assign_type = dev->name_assign_type;
1184         dev->name_assign_type = NET_NAME_RENAMED;
1185
1186 rollback:
1187         ret = device_rename(&dev->dev, dev->name);
1188         if (ret) {
1189                 memcpy(dev->name, oldname, IFNAMSIZ);
1190                 dev->name_assign_type = old_assign_type;
1191                 up_write(&devnet_rename_sem);
1192                 return ret;
1193         }
1194
1195         up_write(&devnet_rename_sem);
1196
1197         netdev_adjacent_rename_links(dev, oldname);
1198
1199         write_lock(&dev_base_lock);
1200         netdev_name_node_del(dev->name_node);
1201         write_unlock(&dev_base_lock);
1202
1203         synchronize_rcu();
1204
1205         write_lock(&dev_base_lock);
1206         netdev_name_node_add(net, dev->name_node);
1207         write_unlock(&dev_base_lock);
1208
1209         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1210         ret = notifier_to_errno(ret);
1211
1212         if (ret) {
1213                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1214                 if (err >= 0) {
1215                         err = ret;
1216                         down_write(&devnet_rename_sem);
1217                         memcpy(dev->name, oldname, IFNAMSIZ);
1218                         memcpy(oldname, newname, IFNAMSIZ);
1219                         dev->name_assign_type = old_assign_type;
1220                         old_assign_type = NET_NAME_RENAMED;
1221                         goto rollback;
1222                 } else {
1223                         netdev_err(dev, "name change rollback failed: %d\n",
1224                                    ret);
1225                 }
1226         }
1227
1228         return err;
1229 }
1230
1231 /**
1232  *      dev_set_alias - change ifalias of a device
1233  *      @dev: device
1234  *      @alias: name up to IFALIASZ
1235  *      @len: limit of bytes to copy from info
1236  *
1237  *      Set ifalias for a device,
1238  */
1239 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1240 {
1241         struct dev_ifalias *new_alias = NULL;
1242
1243         if (len >= IFALIASZ)
1244                 return -EINVAL;
1245
1246         if (len) {
1247                 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1248                 if (!new_alias)
1249                         return -ENOMEM;
1250
1251                 memcpy(new_alias->ifalias, alias, len);
1252                 new_alias->ifalias[len] = 0;
1253         }
1254
1255         mutex_lock(&ifalias_mutex);
1256         new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1257                                         mutex_is_locked(&ifalias_mutex));
1258         mutex_unlock(&ifalias_mutex);
1259
1260         if (new_alias)
1261                 kfree_rcu(new_alias, rcuhead);
1262
1263         return len;
1264 }
1265 EXPORT_SYMBOL(dev_set_alias);
1266
1267 /**
1268  *      dev_get_alias - get ifalias of a device
1269  *      @dev: device
1270  *      @name: buffer to store name of ifalias
1271  *      @len: size of buffer
1272  *
1273  *      get ifalias for a device.  Caller must make sure dev cannot go
1274  *      away,  e.g. rcu read lock or own a reference count to device.
1275  */
1276 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1277 {
1278         const struct dev_ifalias *alias;
1279         int ret = 0;
1280
1281         rcu_read_lock();
1282         alias = rcu_dereference(dev->ifalias);
1283         if (alias)
1284                 ret = snprintf(name, len, "%s", alias->ifalias);
1285         rcu_read_unlock();
1286
1287         return ret;
1288 }
1289
1290 /**
1291  *      netdev_features_change - device changes features
1292  *      @dev: device to cause notification
1293  *
1294  *      Called to indicate a device has changed features.
1295  */
1296 void netdev_features_change(struct net_device *dev)
1297 {
1298         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1299 }
1300 EXPORT_SYMBOL(netdev_features_change);
1301
1302 /**
1303  *      netdev_state_change - device changes state
1304  *      @dev: device to cause notification
1305  *
1306  *      Called to indicate a device has changed state. This function calls
1307  *      the notifier chains for netdev_chain and sends a NEWLINK message
1308  *      to the routing socket.
1309  */
1310 void netdev_state_change(struct net_device *dev)
1311 {
1312         if (dev->flags & IFF_UP) {
1313                 struct netdev_notifier_change_info change_info = {
1314                         .info.dev = dev,
1315                 };
1316
1317                 call_netdevice_notifiers_info(NETDEV_CHANGE,
1318                                               &change_info.info);
1319                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
1320         }
1321 }
1322 EXPORT_SYMBOL(netdev_state_change);
1323
1324 /**
1325  * __netdev_notify_peers - notify network peers about existence of @dev,
1326  * to be called when rtnl lock is already held.
1327  * @dev: network device
1328  *
1329  * Generate traffic such that interested network peers are aware of
1330  * @dev, such as by generating a gratuitous ARP. This may be used when
1331  * a device wants to inform the rest of the network about some sort of
1332  * reconfiguration such as a failover event or virtual machine
1333  * migration.
1334  */
1335 void __netdev_notify_peers(struct net_device *dev)
1336 {
1337         ASSERT_RTNL();
1338         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1339         call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1340 }
1341 EXPORT_SYMBOL(__netdev_notify_peers);
1342
1343 /**
1344  * netdev_notify_peers - notify network peers about existence of @dev
1345  * @dev: network device
1346  *
1347  * Generate traffic such that interested network peers are aware of
1348  * @dev, such as by generating a gratuitous ARP. This may be used when
1349  * a device wants to inform the rest of the network about some sort of
1350  * reconfiguration such as a failover event or virtual machine
1351  * migration.
1352  */
1353 void netdev_notify_peers(struct net_device *dev)
1354 {
1355         rtnl_lock();
1356         __netdev_notify_peers(dev);
1357         rtnl_unlock();
1358 }
1359 EXPORT_SYMBOL(netdev_notify_peers);
1360
1361 static int napi_threaded_poll(void *data);
1362
1363 static int napi_kthread_create(struct napi_struct *n)
1364 {
1365         int err = 0;
1366
1367         /* Create and wake up the kthread once to put it in
1368          * TASK_INTERRUPTIBLE mode to avoid the blocked task
1369          * warning and work with loadavg.
1370          */
1371         n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1372                                 n->dev->name, n->napi_id);
1373         if (IS_ERR(n->thread)) {
1374                 err = PTR_ERR(n->thread);
1375                 pr_err("kthread_run failed with err %d\n", err);
1376                 n->thread = NULL;
1377         }
1378
1379         return err;
1380 }
1381
1382 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1383 {
1384         const struct net_device_ops *ops = dev->netdev_ops;
1385         int ret;
1386
1387         ASSERT_RTNL();
1388         dev_addr_check(dev);
1389
1390         if (!netif_device_present(dev)) {
1391                 /* may be detached because parent is runtime-suspended */
1392                 if (dev->dev.parent)
1393                         pm_runtime_resume(dev->dev.parent);
1394                 if (!netif_device_present(dev))
1395                         return -ENODEV;
1396         }
1397
1398         /* Block netpoll from trying to do any rx path servicing.
1399          * If we don't do this there is a chance ndo_poll_controller
1400          * or ndo_poll may be running while we open the device
1401          */
1402         netpoll_poll_disable(dev);
1403
1404         ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1405         ret = notifier_to_errno(ret);
1406         if (ret)
1407                 return ret;
1408
1409         set_bit(__LINK_STATE_START, &dev->state);
1410
1411         if (ops->ndo_validate_addr)
1412                 ret = ops->ndo_validate_addr(dev);
1413
1414         if (!ret && ops->ndo_open)
1415                 ret = ops->ndo_open(dev);
1416
1417         netpoll_poll_enable(dev);
1418
1419         if (ret)
1420                 clear_bit(__LINK_STATE_START, &dev->state);
1421         else {
1422                 dev->flags |= IFF_UP;
1423                 dev_set_rx_mode(dev);
1424                 dev_activate(dev);
1425                 add_device_randomness(dev->dev_addr, dev->addr_len);
1426         }
1427
1428         return ret;
1429 }
1430
1431 /**
1432  *      dev_open        - prepare an interface for use.
1433  *      @dev: device to open
1434  *      @extack: netlink extended ack
1435  *
1436  *      Takes a device from down to up state. The device's private open
1437  *      function is invoked and then the multicast lists are loaded. Finally
1438  *      the device is moved into the up state and a %NETDEV_UP message is
1439  *      sent to the netdev notifier chain.
1440  *
1441  *      Calling this function on an active interface is a nop. On a failure
1442  *      a negative errno code is returned.
1443  */
1444 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1445 {
1446         int ret;
1447
1448         if (dev->flags & IFF_UP)
1449                 return 0;
1450
1451         ret = __dev_open(dev, extack);
1452         if (ret < 0)
1453                 return ret;
1454
1455         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1456         call_netdevice_notifiers(NETDEV_UP, dev);
1457
1458         return ret;
1459 }
1460 EXPORT_SYMBOL(dev_open);
1461
1462 static void __dev_close_many(struct list_head *head)
1463 {
1464         struct net_device *dev;
1465
1466         ASSERT_RTNL();
1467         might_sleep();
1468
1469         list_for_each_entry(dev, head, close_list) {
1470                 /* Temporarily disable netpoll until the interface is down */
1471                 netpoll_poll_disable(dev);
1472
1473                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1474
1475                 clear_bit(__LINK_STATE_START, &dev->state);
1476
1477                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1478                  * can be even on different cpu. So just clear netif_running().
1479                  *
1480                  * dev->stop() will invoke napi_disable() on all of it's
1481                  * napi_struct instances on this device.
1482                  */
1483                 smp_mb__after_atomic(); /* Commit netif_running(). */
1484         }
1485
1486         dev_deactivate_many(head);
1487
1488         list_for_each_entry(dev, head, close_list) {
1489                 const struct net_device_ops *ops = dev->netdev_ops;
1490
1491                 /*
1492                  *      Call the device specific close. This cannot fail.
1493                  *      Only if device is UP
1494                  *
1495                  *      We allow it to be called even after a DETACH hot-plug
1496                  *      event.
1497                  */
1498                 if (ops->ndo_stop)
1499                         ops->ndo_stop(dev);
1500
1501                 dev->flags &= ~IFF_UP;
1502                 netpoll_poll_enable(dev);
1503         }
1504 }
1505
1506 static void __dev_close(struct net_device *dev)
1507 {
1508         LIST_HEAD(single);
1509
1510         list_add(&dev->close_list, &single);
1511         __dev_close_many(&single);
1512         list_del(&single);
1513 }
1514
1515 void dev_close_many(struct list_head *head, bool unlink)
1516 {
1517         struct net_device *dev, *tmp;
1518
1519         /* Remove the devices that don't need to be closed */
1520         list_for_each_entry_safe(dev, tmp, head, close_list)
1521                 if (!(dev->flags & IFF_UP))
1522                         list_del_init(&dev->close_list);
1523
1524         __dev_close_many(head);
1525
1526         list_for_each_entry_safe(dev, tmp, head, close_list) {
1527                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1528                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1529                 if (unlink)
1530                         list_del_init(&dev->close_list);
1531         }
1532 }
1533 EXPORT_SYMBOL(dev_close_many);
1534
1535 /**
1536  *      dev_close - shutdown an interface.
1537  *      @dev: device to shutdown
1538  *
1539  *      This function moves an active device into down state. A
1540  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1541  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1542  *      chain.
1543  */
1544 void dev_close(struct net_device *dev)
1545 {
1546         if (dev->flags & IFF_UP) {
1547                 LIST_HEAD(single);
1548
1549                 list_add(&dev->close_list, &single);
1550                 dev_close_many(&single, true);
1551                 list_del(&single);
1552         }
1553 }
1554 EXPORT_SYMBOL(dev_close);
1555
1556
1557 /**
1558  *      dev_disable_lro - disable Large Receive Offload on a device
1559  *      @dev: device
1560  *
1561  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1562  *      called under RTNL.  This is needed if received packets may be
1563  *      forwarded to another interface.
1564  */
1565 void dev_disable_lro(struct net_device *dev)
1566 {
1567         struct net_device *lower_dev;
1568         struct list_head *iter;
1569
1570         dev->wanted_features &= ~NETIF_F_LRO;
1571         netdev_update_features(dev);
1572
1573         if (unlikely(dev->features & NETIF_F_LRO))
1574                 netdev_WARN(dev, "failed to disable LRO!\n");
1575
1576         netdev_for_each_lower_dev(dev, lower_dev, iter)
1577                 dev_disable_lro(lower_dev);
1578 }
1579 EXPORT_SYMBOL(dev_disable_lro);
1580
1581 /**
1582  *      dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1583  *      @dev: device
1584  *
1585  *      Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1586  *      called under RTNL.  This is needed if Generic XDP is installed on
1587  *      the device.
1588  */
1589 static void dev_disable_gro_hw(struct net_device *dev)
1590 {
1591         dev->wanted_features &= ~NETIF_F_GRO_HW;
1592         netdev_update_features(dev);
1593
1594         if (unlikely(dev->features & NETIF_F_GRO_HW))
1595                 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1596 }
1597
1598 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1599 {
1600 #define N(val)                                          \
1601         case NETDEV_##val:                              \
1602                 return "NETDEV_" __stringify(val);
1603         switch (cmd) {
1604         N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1605         N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1606         N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1607         N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
1608         N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
1609         N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
1610         N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1611         N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1612         N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1613         N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1614         N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1615         N(XDP_FEAT_CHANGE)
1616         }
1617 #undef N
1618         return "UNKNOWN_NETDEV_EVENT";
1619 }
1620 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1621
1622 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1623                                    struct net_device *dev)
1624 {
1625         struct netdev_notifier_info info = {
1626                 .dev = dev,
1627         };
1628
1629         return nb->notifier_call(nb, val, &info);
1630 }
1631
1632 static int call_netdevice_register_notifiers(struct notifier_block *nb,
1633                                              struct net_device *dev)
1634 {
1635         int err;
1636
1637         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1638         err = notifier_to_errno(err);
1639         if (err)
1640                 return err;
1641
1642         if (!(dev->flags & IFF_UP))
1643                 return 0;
1644
1645         call_netdevice_notifier(nb, NETDEV_UP, dev);
1646         return 0;
1647 }
1648
1649 static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1650                                                 struct net_device *dev)
1651 {
1652         if (dev->flags & IFF_UP) {
1653                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1654                                         dev);
1655                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1656         }
1657         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1658 }
1659
1660 static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1661                                                  struct net *net)
1662 {
1663         struct net_device *dev;
1664         int err;
1665
1666         for_each_netdev(net, dev) {
1667                 err = call_netdevice_register_notifiers(nb, dev);
1668                 if (err)
1669                         goto rollback;
1670         }
1671         return 0;
1672
1673 rollback:
1674         for_each_netdev_continue_reverse(net, dev)
1675                 call_netdevice_unregister_notifiers(nb, dev);
1676         return err;
1677 }
1678
1679 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1680                                                     struct net *net)
1681 {
1682         struct net_device *dev;
1683
1684         for_each_netdev(net, dev)
1685                 call_netdevice_unregister_notifiers(nb, dev);
1686 }
1687
1688 static int dev_boot_phase = 1;
1689
1690 /**
1691  * register_netdevice_notifier - register a network notifier block
1692  * @nb: notifier
1693  *
1694  * Register a notifier to be called when network device events occur.
1695  * The notifier passed is linked into the kernel structures and must
1696  * not be reused until it has been unregistered. A negative errno code
1697  * is returned on a failure.
1698  *
1699  * When registered all registration and up events are replayed
1700  * to the new notifier to allow device to have a race free
1701  * view of the network device list.
1702  */
1703
1704 int register_netdevice_notifier(struct notifier_block *nb)
1705 {
1706         struct net *net;
1707         int err;
1708
1709         /* Close race with setup_net() and cleanup_net() */
1710         down_write(&pernet_ops_rwsem);
1711         rtnl_lock();
1712         err = raw_notifier_chain_register(&netdev_chain, nb);
1713         if (err)
1714                 goto unlock;
1715         if (dev_boot_phase)
1716                 goto unlock;
1717         for_each_net(net) {
1718                 err = call_netdevice_register_net_notifiers(nb, net);
1719                 if (err)
1720                         goto rollback;
1721         }
1722
1723 unlock:
1724         rtnl_unlock();
1725         up_write(&pernet_ops_rwsem);
1726         return err;
1727
1728 rollback:
1729         for_each_net_continue_reverse(net)
1730                 call_netdevice_unregister_net_notifiers(nb, net);
1731
1732         raw_notifier_chain_unregister(&netdev_chain, nb);
1733         goto unlock;
1734 }
1735 EXPORT_SYMBOL(register_netdevice_notifier);
1736
1737 /**
1738  * unregister_netdevice_notifier - unregister a network notifier block
1739  * @nb: notifier
1740  *
1741  * Unregister a notifier previously registered by
1742  * register_netdevice_notifier(). The notifier is unlinked into the
1743  * kernel structures and may then be reused. A negative errno code
1744  * is returned on a failure.
1745  *
1746  * After unregistering unregister and down device events are synthesized
1747  * for all devices on the device list to the removed notifier to remove
1748  * the need for special case cleanup code.
1749  */
1750
1751 int unregister_netdevice_notifier(struct notifier_block *nb)
1752 {
1753         struct net *net;
1754         int err;
1755
1756         /* Close race with setup_net() and cleanup_net() */
1757         down_write(&pernet_ops_rwsem);
1758         rtnl_lock();
1759         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1760         if (err)
1761                 goto unlock;
1762
1763         for_each_net(net)
1764                 call_netdevice_unregister_net_notifiers(nb, net);
1765
1766 unlock:
1767         rtnl_unlock();
1768         up_write(&pernet_ops_rwsem);
1769         return err;
1770 }
1771 EXPORT_SYMBOL(unregister_netdevice_notifier);
1772
1773 static int __register_netdevice_notifier_net(struct net *net,
1774                                              struct notifier_block *nb,
1775                                              bool ignore_call_fail)
1776 {
1777         int err;
1778
1779         err = raw_notifier_chain_register(&net->netdev_chain, nb);
1780         if (err)
1781                 return err;
1782         if (dev_boot_phase)
1783                 return 0;
1784
1785         err = call_netdevice_register_net_notifiers(nb, net);
1786         if (err && !ignore_call_fail)
1787                 goto chain_unregister;
1788
1789         return 0;
1790
1791 chain_unregister:
1792         raw_notifier_chain_unregister(&net->netdev_chain, nb);
1793         return err;
1794 }
1795
1796 static int __unregister_netdevice_notifier_net(struct net *net,
1797                                                struct notifier_block *nb)
1798 {
1799         int err;
1800
1801         err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1802         if (err)
1803                 return err;
1804
1805         call_netdevice_unregister_net_notifiers(nb, net);
1806         return 0;
1807 }
1808
1809 /**
1810  * register_netdevice_notifier_net - register a per-netns network notifier block
1811  * @net: network namespace
1812  * @nb: notifier
1813  *
1814  * Register a notifier to be called when network device events occur.
1815  * The notifier passed is linked into the kernel structures and must
1816  * not be reused until it has been unregistered. A negative errno code
1817  * is returned on a failure.
1818  *
1819  * When registered all registration and up events are replayed
1820  * to the new notifier to allow device to have a race free
1821  * view of the network device list.
1822  */
1823
1824 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1825 {
1826         int err;
1827
1828         rtnl_lock();
1829         err = __register_netdevice_notifier_net(net, nb, false);
1830         rtnl_unlock();
1831         return err;
1832 }
1833 EXPORT_SYMBOL(register_netdevice_notifier_net);
1834
1835 /**
1836  * unregister_netdevice_notifier_net - unregister a per-netns
1837  *                                     network notifier block
1838  * @net: network namespace
1839  * @nb: notifier
1840  *
1841  * Unregister a notifier previously registered by
1842  * register_netdevice_notifier_net(). The notifier is unlinked from the
1843  * kernel structures and may then be reused. A negative errno code
1844  * is returned on a failure.
1845  *
1846  * After unregistering unregister and down device events are synthesized
1847  * for all devices on the device list to the removed notifier to remove
1848  * the need for special case cleanup code.
1849  */
1850
1851 int unregister_netdevice_notifier_net(struct net *net,
1852                                       struct notifier_block *nb)
1853 {
1854         int err;
1855
1856         rtnl_lock();
1857         err = __unregister_netdevice_notifier_net(net, nb);
1858         rtnl_unlock();
1859         return err;
1860 }
1861 EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1862
1863 static void __move_netdevice_notifier_net(struct net *src_net,
1864                                           struct net *dst_net,
1865                                           struct notifier_block *nb)
1866 {
1867         __unregister_netdevice_notifier_net(src_net, nb);
1868         __register_netdevice_notifier_net(dst_net, nb, true);
1869 }
1870
1871 int register_netdevice_notifier_dev_net(struct net_device *dev,
1872                                         struct notifier_block *nb,
1873                                         struct netdev_net_notifier *nn)
1874 {
1875         int err;
1876
1877         rtnl_lock();
1878         err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1879         if (!err) {
1880                 nn->nb = nb;
1881                 list_add(&nn->list, &dev->net_notifier_list);
1882         }
1883         rtnl_unlock();
1884         return err;
1885 }
1886 EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1887
1888 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1889                                           struct notifier_block *nb,
1890                                           struct netdev_net_notifier *nn)
1891 {
1892         int err;
1893
1894         rtnl_lock();
1895         list_del(&nn->list);
1896         err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1897         rtnl_unlock();
1898         return err;
1899 }
1900 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1901
1902 static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1903                                              struct net *net)
1904 {
1905         struct netdev_net_notifier *nn;
1906
1907         list_for_each_entry(nn, &dev->net_notifier_list, list)
1908                 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
1909 }
1910
1911 /**
1912  *      call_netdevice_notifiers_info - call all network notifier blocks
1913  *      @val: value passed unmodified to notifier function
1914  *      @info: notifier information data
1915  *
1916  *      Call all network notifier blocks.  Parameters and return value
1917  *      are as for raw_notifier_call_chain().
1918  */
1919
1920 int call_netdevice_notifiers_info(unsigned long val,
1921                                   struct netdev_notifier_info *info)
1922 {
1923         struct net *net = dev_net(info->dev);
1924         int ret;
1925
1926         ASSERT_RTNL();
1927
1928         /* Run per-netns notifier block chain first, then run the global one.
1929          * Hopefully, one day, the global one is going to be removed after
1930          * all notifier block registrators get converted to be per-netns.
1931          */
1932         ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1933         if (ret & NOTIFY_STOP_MASK)
1934                 return ret;
1935         return raw_notifier_call_chain(&netdev_chain, val, info);
1936 }
1937
1938 /**
1939  *      call_netdevice_notifiers_info_robust - call per-netns notifier blocks
1940  *                                             for and rollback on error
1941  *      @val_up: value passed unmodified to notifier function
1942  *      @val_down: value passed unmodified to the notifier function when
1943  *                 recovering from an error on @val_up
1944  *      @info: notifier information data
1945  *
1946  *      Call all per-netns network notifier blocks, but not notifier blocks on
1947  *      the global notifier chain. Parameters and return value are as for
1948  *      raw_notifier_call_chain_robust().
1949  */
1950
1951 static int
1952 call_netdevice_notifiers_info_robust(unsigned long val_up,
1953                                      unsigned long val_down,
1954                                      struct netdev_notifier_info *info)
1955 {
1956         struct net *net = dev_net(info->dev);
1957
1958         ASSERT_RTNL();
1959
1960         return raw_notifier_call_chain_robust(&net->netdev_chain,
1961                                               val_up, val_down, info);
1962 }
1963
1964 static int call_netdevice_notifiers_extack(unsigned long val,
1965                                            struct net_device *dev,
1966                                            struct netlink_ext_ack *extack)
1967 {
1968         struct netdev_notifier_info info = {
1969                 .dev = dev,
1970                 .extack = extack,
1971         };
1972
1973         return call_netdevice_notifiers_info(val, &info);
1974 }
1975
1976 /**
1977  *      call_netdevice_notifiers - call all network notifier blocks
1978  *      @val: value passed unmodified to notifier function
1979  *      @dev: net_device pointer passed unmodified to notifier function
1980  *
1981  *      Call all network notifier blocks.  Parameters and return value
1982  *      are as for raw_notifier_call_chain().
1983  */
1984
1985 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1986 {
1987         return call_netdevice_notifiers_extack(val, dev, NULL);
1988 }
1989 EXPORT_SYMBOL(call_netdevice_notifiers);
1990
1991 /**
1992  *      call_netdevice_notifiers_mtu - call all network notifier blocks
1993  *      @val: value passed unmodified to notifier function
1994  *      @dev: net_device pointer passed unmodified to notifier function
1995  *      @arg: additional u32 argument passed to the notifier function
1996  *
1997  *      Call all network notifier blocks.  Parameters and return value
1998  *      are as for raw_notifier_call_chain().
1999  */
2000 static int call_netdevice_notifiers_mtu(unsigned long val,
2001                                         struct net_device *dev, u32 arg)
2002 {
2003         struct netdev_notifier_info_ext info = {
2004                 .info.dev = dev,
2005                 .ext.mtu = arg,
2006         };
2007
2008         BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2009
2010         return call_netdevice_notifiers_info(val, &info.info);
2011 }
2012
2013 #ifdef CONFIG_NET_INGRESS
2014 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2015
2016 void net_inc_ingress_queue(void)
2017 {
2018         static_branch_inc(&ingress_needed_key);
2019 }
2020 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2021
2022 void net_dec_ingress_queue(void)
2023 {
2024         static_branch_dec(&ingress_needed_key);
2025 }
2026 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2027 #endif
2028
2029 #ifdef CONFIG_NET_EGRESS
2030 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2031
2032 void net_inc_egress_queue(void)
2033 {
2034         static_branch_inc(&egress_needed_key);
2035 }
2036 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2037
2038 void net_dec_egress_queue(void)
2039 {
2040         static_branch_dec(&egress_needed_key);
2041 }
2042 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2043 #endif
2044
2045 DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2046 EXPORT_SYMBOL(netstamp_needed_key);
2047 #ifdef CONFIG_JUMP_LABEL
2048 static atomic_t netstamp_needed_deferred;
2049 static atomic_t netstamp_wanted;
2050 static void netstamp_clear(struct work_struct *work)
2051 {
2052         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2053         int wanted;
2054
2055         wanted = atomic_add_return(deferred, &netstamp_wanted);
2056         if (wanted > 0)
2057                 static_branch_enable(&netstamp_needed_key);
2058         else
2059                 static_branch_disable(&netstamp_needed_key);
2060 }
2061 static DECLARE_WORK(netstamp_work, netstamp_clear);
2062 #endif
2063
2064 void net_enable_timestamp(void)
2065 {
2066 #ifdef CONFIG_JUMP_LABEL
2067         int wanted = atomic_read(&netstamp_wanted);
2068
2069         while (wanted > 0) {
2070                 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
2071                         return;
2072         }
2073         atomic_inc(&netstamp_needed_deferred);
2074         schedule_work(&netstamp_work);
2075 #else
2076         static_branch_inc(&netstamp_needed_key);
2077 #endif
2078 }
2079 EXPORT_SYMBOL(net_enable_timestamp);
2080
2081 void net_disable_timestamp(void)
2082 {
2083 #ifdef CONFIG_JUMP_LABEL
2084         int wanted = atomic_read(&netstamp_wanted);
2085
2086         while (wanted > 1) {
2087                 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
2088                         return;
2089         }
2090         atomic_dec(&netstamp_needed_deferred);
2091         schedule_work(&netstamp_work);
2092 #else
2093         static_branch_dec(&netstamp_needed_key);
2094 #endif
2095 }
2096 EXPORT_SYMBOL(net_disable_timestamp);
2097
2098 static inline void net_timestamp_set(struct sk_buff *skb)
2099 {
2100         skb->tstamp = 0;
2101         skb->mono_delivery_time = 0;
2102         if (static_branch_unlikely(&netstamp_needed_key))
2103                 skb->tstamp = ktime_get_real();
2104 }
2105
2106 #define net_timestamp_check(COND, SKB)                          \
2107         if (static_branch_unlikely(&netstamp_needed_key)) {     \
2108                 if ((COND) && !(SKB)->tstamp)                   \
2109                         (SKB)->tstamp = ktime_get_real();       \
2110         }                                                       \
2111
2112 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2113 {
2114         return __is_skb_forwardable(dev, skb, true);
2115 }
2116 EXPORT_SYMBOL_GPL(is_skb_forwardable);
2117
2118 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2119                               bool check_mtu)
2120 {
2121         int ret = ____dev_forward_skb(dev, skb, check_mtu);
2122
2123         if (likely(!ret)) {
2124                 skb->protocol = eth_type_trans(skb, dev);
2125                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2126         }
2127
2128         return ret;
2129 }
2130
2131 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2132 {
2133         return __dev_forward_skb2(dev, skb, true);
2134 }
2135 EXPORT_SYMBOL_GPL(__dev_forward_skb);
2136
2137 /**
2138  * dev_forward_skb - loopback an skb to another netif
2139  *
2140  * @dev: destination network device
2141  * @skb: buffer to forward
2142  *
2143  * return values:
2144  *      NET_RX_SUCCESS  (no congestion)
2145  *      NET_RX_DROP     (packet was dropped, but freed)
2146  *
2147  * dev_forward_skb can be used for injecting an skb from the
2148  * start_xmit function of one device into the receive queue
2149  * of another device.
2150  *
2151  * The receiving device may be in another namespace, so
2152  * we have to clear all information in the skb that could
2153  * impact namespace isolation.
2154  */
2155 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2156 {
2157         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2158 }
2159 EXPORT_SYMBOL_GPL(dev_forward_skb);
2160
2161 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2162 {
2163         return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2164 }
2165
2166 static inline int deliver_skb(struct sk_buff *skb,
2167                               struct packet_type *pt_prev,
2168                               struct net_device *orig_dev)
2169 {
2170         if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2171                 return -ENOMEM;
2172         refcount_inc(&skb->users);
2173         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2174 }
2175
2176 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2177                                           struct packet_type **pt,
2178                                           struct net_device *orig_dev,
2179                                           __be16 type,
2180                                           struct list_head *ptype_list)
2181 {
2182         struct packet_type *ptype, *pt_prev = *pt;
2183
2184         list_for_each_entry_rcu(ptype, ptype_list, list) {
2185                 if (ptype->type != type)
2186                         continue;
2187                 if (pt_prev)
2188                         deliver_skb(skb, pt_prev, orig_dev);
2189                 pt_prev = ptype;
2190         }
2191         *pt = pt_prev;
2192 }
2193
2194 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2195 {
2196         if (!ptype->af_packet_priv || !skb->sk)
2197                 return false;
2198
2199         if (ptype->id_match)
2200                 return ptype->id_match(ptype, skb->sk);
2201         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2202                 return true;
2203
2204         return false;
2205 }
2206
2207 /**
2208  * dev_nit_active - return true if any network interface taps are in use
2209  *
2210  * @dev: network device to check for the presence of taps
2211  */
2212 bool dev_nit_active(struct net_device *dev)
2213 {
2214         return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2215 }
2216 EXPORT_SYMBOL_GPL(dev_nit_active);
2217
2218 /*
2219  *      Support routine. Sends outgoing frames to any network
2220  *      taps currently in use.
2221  */
2222
2223 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2224 {
2225         struct packet_type *ptype;
2226         struct sk_buff *skb2 = NULL;
2227         struct packet_type *pt_prev = NULL;
2228         struct list_head *ptype_list = &ptype_all;
2229
2230         rcu_read_lock();
2231 again:
2232         list_for_each_entry_rcu(ptype, ptype_list, list) {
2233                 if (ptype->ignore_outgoing)
2234                         continue;
2235
2236                 /* Never send packets back to the socket
2237                  * they originated from - MvS (miquels@drinkel.ow.org)
2238                  */
2239                 if (skb_loop_sk(ptype, skb))
2240                         continue;
2241
2242                 if (pt_prev) {
2243                         deliver_skb(skb2, pt_prev, skb->dev);
2244                         pt_prev = ptype;
2245                         continue;
2246                 }
2247
2248                 /* need to clone skb, done only once */
2249                 skb2 = skb_clone(skb, GFP_ATOMIC);
2250                 if (!skb2)
2251                         goto out_unlock;
2252
2253                 net_timestamp_set(skb2);
2254
2255                 /* skb->nh should be correctly
2256                  * set by sender, so that the second statement is
2257                  * just protection against buggy protocols.
2258                  */
2259                 skb_reset_mac_header(skb2);
2260
2261                 if (skb_network_header(skb2) < skb2->data ||
2262                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2263                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2264                                              ntohs(skb2->protocol),
2265                                              dev->name);
2266                         skb_reset_network_header(skb2);
2267                 }
2268
2269                 skb2->transport_header = skb2->network_header;
2270                 skb2->pkt_type = PACKET_OUTGOING;
2271                 pt_prev = ptype;
2272         }
2273
2274         if (ptype_list == &ptype_all) {
2275                 ptype_list = &dev->ptype_all;
2276                 goto again;
2277         }
2278 out_unlock:
2279         if (pt_prev) {
2280                 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2281                         pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2282                 else
2283                         kfree_skb(skb2);
2284         }
2285         rcu_read_unlock();
2286 }
2287 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2288
2289 /**
2290  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2291  * @dev: Network device
2292  * @txq: number of queues available
2293  *
2294  * If real_num_tx_queues is changed the tc mappings may no longer be
2295  * valid. To resolve this verify the tc mapping remains valid and if
2296  * not NULL the mapping. With no priorities mapping to this
2297  * offset/count pair it will no longer be used. In the worst case TC0
2298  * is invalid nothing can be done so disable priority mappings. If is
2299  * expected that drivers will fix this mapping if they can before
2300  * calling netif_set_real_num_tx_queues.
2301  */
2302 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2303 {
2304         int i;
2305         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2306
2307         /* If TC0 is invalidated disable TC mapping */
2308         if (tc->offset + tc->count > txq) {
2309                 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2310                 dev->num_tc = 0;
2311                 return;
2312         }
2313
2314         /* Invalidated prio to tc mappings set to TC0 */
2315         for (i = 1; i < TC_BITMASK + 1; i++) {
2316                 int q = netdev_get_prio_tc_map(dev, i);
2317
2318                 tc = &dev->tc_to_txq[q];
2319                 if (tc->offset + tc->count > txq) {
2320                         netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2321                                     i, q);
2322                         netdev_set_prio_tc_map(dev, i, 0);
2323                 }
2324         }
2325 }
2326
2327 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2328 {
2329         if (dev->num_tc) {
2330                 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2331                 int i;
2332
2333                 /* walk through the TCs and see if it falls into any of them */
2334                 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2335                         if ((txq - tc->offset) < tc->count)
2336                                 return i;
2337                 }
2338
2339                 /* didn't find it, just return -1 to indicate no match */
2340                 return -1;
2341         }
2342
2343         return 0;
2344 }
2345 EXPORT_SYMBOL(netdev_txq_to_tc);
2346
2347 #ifdef CONFIG_XPS
2348 static struct static_key xps_needed __read_mostly;
2349 static struct static_key xps_rxqs_needed __read_mostly;
2350 static DEFINE_MUTEX(xps_map_mutex);
2351 #define xmap_dereference(P)             \
2352         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2353
2354 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2355                              struct xps_dev_maps *old_maps, int tci, u16 index)
2356 {
2357         struct xps_map *map = NULL;
2358         int pos;
2359
2360         if (dev_maps)
2361                 map = xmap_dereference(dev_maps->attr_map[tci]);
2362         if (!map)
2363                 return false;
2364
2365         for (pos = map->len; pos--;) {
2366                 if (map->queues[pos] != index)
2367                         continue;
2368
2369                 if (map->len > 1) {
2370                         map->queues[pos] = map->queues[--map->len];
2371                         break;
2372                 }
2373
2374                 if (old_maps)
2375                         RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2376                 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2377                 kfree_rcu(map, rcu);
2378                 return false;
2379         }
2380
2381         return true;
2382 }
2383
2384 static bool remove_xps_queue_cpu(struct net_device *dev,
2385                                  struct xps_dev_maps *dev_maps,
2386                                  int cpu, u16 offset, u16 count)
2387 {
2388         int num_tc = dev_maps->num_tc;
2389         bool active = false;
2390         int tci;
2391
2392         for (tci = cpu * num_tc; num_tc--; tci++) {
2393                 int i, j;
2394
2395                 for (i = count, j = offset; i--; j++) {
2396                         if (!remove_xps_queue(dev_maps, NULL, tci, j))
2397                                 break;
2398                 }
2399
2400                 active |= i < 0;
2401         }
2402
2403         return active;
2404 }
2405
2406 static void reset_xps_maps(struct net_device *dev,
2407                            struct xps_dev_maps *dev_maps,
2408                            enum xps_map_type type)
2409 {
2410         static_key_slow_dec_cpuslocked(&xps_needed);
2411         if (type == XPS_RXQS)
2412                 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2413
2414         RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2415
2416         kfree_rcu(dev_maps, rcu);
2417 }
2418
2419 static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2420                            u16 offset, u16 count)
2421 {
2422         struct xps_dev_maps *dev_maps;
2423         bool active = false;
2424         int i, j;
2425
2426         dev_maps = xmap_dereference(dev->xps_maps[type]);
2427         if (!dev_maps)
2428                 return;
2429
2430         for (j = 0; j < dev_maps->nr_ids; j++)
2431                 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2432         if (!active)
2433                 reset_xps_maps(dev, dev_maps, type);
2434
2435         if (type == XPS_CPUS) {
2436                 for (i = offset + (count - 1); count--; i--)
2437                         netdev_queue_numa_node_write(
2438                                 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2439         }
2440 }
2441
2442 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2443                                    u16 count)
2444 {
2445         if (!static_key_false(&xps_needed))
2446                 return;
2447
2448         cpus_read_lock();
2449         mutex_lock(&xps_map_mutex);
2450
2451         if (static_key_false(&xps_rxqs_needed))
2452                 clean_xps_maps(dev, XPS_RXQS, offset, count);
2453
2454         clean_xps_maps(dev, XPS_CPUS, offset, count);
2455
2456         mutex_unlock(&xps_map_mutex);
2457         cpus_read_unlock();
2458 }
2459
2460 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2461 {
2462         netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2463 }
2464
2465 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2466                                       u16 index, bool is_rxqs_map)
2467 {
2468         struct xps_map *new_map;
2469         int alloc_len = XPS_MIN_MAP_ALLOC;
2470         int i, pos;
2471
2472         for (pos = 0; map && pos < map->len; pos++) {
2473                 if (map->queues[pos] != index)
2474                         continue;
2475                 return map;
2476         }
2477
2478         /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2479         if (map) {
2480                 if (pos < map->alloc_len)
2481                         return map;
2482
2483                 alloc_len = map->alloc_len * 2;
2484         }
2485
2486         /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2487          *  map
2488          */
2489         if (is_rxqs_map)
2490                 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2491         else
2492                 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2493                                        cpu_to_node(attr_index));
2494         if (!new_map)
2495                 return NULL;
2496
2497         for (i = 0; i < pos; i++)
2498                 new_map->queues[i] = map->queues[i];
2499         new_map->alloc_len = alloc_len;
2500         new_map->len = pos;
2501
2502         return new_map;
2503 }
2504
2505 /* Copy xps maps at a given index */
2506 static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2507                               struct xps_dev_maps *new_dev_maps, int index,
2508                               int tc, bool skip_tc)
2509 {
2510         int i, tci = index * dev_maps->num_tc;
2511         struct xps_map *map;
2512
2513         /* copy maps belonging to foreign traffic classes */
2514         for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2515                 if (i == tc && skip_tc)
2516                         continue;
2517
2518                 /* fill in the new device map from the old device map */
2519                 map = xmap_dereference(dev_maps->attr_map[tci]);
2520                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2521         }
2522 }
2523
2524 /* Must be called under cpus_read_lock */
2525 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2526                           u16 index, enum xps_map_type type)
2527 {
2528         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2529         const unsigned long *online_mask = NULL;
2530         bool active = false, copy = false;
2531         int i, j, tci, numa_node_id = -2;
2532         int maps_sz, num_tc = 1, tc = 0;
2533         struct xps_map *map, *new_map;
2534         unsigned int nr_ids;
2535
2536         WARN_ON_ONCE(index >= dev->num_tx_queues);
2537
2538         if (dev->num_tc) {
2539                 /* Do not allow XPS on subordinate device directly */
2540                 num_tc = dev->num_tc;
2541                 if (num_tc < 0)
2542                         return -EINVAL;
2543
2544                 /* If queue belongs to subordinate dev use its map */
2545                 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2546
2547                 tc = netdev_txq_to_tc(dev, index);
2548                 if (tc < 0)
2549                         return -EINVAL;
2550         }
2551
2552         mutex_lock(&xps_map_mutex);
2553
2554         dev_maps = xmap_dereference(dev->xps_maps[type]);
2555         if (type == XPS_RXQS) {
2556                 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2557                 nr_ids = dev->num_rx_queues;
2558         } else {
2559                 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2560                 if (num_possible_cpus() > 1)
2561                         online_mask = cpumask_bits(cpu_online_mask);
2562                 nr_ids = nr_cpu_ids;
2563         }
2564
2565         if (maps_sz < L1_CACHE_BYTES)
2566                 maps_sz = L1_CACHE_BYTES;
2567
2568         /* The old dev_maps could be larger or smaller than the one we're
2569          * setting up now, as dev->num_tc or nr_ids could have been updated in
2570          * between. We could try to be smart, but let's be safe instead and only
2571          * copy foreign traffic classes if the two map sizes match.
2572          */
2573         if (dev_maps &&
2574             dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2575                 copy = true;
2576
2577         /* allocate memory for queue storage */
2578         for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2579              j < nr_ids;) {
2580                 if (!new_dev_maps) {
2581                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2582                         if (!new_dev_maps) {
2583                                 mutex_unlock(&xps_map_mutex);
2584                                 return -ENOMEM;
2585                         }
2586
2587                         new_dev_maps->nr_ids = nr_ids;
2588                         new_dev_maps->num_tc = num_tc;
2589                 }
2590
2591                 tci = j * num_tc + tc;
2592                 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2593
2594                 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2595                 if (!map)
2596                         goto error;
2597
2598                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2599         }
2600
2601         if (!new_dev_maps)
2602                 goto out_no_new_maps;
2603
2604         if (!dev_maps) {
2605                 /* Increment static keys at most once per type */
2606                 static_key_slow_inc_cpuslocked(&xps_needed);
2607                 if (type == XPS_RXQS)
2608                         static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2609         }
2610
2611         for (j = 0; j < nr_ids; j++) {
2612                 bool skip_tc = false;
2613
2614                 tci = j * num_tc + tc;
2615                 if (netif_attr_test_mask(j, mask, nr_ids) &&
2616                     netif_attr_test_online(j, online_mask, nr_ids)) {
2617                         /* add tx-queue to CPU/rx-queue maps */
2618                         int pos = 0;
2619
2620                         skip_tc = true;
2621
2622                         map = xmap_dereference(new_dev_maps->attr_map[tci]);
2623                         while ((pos < map->len) && (map->queues[pos] != index))
2624                                 pos++;
2625
2626                         if (pos == map->len)
2627                                 map->queues[map->len++] = index;
2628 #ifdef CONFIG_NUMA
2629                         if (type == XPS_CPUS) {
2630                                 if (numa_node_id == -2)
2631                                         numa_node_id = cpu_to_node(j);
2632                                 else if (numa_node_id != cpu_to_node(j))
2633                                         numa_node_id = -1;
2634                         }
2635 #endif
2636                 }
2637
2638                 if (copy)
2639                         xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2640                                           skip_tc);
2641         }
2642
2643         rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2644
2645         /* Cleanup old maps */
2646         if (!dev_maps)
2647                 goto out_no_old_maps;
2648
2649         for (j = 0; j < dev_maps->nr_ids; j++) {
2650                 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2651                         map = xmap_dereference(dev_maps->attr_map[tci]);
2652                         if (!map)
2653                                 continue;
2654
2655                         if (copy) {
2656                                 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2657                                 if (map == new_map)
2658                                         continue;
2659                         }
2660
2661                         RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2662                         kfree_rcu(map, rcu);
2663                 }
2664         }
2665
2666         old_dev_maps = dev_maps;
2667
2668 out_no_old_maps:
2669         dev_maps = new_dev_maps;
2670         active = true;
2671
2672 out_no_new_maps:
2673         if (type == XPS_CPUS)
2674                 /* update Tx queue numa node */
2675                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2676                                              (numa_node_id >= 0) ?
2677                                              numa_node_id : NUMA_NO_NODE);
2678
2679         if (!dev_maps)
2680                 goto out_no_maps;
2681
2682         /* removes tx-queue from unused CPUs/rx-queues */
2683         for (j = 0; j < dev_maps->nr_ids; j++) {
2684                 tci = j * dev_maps->num_tc;
2685
2686                 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2687                         if (i == tc &&
2688                             netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2689                             netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2690                                 continue;
2691
2692                         active |= remove_xps_queue(dev_maps,
2693                                                    copy ? old_dev_maps : NULL,
2694                                                    tci, index);
2695                 }
2696         }
2697
2698         if (old_dev_maps)
2699                 kfree_rcu(old_dev_maps, rcu);
2700
2701         /* free map if not active */
2702         if (!active)
2703                 reset_xps_maps(dev, dev_maps, type);
2704
2705 out_no_maps:
2706         mutex_unlock(&xps_map_mutex);
2707
2708         return 0;
2709 error:
2710         /* remove any maps that we added */
2711         for (j = 0; j < nr_ids; j++) {
2712                 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2713                         new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2714                         map = copy ?
2715                               xmap_dereference(dev_maps->attr_map[tci]) :
2716                               NULL;
2717                         if (new_map && new_map != map)
2718                                 kfree(new_map);
2719                 }
2720         }
2721
2722         mutex_unlock(&xps_map_mutex);
2723
2724         kfree(new_dev_maps);
2725         return -ENOMEM;
2726 }
2727 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2728
2729 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2730                         u16 index)
2731 {
2732         int ret;
2733
2734         cpus_read_lock();
2735         ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2736         cpus_read_unlock();
2737
2738         return ret;
2739 }
2740 EXPORT_SYMBOL(netif_set_xps_queue);
2741
2742 #endif
2743 static void netdev_unbind_all_sb_channels(struct net_device *dev)
2744 {
2745         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2746
2747         /* Unbind any subordinate channels */
2748         while (txq-- != &dev->_tx[0]) {
2749                 if (txq->sb_dev)
2750                         netdev_unbind_sb_channel(dev, txq->sb_dev);
2751         }
2752 }
2753
2754 void netdev_reset_tc(struct net_device *dev)
2755 {
2756 #ifdef CONFIG_XPS
2757         netif_reset_xps_queues_gt(dev, 0);
2758 #endif
2759         netdev_unbind_all_sb_channels(dev);
2760
2761         /* Reset TC configuration of device */
2762         dev->num_tc = 0;
2763         memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2764         memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2765 }
2766 EXPORT_SYMBOL(netdev_reset_tc);
2767
2768 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2769 {
2770         if (tc >= dev->num_tc)
2771                 return -EINVAL;
2772
2773 #ifdef CONFIG_XPS
2774         netif_reset_xps_queues(dev, offset, count);
2775 #endif
2776         dev->tc_to_txq[tc].count = count;
2777         dev->tc_to_txq[tc].offset = offset;
2778         return 0;
2779 }
2780 EXPORT_SYMBOL(netdev_set_tc_queue);
2781
2782 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2783 {
2784         if (num_tc > TC_MAX_QUEUE)
2785                 return -EINVAL;
2786
2787 #ifdef CONFIG_XPS
2788         netif_reset_xps_queues_gt(dev, 0);
2789 #endif
2790         netdev_unbind_all_sb_channels(dev);
2791
2792         dev->num_tc = num_tc;
2793         return 0;
2794 }
2795 EXPORT_SYMBOL(netdev_set_num_tc);
2796
2797 void netdev_unbind_sb_channel(struct net_device *dev,
2798                               struct net_device *sb_dev)
2799 {
2800         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2801
2802 #ifdef CONFIG_XPS
2803         netif_reset_xps_queues_gt(sb_dev, 0);
2804 #endif
2805         memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2806         memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2807
2808         while (txq-- != &dev->_tx[0]) {
2809                 if (txq->sb_dev == sb_dev)
2810                         txq->sb_dev = NULL;
2811         }
2812 }
2813 EXPORT_SYMBOL(netdev_unbind_sb_channel);
2814
2815 int netdev_bind_sb_channel_queue(struct net_device *dev,
2816                                  struct net_device *sb_dev,
2817                                  u8 tc, u16 count, u16 offset)
2818 {
2819         /* Make certain the sb_dev and dev are already configured */
2820         if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2821                 return -EINVAL;
2822
2823         /* We cannot hand out queues we don't have */
2824         if ((offset + count) > dev->real_num_tx_queues)
2825                 return -EINVAL;
2826
2827         /* Record the mapping */
2828         sb_dev->tc_to_txq[tc].count = count;
2829         sb_dev->tc_to_txq[tc].offset = offset;
2830
2831         /* Provide a way for Tx queue to find the tc_to_txq map or
2832          * XPS map for itself.
2833          */
2834         while (count--)
2835                 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2836
2837         return 0;
2838 }
2839 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2840
2841 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2842 {
2843         /* Do not use a multiqueue device to represent a subordinate channel */
2844         if (netif_is_multiqueue(dev))
2845                 return -ENODEV;
2846
2847         /* We allow channels 1 - 32767 to be used for subordinate channels.
2848          * Channel 0 is meant to be "native" mode and used only to represent
2849          * the main root device. We allow writing 0 to reset the device back
2850          * to normal mode after being used as a subordinate channel.
2851          */
2852         if (channel > S16_MAX)
2853                 return -EINVAL;
2854
2855         dev->num_tc = -channel;
2856
2857         return 0;
2858 }
2859 EXPORT_SYMBOL(netdev_set_sb_channel);
2860
2861 /*
2862  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2863  * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2864  */
2865 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2866 {
2867         bool disabling;
2868         int rc;
2869
2870         disabling = txq < dev->real_num_tx_queues;
2871
2872         if (txq < 1 || txq > dev->num_tx_queues)
2873                 return -EINVAL;
2874
2875         if (dev->reg_state == NETREG_REGISTERED ||
2876             dev->reg_state == NETREG_UNREGISTERING) {
2877                 ASSERT_RTNL();
2878
2879                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2880                                                   txq);
2881                 if (rc)
2882                         return rc;
2883
2884                 if (dev->num_tc)
2885                         netif_setup_tc(dev, txq);
2886
2887                 dev_qdisc_change_real_num_tx(dev, txq);
2888
2889                 dev->real_num_tx_queues = txq;
2890
2891                 if (disabling) {
2892                         synchronize_net();
2893                         qdisc_reset_all_tx_gt(dev, txq);
2894 #ifdef CONFIG_XPS
2895                         netif_reset_xps_queues_gt(dev, txq);
2896 #endif
2897                 }
2898         } else {
2899                 dev->real_num_tx_queues = txq;
2900         }
2901
2902         return 0;
2903 }
2904 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2905
2906 #ifdef CONFIG_SYSFS
2907 /**
2908  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2909  *      @dev: Network device
2910  *      @rxq: Actual number of RX queues
2911  *
2912  *      This must be called either with the rtnl_lock held or before
2913  *      registration of the net device.  Returns 0 on success, or a
2914  *      negative error code.  If called before registration, it always
2915  *      succeeds.
2916  */
2917 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2918 {
2919         int rc;
2920
2921         if (rxq < 1 || rxq > dev->num_rx_queues)
2922                 return -EINVAL;
2923
2924         if (dev->reg_state == NETREG_REGISTERED) {
2925                 ASSERT_RTNL();
2926
2927                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2928                                                   rxq);
2929                 if (rc)
2930                         return rc;
2931         }
2932
2933         dev->real_num_rx_queues = rxq;
2934         return 0;
2935 }
2936 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2937 #endif
2938
2939 /**
2940  *      netif_set_real_num_queues - set actual number of RX and TX queues used
2941  *      @dev: Network device
2942  *      @txq: Actual number of TX queues
2943  *      @rxq: Actual number of RX queues
2944  *
2945  *      Set the real number of both TX and RX queues.
2946  *      Does nothing if the number of queues is already correct.
2947  */
2948 int netif_set_real_num_queues(struct net_device *dev,
2949                               unsigned int txq, unsigned int rxq)
2950 {
2951         unsigned int old_rxq = dev->real_num_rx_queues;
2952         int err;
2953
2954         if (txq < 1 || txq > dev->num_tx_queues ||
2955             rxq < 1 || rxq > dev->num_rx_queues)
2956                 return -EINVAL;
2957
2958         /* Start from increases, so the error path only does decreases -
2959          * decreases can't fail.
2960          */
2961         if (rxq > dev->real_num_rx_queues) {
2962                 err = netif_set_real_num_rx_queues(dev, rxq);
2963                 if (err)
2964                         return err;
2965         }
2966         if (txq > dev->real_num_tx_queues) {
2967                 err = netif_set_real_num_tx_queues(dev, txq);
2968                 if (err)
2969                         goto undo_rx;
2970         }
2971         if (rxq < dev->real_num_rx_queues)
2972                 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
2973         if (txq < dev->real_num_tx_queues)
2974                 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
2975
2976         return 0;
2977 undo_rx:
2978         WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
2979         return err;
2980 }
2981 EXPORT_SYMBOL(netif_set_real_num_queues);
2982
2983 /**
2984  * netif_set_tso_max_size() - set the max size of TSO frames supported
2985  * @dev:        netdev to update
2986  * @size:       max skb->len of a TSO frame
2987  *
2988  * Set the limit on the size of TSO super-frames the device can handle.
2989  * Unless explicitly set the stack will assume the value of
2990  * %GSO_LEGACY_MAX_SIZE.
2991  */
2992 void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
2993 {
2994         dev->tso_max_size = min(GSO_MAX_SIZE, size);
2995         if (size < READ_ONCE(dev->gso_max_size))
2996                 netif_set_gso_max_size(dev, size);
2997         if (size < READ_ONCE(dev->gso_ipv4_max_size))
2998                 netif_set_gso_ipv4_max_size(dev, size);
2999 }
3000 EXPORT_SYMBOL(netif_set_tso_max_size);
3001
3002 /**
3003  * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3004  * @dev:        netdev to update
3005  * @segs:       max number of TCP segments
3006  *
3007  * Set the limit on the number of TCP segments the device can generate from
3008  * a single TSO super-frame.
3009  * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3010  */
3011 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3012 {
3013         dev->tso_max_segs = segs;
3014         if (segs < READ_ONCE(dev->gso_max_segs))
3015                 netif_set_gso_max_segs(dev, segs);
3016 }
3017 EXPORT_SYMBOL(netif_set_tso_max_segs);
3018
3019 /**
3020  * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3021  * @to:         netdev to update
3022  * @from:       netdev from which to copy the limits
3023  */
3024 void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3025 {
3026         netif_set_tso_max_size(to, from->tso_max_size);
3027         netif_set_tso_max_segs(to, from->tso_max_segs);
3028 }
3029 EXPORT_SYMBOL(netif_inherit_tso_max);
3030
3031 /**
3032  * netif_get_num_default_rss_queues - default number of RSS queues
3033  *
3034  * Default value is the number of physical cores if there are only 1 or 2, or
3035  * divided by 2 if there are more.
3036  */
3037 int netif_get_num_default_rss_queues(void)
3038 {
3039         cpumask_var_t cpus;
3040         int cpu, count = 0;
3041
3042         if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3043                 return 1;
3044
3045         cpumask_copy(cpus, cpu_online_mask);
3046         for_each_cpu(cpu, cpus) {
3047                 ++count;
3048                 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3049         }
3050         free_cpumask_var(cpus);
3051
3052         return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3053 }
3054 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3055
3056 static void __netif_reschedule(struct Qdisc *q)
3057 {
3058         struct softnet_data *sd;
3059         unsigned long flags;
3060
3061         local_irq_save(flags);
3062         sd = this_cpu_ptr(&softnet_data);
3063         q->next_sched = NULL;
3064         *sd->output_queue_tailp = q;
3065         sd->output_queue_tailp = &q->next_sched;
3066         raise_softirq_irqoff(NET_TX_SOFTIRQ);
3067         local_irq_restore(flags);
3068 }
3069
3070 void __netif_schedule(struct Qdisc *q)
3071 {
3072         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3073                 __netif_reschedule(q);
3074 }
3075 EXPORT_SYMBOL(__netif_schedule);
3076
3077 struct dev_kfree_skb_cb {
3078         enum skb_drop_reason reason;
3079 };
3080
3081 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3082 {
3083         return (struct dev_kfree_skb_cb *)skb->cb;
3084 }
3085
3086 void netif_schedule_queue(struct netdev_queue *txq)
3087 {
3088         rcu_read_lock();
3089         if (!netif_xmit_stopped(txq)) {
3090                 struct Qdisc *q = rcu_dereference(txq->qdisc);
3091
3092                 __netif_schedule(q);
3093         }
3094         rcu_read_unlock();
3095 }
3096 EXPORT_SYMBOL(netif_schedule_queue);
3097
3098 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3099 {
3100         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3101                 struct Qdisc *q;
3102
3103                 rcu_read_lock();
3104                 q = rcu_dereference(dev_queue->qdisc);
3105                 __netif_schedule(q);
3106                 rcu_read_unlock();
3107         }
3108 }
3109 EXPORT_SYMBOL(netif_tx_wake_queue);
3110
3111 void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3112 {
3113         unsigned long flags;
3114
3115         if (unlikely(!skb))
3116                 return;
3117
3118         if (likely(refcount_read(&skb->users) == 1)) {
3119                 smp_rmb();
3120                 refcount_set(&skb->users, 0);
3121         } else if (likely(!refcount_dec_and_test(&skb->users))) {
3122                 return;
3123         }
3124         get_kfree_skb_cb(skb)->reason = reason;
3125         local_irq_save(flags);
3126         skb->next = __this_cpu_read(softnet_data.completion_queue);
3127         __this_cpu_write(softnet_data.completion_queue, skb);
3128         raise_softirq_irqoff(NET_TX_SOFTIRQ);
3129         local_irq_restore(flags);
3130 }
3131 EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
3132
3133 void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3134 {
3135         if (in_hardirq() || irqs_disabled())
3136                 dev_kfree_skb_irq_reason(skb, reason);
3137         else
3138                 kfree_skb_reason(skb, reason);
3139 }
3140 EXPORT_SYMBOL(dev_kfree_skb_any_reason);
3141
3142
3143 /**
3144  * netif_device_detach - mark device as removed
3145  * @dev: network device
3146  *
3147  * Mark device as removed from system and therefore no longer available.
3148  */
3149 void netif_device_detach(struct net_device *dev)
3150 {
3151         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3152             netif_running(dev)) {
3153                 netif_tx_stop_all_queues(dev);
3154         }
3155 }
3156 EXPORT_SYMBOL(netif_device_detach);
3157
3158 /**
3159  * netif_device_attach - mark device as attached
3160  * @dev: network device
3161  *
3162  * Mark device as attached from system and restart if needed.
3163  */
3164 void netif_device_attach(struct net_device *dev)
3165 {
3166         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3167             netif_running(dev)) {
3168                 netif_tx_wake_all_queues(dev);
3169                 __netdev_watchdog_up(dev);
3170         }
3171 }
3172 EXPORT_SYMBOL(netif_device_attach);
3173
3174 /*
3175  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3176  * to be used as a distribution range.
3177  */
3178 static u16 skb_tx_hash(const struct net_device *dev,
3179                        const struct net_device *sb_dev,
3180                        struct sk_buff *skb)
3181 {
3182         u32 hash;
3183         u16 qoffset = 0;
3184         u16 qcount = dev->real_num_tx_queues;
3185
3186         if (dev->num_tc) {
3187                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3188
3189                 qoffset = sb_dev->tc_to_txq[tc].offset;
3190                 qcount = sb_dev->tc_to_txq[tc].count;
3191                 if (unlikely(!qcount)) {
3192                         net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3193                                              sb_dev->name, qoffset, tc);
3194                         qoffset = 0;
3195                         qcount = dev->real_num_tx_queues;
3196                 }
3197         }
3198
3199         if (skb_rx_queue_recorded(skb)) {
3200                 DEBUG_NET_WARN_ON_ONCE(qcount == 0);
3201                 hash = skb_get_rx_queue(skb);
3202                 if (hash >= qoffset)
3203                         hash -= qoffset;
3204                 while (unlikely(hash >= qcount))
3205                         hash -= qcount;
3206                 return hash + qoffset;
3207         }
3208
3209         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3210 }
3211
3212 static void skb_warn_bad_offload(const struct sk_buff *skb)
3213 {
3214         static const netdev_features_t null_features;
3215         struct net_device *dev = skb->dev;
3216         const char *name = "";
3217
3218         if (!net_ratelimit())
3219                 return;
3220
3221         if (dev) {
3222                 if (dev->dev.parent)
3223                         name = dev_driver_string(dev->dev.parent);
3224                 else
3225                         name = netdev_name(dev);
3226         }
3227         skb_dump(KERN_WARNING, skb, false);
3228         WARN(1, "%s: caps=(%pNF, %pNF)\n",
3229              name, dev ? &dev->features : &null_features,
3230              skb->sk ? &skb->sk->sk_route_caps : &null_features);
3231 }
3232
3233 /*
3234  * Invalidate hardware checksum when packet is to be mangled, and
3235  * complete checksum manually on outgoing path.
3236  */
3237 int skb_checksum_help(struct sk_buff *skb)
3238 {
3239         __wsum csum;
3240         int ret = 0, offset;
3241
3242         if (skb->ip_summed == CHECKSUM_COMPLETE)
3243                 goto out_set_summed;
3244
3245         if (unlikely(skb_is_gso(skb))) {
3246                 skb_warn_bad_offload(skb);
3247                 return -EINVAL;
3248         }
3249
3250         /* Before computing a checksum, we should make sure no frag could
3251          * be modified by an external entity : checksum could be wrong.
3252          */
3253         if (skb_has_shared_frag(skb)) {
3254                 ret = __skb_linearize(skb);
3255                 if (ret)
3256                         goto out;
3257         }
3258
3259         offset = skb_checksum_start_offset(skb);
3260         ret = -EINVAL;
3261         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3262                 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3263                 goto out;
3264         }
3265         csum = skb_checksum(skb, offset, skb->len - offset, 0);
3266
3267         offset += skb->csum_offset;
3268         if (WARN_ON_ONCE(offset + sizeof(__sum16) > skb_headlen(skb))) {
3269                 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3270                 goto out;
3271         }
3272         ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3273         if (ret)
3274                 goto out;
3275
3276         *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3277 out_set_summed:
3278         skb->ip_summed = CHECKSUM_NONE;
3279 out:
3280         return ret;
3281 }
3282 EXPORT_SYMBOL(skb_checksum_help);
3283
3284 int skb_crc32c_csum_help(struct sk_buff *skb)
3285 {
3286         __le32 crc32c_csum;
3287         int ret = 0, offset, start;
3288
3289         if (skb->ip_summed != CHECKSUM_PARTIAL)
3290                 goto out;
3291
3292         if (unlikely(skb_is_gso(skb)))
3293                 goto out;
3294
3295         /* Before computing a checksum, we should make sure no frag could
3296          * be modified by an external entity : checksum could be wrong.
3297          */
3298         if (unlikely(skb_has_shared_frag(skb))) {
3299                 ret = __skb_linearize(skb);
3300                 if (ret)
3301                         goto out;
3302         }
3303         start = skb_checksum_start_offset(skb);
3304         offset = start + offsetof(struct sctphdr, checksum);
3305         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3306                 ret = -EINVAL;
3307                 goto out;
3308         }
3309
3310         ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3311         if (ret)
3312                 goto out;
3313
3314         crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3315                                                   skb->len - start, ~(__u32)0,
3316                                                   crc32c_csum_stub));
3317         *(__le32 *)(skb->data + offset) = crc32c_csum;
3318         skb_reset_csum_not_inet(skb);
3319 out:
3320         return ret;
3321 }
3322
3323 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3324 {
3325         __be16 type = skb->protocol;
3326
3327         /* Tunnel gso handlers can set protocol to ethernet. */
3328         if (type == htons(ETH_P_TEB)) {
3329                 struct ethhdr *eth;
3330
3331                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3332                         return 0;
3333
3334                 eth = (struct ethhdr *)skb->data;
3335                 type = eth->h_proto;
3336         }
3337
3338         return vlan_get_protocol_and_depth(skb, type, depth);
3339 }
3340
3341 /* openvswitch calls this on rx path, so we need a different check.
3342  */
3343 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3344 {
3345         if (tx_path)
3346                 return skb->ip_summed != CHECKSUM_PARTIAL &&
3347                        skb->ip_summed != CHECKSUM_UNNECESSARY;
3348
3349         return skb->ip_summed == CHECKSUM_NONE;
3350 }
3351
3352 /**
3353  *      __skb_gso_segment - Perform segmentation on skb.
3354  *      @skb: buffer to segment
3355  *      @features: features for the output path (see dev->features)
3356  *      @tx_path: whether it is called in TX path
3357  *
3358  *      This function segments the given skb and returns a list of segments.
3359  *
3360  *      It may return NULL if the skb requires no segmentation.  This is
3361  *      only possible when GSO is used for verifying header integrity.
3362  *
3363  *      Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3364  */
3365 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3366                                   netdev_features_t features, bool tx_path)
3367 {
3368         struct sk_buff *segs;
3369
3370         if (unlikely(skb_needs_check(skb, tx_path))) {
3371                 int err;
3372
3373                 /* We're going to init ->check field in TCP or UDP header */
3374                 err = skb_cow_head(skb, 0);
3375                 if (err < 0)
3376                         return ERR_PTR(err);
3377         }
3378
3379         /* Only report GSO partial support if it will enable us to
3380          * support segmentation on this frame without needing additional
3381          * work.
3382          */
3383         if (features & NETIF_F_GSO_PARTIAL) {
3384                 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3385                 struct net_device *dev = skb->dev;
3386
3387                 partial_features |= dev->features & dev->gso_partial_features;
3388                 if (!skb_gso_ok(skb, features | partial_features))
3389                         features &= ~NETIF_F_GSO_PARTIAL;
3390         }
3391
3392         BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3393                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3394
3395         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3396         SKB_GSO_CB(skb)->encap_level = 0;
3397
3398         skb_reset_mac_header(skb);
3399         skb_reset_mac_len(skb);
3400
3401         segs = skb_mac_gso_segment(skb, features);
3402
3403         if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3404                 skb_warn_bad_offload(skb);
3405
3406         return segs;
3407 }
3408 EXPORT_SYMBOL(__skb_gso_segment);
3409
3410 /* Take action when hardware reception checksum errors are detected. */
3411 #ifdef CONFIG_BUG
3412 static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3413 {
3414         netdev_err(dev, "hw csum failure\n");
3415         skb_dump(KERN_ERR, skb, true);
3416         dump_stack();
3417 }
3418
3419 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3420 {
3421         DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3422 }
3423 EXPORT_SYMBOL(netdev_rx_csum_fault);
3424 #endif
3425
3426 /* XXX: check that highmem exists at all on the given machine. */
3427 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3428 {
3429 #ifdef CONFIG_HIGHMEM
3430         int i;
3431
3432         if (!(dev->features & NETIF_F_HIGHDMA)) {
3433                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3434                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3435
3436                         if (PageHighMem(skb_frag_page(frag)))
3437                                 return 1;
3438                 }
3439         }
3440 #endif
3441         return 0;
3442 }
3443
3444 /* If MPLS offload request, verify we are testing hardware MPLS features
3445  * instead of standard features for the netdev.
3446  */
3447 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3448 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3449                                            netdev_features_t features,
3450                                            __be16 type)
3451 {
3452         if (eth_p_mpls(type))
3453                 features &= skb->dev->mpls_features;
3454
3455         return features;
3456 }
3457 #else
3458 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3459                                            netdev_features_t features,
3460                                            __be16 type)
3461 {
3462         return features;
3463 }
3464 #endif
3465
3466 static netdev_features_t harmonize_features(struct sk_buff *skb,
3467         netdev_features_t features)
3468 {
3469         __be16 type;
3470
3471         type = skb_network_protocol(skb, NULL);
3472         features = net_mpls_features(skb, features, type);
3473
3474         if (skb->ip_summed != CHECKSUM_NONE &&
3475             !can_checksum_protocol(features, type)) {
3476                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3477         }
3478         if (illegal_highdma(skb->dev, skb))
3479                 features &= ~NETIF_F_SG;
3480
3481         return features;
3482 }
3483
3484 netdev_features_t passthru_features_check(struct sk_buff *skb,
3485                                           struct net_device *dev,
3486                                           netdev_features_t features)
3487 {
3488         return features;
3489 }
3490 EXPORT_SYMBOL(passthru_features_check);
3491
3492 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3493                                              struct net_device *dev,
3494                                              netdev_features_t features)
3495 {
3496         return vlan_features_check(skb, features);
3497 }
3498
3499 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3500                                             struct net_device *dev,
3501                                             netdev_features_t features)
3502 {
3503         u16 gso_segs = skb_shinfo(skb)->gso_segs;
3504
3505         if (gso_segs > READ_ONCE(dev->gso_max_segs))
3506                 return features & ~NETIF_F_GSO_MASK;
3507
3508         if (!skb_shinfo(skb)->gso_type) {
3509                 skb_warn_bad_offload(skb);
3510                 return features & ~NETIF_F_GSO_MASK;
3511         }
3512
3513         /* Support for GSO partial features requires software
3514          * intervention before we can actually process the packets
3515          * so we need to strip support for any partial features now
3516          * and we can pull them back in after we have partially
3517          * segmented the frame.
3518          */
3519         if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3520                 features &= ~dev->gso_partial_features;
3521
3522         /* Make sure to clear the IPv4 ID mangling feature if the
3523          * IPv4 header has the potential to be fragmented.
3524          */
3525         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3526                 struct iphdr *iph = skb->encapsulation ?
3527                                     inner_ip_hdr(skb) : ip_hdr(skb);
3528
3529                 if (!(iph->frag_off & htons(IP_DF)))
3530                         features &= ~NETIF_F_TSO_MANGLEID;
3531         }
3532
3533         return features;
3534 }
3535
3536 netdev_features_t netif_skb_features(struct sk_buff *skb)
3537 {
3538         struct net_device *dev = skb->dev;
3539         netdev_features_t features = dev->features;
3540
3541         if (skb_is_gso(skb))
3542                 features = gso_features_check(skb, dev, features);
3543
3544         /* If encapsulation offload request, verify we are testing
3545          * hardware encapsulation features instead of standard
3546          * features for the netdev
3547          */
3548         if (skb->encapsulation)
3549                 features &= dev->hw_enc_features;
3550
3551         if (skb_vlan_tagged(skb))
3552                 features = netdev_intersect_features(features,
3553                                                      dev->vlan_features |
3554                                                      NETIF_F_HW_VLAN_CTAG_TX |
3555                                                      NETIF_F_HW_VLAN_STAG_TX);
3556
3557         if (dev->netdev_ops->ndo_features_check)
3558                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3559                                                                 features);
3560         else
3561                 features &= dflt_features_check(skb, dev, features);
3562
3563         return harmonize_features(skb, features);
3564 }
3565 EXPORT_SYMBOL(netif_skb_features);
3566
3567 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3568                     struct netdev_queue *txq, bool more)
3569 {
3570         unsigned int len;
3571         int rc;
3572
3573         if (dev_nit_active(dev))
3574                 dev_queue_xmit_nit(skb, dev);
3575
3576         len = skb->len;
3577         trace_net_dev_start_xmit(skb, dev);
3578         rc = netdev_start_xmit(skb, dev, txq, more);
3579         trace_net_dev_xmit(skb, rc, dev, len);
3580
3581         return rc;
3582 }
3583
3584 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3585                                     struct netdev_queue *txq, int *ret)
3586 {
3587         struct sk_buff *skb = first;
3588         int rc = NETDEV_TX_OK;
3589
3590         while (skb) {
3591                 struct sk_buff *next = skb->next;
3592
3593                 skb_mark_not_on_list(skb);
3594                 rc = xmit_one(skb, dev, txq, next != NULL);
3595                 if (unlikely(!dev_xmit_complete(rc))) {
3596                         skb->next = next;
3597                         goto out;
3598                 }
3599
3600                 skb = next;
3601                 if (netif_tx_queue_stopped(txq) && skb) {
3602                         rc = NETDEV_TX_BUSY;
3603                         break;
3604                 }
3605         }
3606
3607 out:
3608         *ret = rc;
3609         return skb;
3610 }
3611
3612 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3613                                           netdev_features_t features)
3614 {
3615         if (skb_vlan_tag_present(skb) &&
3616             !vlan_hw_offload_capable(features, skb->vlan_proto))
3617                 skb = __vlan_hwaccel_push_inside(skb);
3618         return skb;
3619 }
3620
3621 int skb_csum_hwoffload_help(struct sk_buff *skb,
3622                             const netdev_features_t features)
3623 {
3624         if (unlikely(skb_csum_is_sctp(skb)))
3625                 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3626                         skb_crc32c_csum_help(skb);
3627
3628         if (features & NETIF_F_HW_CSUM)
3629                 return 0;
3630
3631         if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3632                 switch (skb->csum_offset) {
3633                 case offsetof(struct tcphdr, check):
3634                 case offsetof(struct udphdr, check):
3635                         return 0;
3636                 }
3637         }
3638
3639         return skb_checksum_help(skb);
3640 }
3641 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3642
3643 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3644 {
3645         netdev_features_t features;
3646
3647         features = netif_skb_features(skb);
3648         skb = validate_xmit_vlan(skb, features);
3649         if (unlikely(!skb))
3650                 goto out_null;
3651
3652         skb = sk_validate_xmit_skb(skb, dev);
3653         if (unlikely(!skb))
3654                 goto out_null;
3655
3656         if (netif_needs_gso(skb, features)) {
3657                 struct sk_buff *segs;
3658
3659                 segs = skb_gso_segment(skb, features);
3660                 if (IS_ERR(segs)) {
3661                         goto out_kfree_skb;
3662                 } else if (segs) {
3663                         consume_skb(skb);
3664                         skb = segs;
3665                 }
3666         } else {
3667                 if (skb_needs_linearize(skb, features) &&
3668                     __skb_linearize(skb))
3669                         goto out_kfree_skb;
3670
3671                 /* If packet is not checksummed and device does not
3672                  * support checksumming for this protocol, complete
3673                  * checksumming here.
3674                  */
3675                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3676                         if (skb->encapsulation)
3677                                 skb_set_inner_transport_header(skb,
3678                                                                skb_checksum_start_offset(skb));
3679                         else
3680                                 skb_set_transport_header(skb,
3681                                                          skb_checksum_start_offset(skb));
3682                         if (skb_csum_hwoffload_help(skb, features))
3683                                 goto out_kfree_skb;
3684                 }
3685         }
3686
3687         skb = validate_xmit_xfrm(skb, features, again);
3688
3689         return skb;
3690
3691 out_kfree_skb:
3692         kfree_skb(skb);
3693 out_null:
3694         dev_core_stats_tx_dropped_inc(dev);
3695         return NULL;
3696 }
3697
3698 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3699 {
3700         struct sk_buff *next, *head = NULL, *tail;
3701
3702         for (; skb != NULL; skb = next) {
3703                 next = skb->next;
3704                 skb_mark_not_on_list(skb);
3705
3706                 /* in case skb wont be segmented, point to itself */
3707                 skb->prev = skb;
3708
3709                 skb = validate_xmit_skb(skb, dev, again);
3710                 if (!skb)
3711                         continue;
3712
3713                 if (!head)
3714                         head = skb;
3715                 else
3716                         tail->next = skb;
3717                 /* If skb was segmented, skb->prev points to
3718                  * the last segment. If not, it still contains skb.
3719                  */
3720                 tail = skb->prev;
3721         }
3722         return head;
3723 }
3724 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3725
3726 static void qdisc_pkt_len_init(struct sk_buff *skb)
3727 {
3728         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3729
3730         qdisc_skb_cb(skb)->pkt_len = skb->len;
3731
3732         /* To get more precise estimation of bytes sent on wire,
3733          * we add to pkt_len the headers size of all segments
3734          */
3735         if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3736                 u16 gso_segs = shinfo->gso_segs;
3737                 unsigned int hdr_len;
3738
3739                 /* mac layer + network layer */
3740                 hdr_len = skb_transport_offset(skb);
3741
3742                 /* + transport layer */
3743                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3744                         const struct tcphdr *th;
3745                         struct tcphdr _tcphdr;
3746
3747                         th = skb_header_pointer(skb, hdr_len,
3748                                                 sizeof(_tcphdr), &_tcphdr);
3749                         if (likely(th))
3750                                 hdr_len += __tcp_hdrlen(th);
3751                 } else {
3752                         struct udphdr _udphdr;
3753
3754                         if (skb_header_pointer(skb, hdr_len,
3755                                                sizeof(_udphdr), &_udphdr))
3756                                 hdr_len += sizeof(struct udphdr);
3757                 }
3758
3759                 if (shinfo->gso_type & SKB_GSO_DODGY)
3760                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3761                                                 shinfo->gso_size);
3762
3763                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3764         }
3765 }
3766
3767 static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3768                              struct sk_buff **to_free,
3769                              struct netdev_queue *txq)
3770 {
3771         int rc;
3772
3773         rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3774         if (rc == NET_XMIT_SUCCESS)
3775                 trace_qdisc_enqueue(q, txq, skb);
3776         return rc;
3777 }
3778
3779 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3780                                  struct net_device *dev,
3781                                  struct netdev_queue *txq)
3782 {
3783         spinlock_t *root_lock = qdisc_lock(q);
3784         struct sk_buff *to_free = NULL;
3785         bool contended;
3786         int rc;
3787
3788         qdisc_calculate_pkt_len(skb, q);
3789
3790         if (q->flags & TCQ_F_NOLOCK) {
3791                 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3792                     qdisc_run_begin(q)) {
3793                         /* Retest nolock_qdisc_is_empty() within the protection
3794                          * of q->seqlock to protect from racing with requeuing.
3795                          */
3796                         if (unlikely(!nolock_qdisc_is_empty(q))) {
3797                                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3798                                 __qdisc_run(q);
3799                                 qdisc_run_end(q);
3800
3801                                 goto no_lock_out;
3802                         }
3803
3804                         qdisc_bstats_cpu_update(q, skb);
3805                         if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3806                             !nolock_qdisc_is_empty(q))
3807                                 __qdisc_run(q);
3808
3809                         qdisc_run_end(q);
3810                         return NET_XMIT_SUCCESS;
3811                 }
3812
3813                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3814                 qdisc_run(q);
3815
3816 no_lock_out:
3817                 if (unlikely(to_free))
3818                         kfree_skb_list_reason(to_free,
3819                                               SKB_DROP_REASON_QDISC_DROP);
3820                 return rc;
3821         }
3822
3823         /*
3824          * Heuristic to force contended enqueues to serialize on a
3825          * separate lock before trying to get qdisc main lock.
3826          * This permits qdisc->running owner to get the lock more
3827          * often and dequeue packets faster.
3828          * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3829          * and then other tasks will only enqueue packets. The packets will be
3830          * sent after the qdisc owner is scheduled again. To prevent this
3831          * scenario the task always serialize on the lock.
3832          */
3833         contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3834         if (unlikely(contended))
3835                 spin_lock(&q->busylock);
3836
3837         spin_lock(root_lock);
3838         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3839                 __qdisc_drop(skb, &to_free);
3840                 rc = NET_XMIT_DROP;
3841         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3842                    qdisc_run_begin(q)) {
3843                 /*
3844                  * This is a work-conserving queue; there are no old skbs
3845                  * waiting to be sent out; and the qdisc is not running -
3846                  * xmit the skb directly.
3847                  */
3848
3849                 qdisc_bstats_update(q, skb);
3850
3851                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3852                         if (unlikely(contended)) {
3853                                 spin_unlock(&q->busylock);
3854                                 contended = false;
3855                         }
3856                         __qdisc_run(q);
3857                 }
3858
3859                 qdisc_run_end(q);
3860                 rc = NET_XMIT_SUCCESS;
3861         } else {
3862                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3863                 if (qdisc_run_begin(q)) {
3864                         if (unlikely(contended)) {
3865                                 spin_unlock(&q->busylock);
3866                                 contended = false;
3867                         }
3868                         __qdisc_run(q);
3869                         qdisc_run_end(q);
3870                 }
3871         }
3872         spin_unlock(root_lock);
3873         if (unlikely(to_free))
3874                 kfree_skb_list_reason(to_free, SKB_DROP_REASON_QDISC_DROP);
3875         if (unlikely(contended))
3876                 spin_unlock(&q->busylock);
3877         return rc;
3878 }
3879
3880 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3881 static void skb_update_prio(struct sk_buff *skb)
3882 {
3883         const struct netprio_map *map;
3884         const struct sock *sk;
3885         unsigned int prioidx;
3886
3887         if (skb->priority)
3888                 return;
3889         map = rcu_dereference_bh(skb->dev->priomap);
3890         if (!map)
3891                 return;
3892         sk = skb_to_full_sk(skb);
3893         if (!sk)
3894                 return;
3895
3896         prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3897
3898         if (prioidx < map->priomap_len)
3899                 skb->priority = map->priomap[prioidx];
3900 }
3901 #else
3902 #define skb_update_prio(skb)
3903 #endif
3904
3905 /**
3906  *      dev_loopback_xmit - loop back @skb
3907  *      @net: network namespace this loopback is happening in
3908  *      @sk:  sk needed to be a netfilter okfn
3909  *      @skb: buffer to transmit
3910  */
3911 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3912 {
3913         skb_reset_mac_header(skb);
3914         __skb_pull(skb, skb_network_offset(skb));
3915         skb->pkt_type = PACKET_LOOPBACK;
3916         if (skb->ip_summed == CHECKSUM_NONE)
3917                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3918         DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
3919         skb_dst_force(skb);
3920         netif_rx(skb);
3921         return 0;
3922 }
3923 EXPORT_SYMBOL(dev_loopback_xmit);
3924
3925 #ifdef CONFIG_NET_EGRESS
3926 static struct sk_buff *
3927 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3928 {
3929 #ifdef CONFIG_NET_CLS_ACT
3930         struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3931         struct tcf_result cl_res;
3932
3933         if (!miniq)
3934                 return skb;
3935
3936         /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3937         tc_skb_cb(skb)->mru = 0;
3938         tc_skb_cb(skb)->post_ct = false;
3939         mini_qdisc_bstats_cpu_update(miniq, skb);
3940
3941         switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
3942         case TC_ACT_OK:
3943         case TC_ACT_RECLASSIFY:
3944                 skb->tc_index = TC_H_MIN(cl_res.classid);
3945                 break;
3946         case TC_ACT_SHOT:
3947                 mini_qdisc_qstats_cpu_drop(miniq);
3948                 *ret = NET_XMIT_DROP;
3949                 kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS);
3950                 return NULL;
3951         case TC_ACT_STOLEN:
3952         case TC_ACT_QUEUED:
3953         case TC_ACT_TRAP:
3954                 *ret = NET_XMIT_SUCCESS;
3955                 consume_skb(skb);
3956                 return NULL;
3957         case TC_ACT_REDIRECT:
3958                 /* No need to push/pop skb's mac_header here on egress! */
3959                 skb_do_redirect(skb);
3960                 *ret = NET_XMIT_SUCCESS;
3961                 return NULL;
3962         default:
3963                 break;
3964         }
3965 #endif /* CONFIG_NET_CLS_ACT */
3966
3967         return skb;
3968 }
3969
3970 static struct netdev_queue *
3971 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
3972 {
3973         int qm = skb_get_queue_mapping(skb);
3974
3975         return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
3976 }
3977
3978 static bool netdev_xmit_txqueue_skipped(void)
3979 {
3980         return __this_cpu_read(softnet_data.xmit.skip_txqueue);
3981 }
3982
3983 void netdev_xmit_skip_txqueue(bool skip)
3984 {
3985         __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
3986 }
3987 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
3988 #endif /* CONFIG_NET_EGRESS */
3989
3990 #ifdef CONFIG_XPS
3991 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3992                                struct xps_dev_maps *dev_maps, unsigned int tci)
3993 {
3994         int tc = netdev_get_prio_tc_map(dev, skb->priority);
3995         struct xps_map *map;
3996         int queue_index = -1;
3997
3998         if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
3999                 return queue_index;
4000
4001         tci *= dev_maps->num_tc;
4002         tci += tc;
4003
4004         map = rcu_dereference(dev_maps->attr_map[tci]);
4005         if (map) {
4006                 if (map->len == 1)
4007                         queue_index = map->queues[0];
4008                 else
4009                         queue_index = map->queues[reciprocal_scale(
4010                                                 skb_get_hash(skb), map->len)];
4011                 if (unlikely(queue_index >= dev->real_num_tx_queues))
4012                         queue_index = -1;
4013         }
4014         return queue_index;
4015 }
4016 #endif
4017
4018 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4019                          struct sk_buff *skb)
4020 {
4021 #ifdef CONFIG_XPS
4022         struct xps_dev_maps *dev_maps;
4023         struct sock *sk = skb->sk;
4024         int queue_index = -1;
4025
4026         if (!static_key_false(&xps_needed))
4027                 return -1;
4028
4029         rcu_read_lock();
4030         if (!static_key_false(&xps_rxqs_needed))
4031                 goto get_cpus_map;
4032
4033         dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4034         if (dev_maps) {
4035                 int tci = sk_rx_queue_get(sk);
4036
4037                 if (tci >= 0)
4038                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4039                                                           tci);
4040         }
4041
4042 get_cpus_map:
4043         if (queue_index < 0) {
4044                 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4045                 if (dev_maps) {
4046                         unsigned int tci = skb->sender_cpu - 1;
4047
4048                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4049                                                           tci);
4050                 }
4051         }
4052         rcu_read_unlock();
4053
4054         return queue_index;
4055 #else
4056         return -1;
4057 #endif
4058 }
4059
4060 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4061                      struct net_device *sb_dev)
4062 {
4063         return 0;
4064 }
4065 EXPORT_SYMBOL(dev_pick_tx_zero);
4066
4067 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4068                        struct net_device *sb_dev)
4069 {
4070         return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4071 }
4072 EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4073
4074 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4075                      struct net_device *sb_dev)
4076 {
4077         struct sock *sk = skb->sk;
4078         int queue_index = sk_tx_queue_get(sk);
4079
4080         sb_dev = sb_dev ? : dev;
4081
4082         if (queue_index < 0 || skb->ooo_okay ||
4083             queue_index >= dev->real_num_tx_queues) {
4084                 int new_index = get_xps_queue(dev, sb_dev, skb);
4085
4086                 if (new_index < 0)
4087                         new_index = skb_tx_hash(dev, sb_dev, skb);
4088
4089                 if (queue_index != new_index && sk &&
4090                     sk_fullsock(sk) &&
4091                     rcu_access_pointer(sk->sk_dst_cache))
4092                         sk_tx_queue_set(sk, new_index);
4093
4094                 queue_index = new_index;
4095         }
4096
4097         return queue_index;
4098 }
4099 EXPORT_SYMBOL(netdev_pick_tx);
4100
4101 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4102                                          struct sk_buff *skb,
4103                                          struct net_device *sb_dev)
4104 {
4105         int queue_index = 0;
4106
4107 #ifdef CONFIG_XPS
4108         u32 sender_cpu = skb->sender_cpu - 1;
4109
4110         if (sender_cpu >= (u32)NR_CPUS)
4111                 skb->sender_cpu = raw_smp_processor_id() + 1;
4112 #endif
4113
4114         if (dev->real_num_tx_queues != 1) {
4115                 const struct net_device_ops *ops = dev->netdev_ops;
4116
4117                 if (ops->ndo_select_queue)
4118                         queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4119                 else
4120                         queue_index = netdev_pick_tx(dev, skb, sb_dev);
4121
4122                 queue_index = netdev_cap_txqueue(dev, queue_index);
4123         }
4124
4125         skb_set_queue_mapping(skb, queue_index);
4126         return netdev_get_tx_queue(dev, queue_index);
4127 }
4128
4129 /**
4130  * __dev_queue_xmit() - transmit a buffer
4131  * @skb:        buffer to transmit
4132  * @sb_dev:     suboordinate device used for L2 forwarding offload
4133  *
4134  * Queue a buffer for transmission to a network device. The caller must
4135  * have set the device and priority and built the buffer before calling
4136  * this function. The function can be called from an interrupt.
4137  *
4138  * When calling this method, interrupts MUST be enabled. This is because
4139  * the BH enable code must have IRQs enabled so that it will not deadlock.
4140  *
4141  * Regardless of the return value, the skb is consumed, so it is currently
4142  * difficult to retry a send to this method. (You can bump the ref count
4143  * before sending to hold a reference for retry if you are careful.)
4144  *
4145  * Return:
4146  * * 0                          - buffer successfully transmitted
4147  * * positive qdisc return code - NET_XMIT_DROP etc.
4148  * * negative errno             - other errors
4149  */
4150 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4151 {
4152         struct net_device *dev = skb->dev;
4153         struct netdev_queue *txq = NULL;
4154         struct Qdisc *q;
4155         int rc = -ENOMEM;
4156         bool again = false;
4157
4158         skb_reset_mac_header(skb);
4159         skb_assert_len(skb);
4160
4161         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4162                 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4163
4164         /* Disable soft irqs for various locks below. Also
4165          * stops preemption for RCU.
4166          */
4167         rcu_read_lock_bh();
4168
4169         skb_update_prio(skb);
4170
4171         qdisc_pkt_len_init(skb);
4172 #ifdef CONFIG_NET_CLS_ACT
4173         skb->tc_at_ingress = 0;
4174 #endif
4175 #ifdef CONFIG_NET_EGRESS
4176         if (static_branch_unlikely(&egress_needed_key)) {
4177                 if (nf_hook_egress_active()) {
4178                         skb = nf_hook_egress(skb, &rc, dev);
4179                         if (!skb)
4180                                 goto out;
4181                 }
4182
4183                 netdev_xmit_skip_txqueue(false);
4184
4185                 nf_skip_egress(skb, true);
4186                 skb = sch_handle_egress(skb, &rc, dev);
4187                 if (!skb)
4188                         goto out;
4189                 nf_skip_egress(skb, false);
4190
4191                 if (netdev_xmit_txqueue_skipped())
4192                         txq = netdev_tx_queue_mapping(dev, skb);
4193         }
4194 #endif
4195         /* If device/qdisc don't need skb->dst, release it right now while
4196          * its hot in this cpu cache.
4197          */
4198         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4199                 skb_dst_drop(skb);
4200         else
4201                 skb_dst_force(skb);
4202
4203         if (!txq)
4204                 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4205
4206         q = rcu_dereference_bh(txq->qdisc);
4207
4208         trace_net_dev_queue(skb);
4209         if (q->enqueue) {
4210                 rc = __dev_xmit_skb(skb, q, dev, txq);
4211                 goto out;
4212         }
4213
4214         /* The device has no queue. Common case for software devices:
4215          * loopback, all the sorts of tunnels...
4216
4217          * Really, it is unlikely that netif_tx_lock protection is necessary
4218          * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
4219          * counters.)
4220          * However, it is possible, that they rely on protection
4221          * made by us here.
4222
4223          * Check this and shot the lock. It is not prone from deadlocks.
4224          *Either shot noqueue qdisc, it is even simpler 8)
4225          */
4226         if (dev->flags & IFF_UP) {
4227                 int cpu = smp_processor_id(); /* ok because BHs are off */
4228
4229                 /* Other cpus might concurrently change txq->xmit_lock_owner
4230                  * to -1 or to their cpu id, but not to our id.
4231                  */
4232                 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4233                         if (dev_xmit_recursion())
4234                                 goto recursion_alert;
4235
4236                         skb = validate_xmit_skb(skb, dev, &again);
4237                         if (!skb)
4238                                 goto out;
4239
4240                         HARD_TX_LOCK(dev, txq, cpu);
4241
4242                         if (!netif_xmit_stopped(txq)) {
4243                                 dev_xmit_recursion_inc();
4244                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4245                                 dev_xmit_recursion_dec();
4246                                 if (dev_xmit_complete(rc)) {
4247                                         HARD_TX_UNLOCK(dev, txq);
4248                                         goto out;
4249                                 }
4250                         }
4251                         HARD_TX_UNLOCK(dev, txq);
4252                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4253                                              dev->name);
4254                 } else {
4255                         /* Recursion is detected! It is possible,
4256                          * unfortunately
4257                          */
4258 recursion_alert:
4259                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4260                                              dev->name);
4261                 }
4262         }
4263
4264         rc = -ENETDOWN;
4265         rcu_read_unlock_bh();
4266
4267         dev_core_stats_tx_dropped_inc(dev);
4268         kfree_skb_list(skb);
4269         return rc;
4270 out:
4271         rcu_read_unlock_bh();
4272         return rc;
4273 }
4274 EXPORT_SYMBOL(__dev_queue_xmit);
4275
4276 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4277 {
4278         struct net_device *dev = skb->dev;
4279         struct sk_buff *orig_skb = skb;
4280         struct netdev_queue *txq;
4281         int ret = NETDEV_TX_BUSY;
4282         bool again = false;
4283
4284         if (unlikely(!netif_running(dev) ||
4285                      !netif_carrier_ok(dev)))
4286                 goto drop;
4287
4288         skb = validate_xmit_skb_list(skb, dev, &again);
4289         if (skb != orig_skb)
4290                 goto drop;
4291
4292         skb_set_queue_mapping(skb, queue_id);
4293         txq = skb_get_tx_queue(dev, skb);
4294
4295         local_bh_disable();
4296
4297         dev_xmit_recursion_inc();
4298         HARD_TX_LOCK(dev, txq, smp_processor_id());
4299         if (!netif_xmit_frozen_or_drv_stopped(txq))
4300                 ret = netdev_start_xmit(skb, dev, txq, false);
4301         HARD_TX_UNLOCK(dev, txq);
4302         dev_xmit_recursion_dec();
4303
4304         local_bh_enable();
4305         return ret;
4306 drop:
4307         dev_core_stats_tx_dropped_inc(dev);
4308         kfree_skb_list(skb);
4309         return NET_XMIT_DROP;
4310 }
4311 EXPORT_SYMBOL(__dev_direct_xmit);
4312
4313 /*************************************************************************
4314  *                      Receiver routines
4315  *************************************************************************/
4316
4317 int netdev_max_backlog __read_mostly = 1000;
4318 EXPORT_SYMBOL(netdev_max_backlog);
4319
4320 int netdev_tstamp_prequeue __read_mostly = 1;
4321 unsigned int sysctl_skb_defer_max __read_mostly = 64;
4322 int netdev_budget __read_mostly = 300;
4323 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4324 unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4325 int weight_p __read_mostly = 64;           /* old backlog weight */
4326 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
4327 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
4328 int dev_rx_weight __read_mostly = 64;
4329 int dev_tx_weight __read_mostly = 64;
4330
4331 /* Called with irq disabled */
4332 static inline void ____napi_schedule(struct softnet_data *sd,
4333                                      struct napi_struct *napi)
4334 {
4335         struct task_struct *thread;
4336
4337         lockdep_assert_irqs_disabled();
4338
4339         if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4340                 /* Paired with smp_mb__before_atomic() in
4341                  * napi_enable()/dev_set_threaded().
4342                  * Use READ_ONCE() to guarantee a complete
4343                  * read on napi->thread. Only call
4344                  * wake_up_process() when it's not NULL.
4345                  */
4346                 thread = READ_ONCE(napi->thread);
4347                 if (thread) {
4348                         /* Avoid doing set_bit() if the thread is in
4349                          * INTERRUPTIBLE state, cause napi_thread_wait()
4350                          * makes sure to proceed with napi polling
4351                          * if the thread is explicitly woken from here.
4352                          */
4353                         if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4354                                 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4355                         wake_up_process(thread);
4356                         return;
4357                 }
4358         }
4359
4360         list_add_tail(&napi->poll_list, &sd->poll_list);
4361         WRITE_ONCE(napi->list_owner, smp_processor_id());
4362         /* If not called from net_rx_action()
4363          * we have to raise NET_RX_SOFTIRQ.
4364          */
4365         if (!sd->in_net_rx_action)
4366                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4367 }
4368
4369 #ifdef CONFIG_RPS
4370
4371 /* One global table that all flow-based protocols share. */
4372 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4373 EXPORT_SYMBOL(rps_sock_flow_table);
4374 u32 rps_cpu_mask __read_mostly;
4375 EXPORT_SYMBOL(rps_cpu_mask);
4376
4377 struct static_key_false rps_needed __read_mostly;
4378 EXPORT_SYMBOL(rps_needed);
4379 struct static_key_false rfs_needed __read_mostly;
4380 EXPORT_SYMBOL(rfs_needed);
4381
4382 static struct rps_dev_flow *
4383 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4384             struct rps_dev_flow *rflow, u16 next_cpu)
4385 {
4386         if (next_cpu < nr_cpu_ids) {
4387 #ifdef CONFIG_RFS_ACCEL
4388                 struct netdev_rx_queue *rxqueue;
4389                 struct rps_dev_flow_table *flow_table;
4390                 struct rps_dev_flow *old_rflow;
4391                 u32 flow_id;
4392                 u16 rxq_index;
4393                 int rc;
4394
4395                 /* Should we steer this flow to a different hardware queue? */
4396                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4397                     !(dev->features & NETIF_F_NTUPLE))
4398                         goto out;
4399                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4400                 if (rxq_index == skb_get_rx_queue(skb))
4401                         goto out;
4402
4403                 rxqueue = dev->_rx + rxq_index;
4404                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4405                 if (!flow_table)
4406                         goto out;
4407                 flow_id = skb_get_hash(skb) & flow_table->mask;
4408                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4409                                                         rxq_index, flow_id);
4410                 if (rc < 0)
4411                         goto out;
4412                 old_rflow = rflow;
4413                 rflow = &flow_table->flows[flow_id];
4414                 rflow->filter = rc;
4415                 if (old_rflow->filter == rflow->filter)
4416                         old_rflow->filter = RPS_NO_FILTER;
4417         out:
4418 #endif
4419                 rflow->last_qtail =
4420                         per_cpu(softnet_data, next_cpu).input_queue_head;
4421         }
4422
4423         rflow->cpu = next_cpu;
4424         return rflow;
4425 }
4426
4427 /*
4428  * get_rps_cpu is called from netif_receive_skb and returns the target
4429  * CPU from the RPS map of the receiving queue for a given skb.
4430  * rcu_read_lock must be held on entry.
4431  */
4432 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4433                        struct rps_dev_flow **rflowp)
4434 {
4435         const struct rps_sock_flow_table *sock_flow_table;
4436         struct netdev_rx_queue *rxqueue = dev->_rx;
4437         struct rps_dev_flow_table *flow_table;
4438         struct rps_map *map;
4439         int cpu = -1;
4440         u32 tcpu;
4441         u32 hash;
4442
4443         if (skb_rx_queue_recorded(skb)) {
4444                 u16 index = skb_get_rx_queue(skb);
4445
4446                 if (unlikely(index >= dev->real_num_rx_queues)) {
4447                         WARN_ONCE(dev->real_num_rx_queues > 1,
4448                                   "%s received packet on queue %u, but number "
4449                                   "of RX queues is %u\n",
4450                                   dev->name, index, dev->real_num_rx_queues);
4451                         goto done;
4452                 }
4453                 rxqueue += index;
4454         }
4455
4456         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4457
4458         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4459         map = rcu_dereference(rxqueue->rps_map);
4460         if (!flow_table && !map)
4461                 goto done;
4462
4463         skb_reset_network_header(skb);
4464         hash = skb_get_hash(skb);
4465         if (!hash)
4466                 goto done;
4467
4468         sock_flow_table = rcu_dereference(rps_sock_flow_table);
4469         if (flow_table && sock_flow_table) {
4470                 struct rps_dev_flow *rflow;
4471                 u32 next_cpu;
4472                 u32 ident;
4473
4474                 /* First check into global flow table if there is a match.
4475                  * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
4476                  */
4477                 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
4478                 if ((ident ^ hash) & ~rps_cpu_mask)
4479                         goto try_rps;
4480
4481                 next_cpu = ident & rps_cpu_mask;
4482
4483                 /* OK, now we know there is a match,
4484                  * we can look at the local (per receive queue) flow table
4485                  */
4486                 rflow = &flow_table->flows[hash & flow_table->mask];
4487                 tcpu = rflow->cpu;
4488
4489                 /*
4490                  * If the desired CPU (where last recvmsg was done) is
4491                  * different from current CPU (one in the rx-queue flow
4492                  * table entry), switch if one of the following holds:
4493                  *   - Current CPU is unset (>= nr_cpu_ids).
4494                  *   - Current CPU is offline.
4495                  *   - The current CPU's queue tail has advanced beyond the
4496                  *     last packet that was enqueued using this table entry.
4497                  *     This guarantees that all previous packets for the flow
4498                  *     have been dequeued, thus preserving in order delivery.
4499                  */
4500                 if (unlikely(tcpu != next_cpu) &&
4501                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4502                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4503                       rflow->last_qtail)) >= 0)) {
4504                         tcpu = next_cpu;
4505                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4506                 }
4507
4508                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4509                         *rflowp = rflow;
4510                         cpu = tcpu;
4511                         goto done;
4512                 }
4513         }
4514
4515 try_rps:
4516
4517         if (map) {
4518                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4519                 if (cpu_online(tcpu)) {
4520                         cpu = tcpu;
4521                         goto done;
4522                 }
4523         }
4524
4525 done:
4526         return cpu;
4527 }
4528
4529 #ifdef CONFIG_RFS_ACCEL
4530
4531 /**
4532  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4533  * @dev: Device on which the filter was set
4534  * @rxq_index: RX queue index
4535  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4536  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4537  *
4538  * Drivers that implement ndo_rx_flow_steer() should periodically call
4539  * this function for each installed filter and remove the filters for
4540  * which it returns %true.
4541  */
4542 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4543                          u32 flow_id, u16 filter_id)
4544 {
4545         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4546         struct rps_dev_flow_table *flow_table;
4547         struct rps_dev_flow *rflow;
4548         bool expire = true;
4549         unsigned int cpu;
4550
4551         rcu_read_lock();
4552         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4553         if (flow_table && flow_id <= flow_table->mask) {
4554                 rflow = &flow_table->flows[flow_id];
4555                 cpu = READ_ONCE(rflow->cpu);
4556                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4557                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4558                            rflow->last_qtail) <
4559                      (int)(10 * flow_table->mask)))
4560                         expire = false;
4561         }
4562         rcu_read_unlock();
4563         return expire;
4564 }
4565 EXPORT_SYMBOL(rps_may_expire_flow);
4566
4567 #endif /* CONFIG_RFS_ACCEL */
4568
4569 /* Called from hardirq (IPI) context */
4570 static void rps_trigger_softirq(void *data)
4571 {
4572         struct softnet_data *sd = data;
4573
4574         ____napi_schedule(sd, &sd->backlog);
4575         sd->received_rps++;
4576 }
4577
4578 #endif /* CONFIG_RPS */
4579
4580 /* Called from hardirq (IPI) context */
4581 static void trigger_rx_softirq(void *data)
4582 {
4583         struct softnet_data *sd = data;
4584
4585         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4586         smp_store_release(&sd->defer_ipi_scheduled, 0);
4587 }
4588
4589 /*
4590  * After we queued a packet into sd->input_pkt_queue,
4591  * we need to make sure this queue is serviced soon.
4592  *
4593  * - If this is another cpu queue, link it to our rps_ipi_list,
4594  *   and make sure we will process rps_ipi_list from net_rx_action().
4595  *
4596  * - If this is our own queue, NAPI schedule our backlog.
4597  *   Note that this also raises NET_RX_SOFTIRQ.
4598  */
4599 static void napi_schedule_rps(struct softnet_data *sd)
4600 {
4601         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4602
4603 #ifdef CONFIG_RPS
4604         if (sd != mysd) {
4605                 sd->rps_ipi_next = mysd->rps_ipi_list;
4606                 mysd->rps_ipi_list = sd;
4607
4608                 /* If not called from net_rx_action() or napi_threaded_poll()
4609                  * we have to raise NET_RX_SOFTIRQ.
4610                  */
4611                 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll)
4612                         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4613                 return;
4614         }
4615 #endif /* CONFIG_RPS */
4616         __napi_schedule_irqoff(&mysd->backlog);
4617 }
4618
4619 #ifdef CONFIG_NET_FLOW_LIMIT
4620 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4621 #endif
4622
4623 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4624 {
4625 #ifdef CONFIG_NET_FLOW_LIMIT
4626         struct sd_flow_limit *fl;
4627         struct softnet_data *sd;
4628         unsigned int old_flow, new_flow;
4629
4630         if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
4631                 return false;
4632
4633         sd = this_cpu_ptr(&softnet_data);
4634
4635         rcu_read_lock();
4636         fl = rcu_dereference(sd->flow_limit);
4637         if (fl) {
4638                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4639                 old_flow = fl->history[fl->history_head];
4640                 fl->history[fl->history_head] = new_flow;
4641
4642                 fl->history_head++;
4643                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4644
4645                 if (likely(fl->buckets[old_flow]))
4646                         fl->buckets[old_flow]--;
4647
4648                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4649                         fl->count++;
4650                         rcu_read_unlock();
4651                         return true;
4652                 }
4653         }
4654         rcu_read_unlock();
4655 #endif
4656         return false;
4657 }
4658
4659 /*
4660  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4661  * queue (may be a remote CPU queue).
4662  */
4663 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4664                               unsigned int *qtail)
4665 {
4666         enum skb_drop_reason reason;
4667         struct softnet_data *sd;
4668         unsigned long flags;
4669         unsigned int qlen;
4670
4671         reason = SKB_DROP_REASON_NOT_SPECIFIED;
4672         sd = &per_cpu(softnet_data, cpu);
4673
4674         rps_lock_irqsave(sd, &flags);
4675         if (!netif_running(skb->dev))
4676                 goto drop;
4677         qlen = skb_queue_len(&sd->input_pkt_queue);
4678         if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
4679                 if (qlen) {
4680 enqueue:
4681                         __skb_queue_tail(&sd->input_pkt_queue, skb);
4682                         input_queue_tail_incr_save(sd, qtail);
4683                         rps_unlock_irq_restore(sd, &flags);
4684                         return NET_RX_SUCCESS;
4685                 }
4686
4687                 /* Schedule NAPI for backlog device
4688                  * We can use non atomic operation since we own the queue lock
4689                  */
4690                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4691                         napi_schedule_rps(sd);
4692                 goto enqueue;
4693         }
4694         reason = SKB_DROP_REASON_CPU_BACKLOG;
4695
4696 drop:
4697         sd->dropped++;
4698         rps_unlock_irq_restore(sd, &flags);
4699
4700         dev_core_stats_rx_dropped_inc(skb->dev);
4701         kfree_skb_reason(skb, reason);
4702         return NET_RX_DROP;
4703 }
4704
4705 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4706 {
4707         struct net_device *dev = skb->dev;
4708         struct netdev_rx_queue *rxqueue;
4709
4710         rxqueue = dev->_rx;
4711
4712         if (skb_rx_queue_recorded(skb)) {
4713                 u16 index = skb_get_rx_queue(skb);
4714
4715                 if (unlikely(index >= dev->real_num_rx_queues)) {
4716                         WARN_ONCE(dev->real_num_rx_queues > 1,
4717                                   "%s received packet on queue %u, but number "
4718                                   "of RX queues is %u\n",
4719                                   dev->name, index, dev->real_num_rx_queues);
4720
4721                         return rxqueue; /* Return first rxqueue */
4722                 }
4723                 rxqueue += index;
4724         }
4725         return rxqueue;
4726 }
4727
4728 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4729                              struct bpf_prog *xdp_prog)
4730 {
4731         void *orig_data, *orig_data_end, *hard_start;
4732         struct netdev_rx_queue *rxqueue;
4733         bool orig_bcast, orig_host;
4734         u32 mac_len, frame_sz;
4735         __be16 orig_eth_type;
4736         struct ethhdr *eth;
4737         u32 metalen, act;
4738         int off;
4739
4740         /* The XDP program wants to see the packet starting at the MAC
4741          * header.
4742          */
4743         mac_len = skb->data - skb_mac_header(skb);
4744         hard_start = skb->data - skb_headroom(skb);
4745
4746         /* SKB "head" area always have tailroom for skb_shared_info */
4747         frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4748         frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4749
4750         rxqueue = netif_get_rxqueue(skb);
4751         xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4752         xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4753                          skb_headlen(skb) + mac_len, true);
4754
4755         orig_data_end = xdp->data_end;
4756         orig_data = xdp->data;
4757         eth = (struct ethhdr *)xdp->data;
4758         orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4759         orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4760         orig_eth_type = eth->h_proto;
4761
4762         act = bpf_prog_run_xdp(xdp_prog, xdp);
4763
4764         /* check if bpf_xdp_adjust_head was used */
4765         off = xdp->data - orig_data;
4766         if (off) {
4767                 if (off > 0)
4768                         __skb_pull(skb, off);
4769                 else if (off < 0)
4770                         __skb_push(skb, -off);
4771
4772                 skb->mac_header += off;
4773                 skb_reset_network_header(skb);
4774         }
4775
4776         /* check if bpf_xdp_adjust_tail was used */
4777         off = xdp->data_end - orig_data_end;
4778         if (off != 0) {
4779                 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4780                 skb->len += off; /* positive on grow, negative on shrink */
4781         }
4782
4783         /* check if XDP changed eth hdr such SKB needs update */
4784         eth = (struct ethhdr *)xdp->data;
4785         if ((orig_eth_type != eth->h_proto) ||
4786             (orig_host != ether_addr_equal_64bits(eth->h_dest,
4787                                                   skb->dev->dev_addr)) ||
4788             (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4789                 __skb_push(skb, ETH_HLEN);
4790                 skb->pkt_type = PACKET_HOST;
4791                 skb->protocol = eth_type_trans(skb, skb->dev);
4792         }
4793
4794         /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4795          * before calling us again on redirect path. We do not call do_redirect
4796          * as we leave that up to the caller.
4797          *
4798          * Caller is responsible for managing lifetime of skb (i.e. calling
4799          * kfree_skb in response to actions it cannot handle/XDP_DROP).
4800          */
4801         switch (act) {
4802         case XDP_REDIRECT:
4803         case XDP_TX:
4804                 __skb_push(skb, mac_len);
4805                 break;
4806         case XDP_PASS:
4807                 metalen = xdp->data - xdp->data_meta;
4808                 if (metalen)
4809                         skb_metadata_set(skb, metalen);
4810                 break;
4811         }
4812
4813         return act;
4814 }
4815
4816 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4817                                      struct xdp_buff *xdp,
4818                                      struct bpf_prog *xdp_prog)
4819 {
4820         u32 act = XDP_DROP;
4821
4822         /* Reinjected packets coming from act_mirred or similar should
4823          * not get XDP generic processing.
4824          */
4825         if (skb_is_redirected(skb))
4826                 return XDP_PASS;
4827
4828         /* XDP packets must be linear and must have sufficient headroom
4829          * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4830          * native XDP provides, thus we need to do it here as well.
4831          */
4832         if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4833             skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4834                 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4835                 int troom = skb->tail + skb->data_len - skb->end;
4836
4837                 /* In case we have to go down the path and also linearize,
4838                  * then lets do the pskb_expand_head() work just once here.
4839                  */
4840                 if (pskb_expand_head(skb,
4841                                      hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4842                                      troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4843                         goto do_drop;
4844                 if (skb_linearize(skb))
4845                         goto do_drop;
4846         }
4847
4848         act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4849         switch (act) {
4850         case XDP_REDIRECT:
4851         case XDP_TX:
4852         case XDP_PASS:
4853                 break;
4854         default:
4855                 bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4856                 fallthrough;
4857         case XDP_ABORTED:
4858                 trace_xdp_exception(skb->dev, xdp_prog, act);
4859                 fallthrough;
4860         case XDP_DROP:
4861         do_drop:
4862                 kfree_skb(skb);
4863                 break;
4864         }
4865
4866         return act;
4867 }
4868
4869 /* When doing generic XDP we have to bypass the qdisc layer and the
4870  * network taps in order to match in-driver-XDP behavior. This also means
4871  * that XDP packets are able to starve other packets going through a qdisc,
4872  * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
4873  * queues, so they do not have this starvation issue.
4874  */
4875 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4876 {
4877         struct net_device *dev = skb->dev;
4878         struct netdev_queue *txq;
4879         bool free_skb = true;
4880         int cpu, rc;
4881
4882         txq = netdev_core_pick_tx(dev, skb, NULL);
4883         cpu = smp_processor_id();
4884         HARD_TX_LOCK(dev, txq, cpu);
4885         if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4886                 rc = netdev_start_xmit(skb, dev, txq, 0);
4887                 if (dev_xmit_complete(rc))
4888                         free_skb = false;
4889         }
4890         HARD_TX_UNLOCK(dev, txq);
4891         if (free_skb) {
4892                 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4893                 dev_core_stats_tx_dropped_inc(dev);
4894                 kfree_skb(skb);
4895         }
4896 }
4897
4898 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4899
4900 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4901 {
4902         if (xdp_prog) {
4903                 struct xdp_buff xdp;
4904                 u32 act;
4905                 int err;
4906
4907                 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4908                 if (act != XDP_PASS) {
4909                         switch (act) {
4910                         case XDP_REDIRECT:
4911                                 err = xdp_do_generic_redirect(skb->dev, skb,
4912                                                               &xdp, xdp_prog);
4913                                 if (err)
4914                                         goto out_redir;
4915                                 break;
4916                         case XDP_TX:
4917                                 generic_xdp_tx(skb, xdp_prog);
4918                                 break;
4919                         }
4920                         return XDP_DROP;
4921                 }
4922         }
4923         return XDP_PASS;
4924 out_redir:
4925         kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
4926         return XDP_DROP;
4927 }
4928 EXPORT_SYMBOL_GPL(do_xdp_generic);
4929
4930 static int netif_rx_internal(struct sk_buff *skb)
4931 {
4932         int ret;
4933
4934         net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
4935
4936         trace_netif_rx(skb);
4937
4938 #ifdef CONFIG_RPS
4939         if (static_branch_unlikely(&rps_needed)) {
4940                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4941                 int cpu;
4942
4943                 rcu_read_lock();
4944
4945                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4946                 if (cpu < 0)
4947                         cpu = smp_processor_id();
4948
4949                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4950
4951                 rcu_read_unlock();
4952         } else
4953 #endif
4954         {
4955                 unsigned int qtail;
4956
4957                 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
4958         }
4959         return ret;
4960 }
4961
4962 /**
4963  *      __netif_rx      -       Slightly optimized version of netif_rx
4964  *      @skb: buffer to post
4965  *
4966  *      This behaves as netif_rx except that it does not disable bottom halves.
4967  *      As a result this function may only be invoked from the interrupt context
4968  *      (either hard or soft interrupt).
4969  */
4970 int __netif_rx(struct sk_buff *skb)
4971 {
4972         int ret;
4973
4974         lockdep_assert_once(hardirq_count() | softirq_count());
4975
4976         trace_netif_rx_entry(skb);
4977         ret = netif_rx_internal(skb);
4978         trace_netif_rx_exit(ret);
4979         return ret;
4980 }
4981 EXPORT_SYMBOL(__netif_rx);
4982
4983 /**
4984  *      netif_rx        -       post buffer to the network code
4985  *      @skb: buffer to post
4986  *
4987  *      This function receives a packet from a device driver and queues it for
4988  *      the upper (protocol) levels to process via the backlog NAPI device. It
4989  *      always succeeds. The buffer may be dropped during processing for
4990  *      congestion control or by the protocol layers.
4991  *      The network buffer is passed via the backlog NAPI device. Modern NIC
4992  *      driver should use NAPI and GRO.
4993  *      This function can used from interrupt and from process context. The
4994  *      caller from process context must not disable interrupts before invoking
4995  *      this function.
4996  *
4997  *      return values:
4998  *      NET_RX_SUCCESS  (no congestion)
4999  *      NET_RX_DROP     (packet was dropped)
5000  *
5001  */
5002 int netif_rx(struct sk_buff *skb)
5003 {
5004         bool need_bh_off = !(hardirq_count() | softirq_count());
5005         int ret;
5006
5007         if (need_bh_off)
5008                 local_bh_disable();
5009         trace_netif_rx_entry(skb);
5010         ret = netif_rx_internal(skb);
5011         trace_netif_rx_exit(ret);
5012         if (need_bh_off)
5013                 local_bh_enable();
5014         return ret;
5015 }
5016 EXPORT_SYMBOL(netif_rx);
5017
5018 static __latent_entropy void net_tx_action(struct softirq_action *h)
5019 {
5020         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5021
5022         if (sd->completion_queue) {
5023                 struct sk_buff *clist;
5024
5025                 local_irq_disable();
5026                 clist = sd->completion_queue;
5027                 sd->completion_queue = NULL;
5028                 local_irq_enable();
5029
5030                 while (clist) {
5031                         struct sk_buff *skb = clist;
5032
5033                         clist = clist->next;
5034
5035                         WARN_ON(refcount_read(&skb->users));
5036                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
5037                                 trace_consume_skb(skb, net_tx_action);
5038                         else
5039                                 trace_kfree_skb(skb, net_tx_action,
5040                                                 get_kfree_skb_cb(skb)->reason);
5041
5042                         if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5043                                 __kfree_skb(skb);
5044                         else
5045                                 __napi_kfree_skb(skb,
5046                                                  get_kfree_skb_cb(skb)->reason);
5047                 }
5048         }
5049
5050         if (sd->output_queue) {
5051                 struct Qdisc *head;
5052
5053                 local_irq_disable();
5054                 head = sd->output_queue;
5055                 sd->output_queue = NULL;
5056                 sd->output_queue_tailp = &sd->output_queue;
5057                 local_irq_enable();
5058
5059                 rcu_read_lock();
5060
5061                 while (head) {
5062                         struct Qdisc *q = head;
5063                         spinlock_t *root_lock = NULL;
5064
5065                         head = head->next_sched;
5066
5067                         /* We need to make sure head->next_sched is read
5068                          * before clearing __QDISC_STATE_SCHED
5069                          */
5070                         smp_mb__before_atomic();
5071
5072                         if (!(q->flags & TCQ_F_NOLOCK)) {
5073                                 root_lock = qdisc_lock(q);
5074                                 spin_lock(root_lock);
5075                         } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5076                                                      &q->state))) {
5077                                 /* There is a synchronize_net() between
5078                                  * STATE_DEACTIVATED flag being set and
5079                                  * qdisc_reset()/some_qdisc_is_busy() in
5080                                  * dev_deactivate(), so we can safely bail out
5081                                  * early here to avoid data race between
5082                                  * qdisc_deactivate() and some_qdisc_is_busy()
5083                                  * for lockless qdisc.
5084                                  */
5085                                 clear_bit(__QDISC_STATE_SCHED, &q->state);
5086                                 continue;
5087                         }
5088
5089                         clear_bit(__QDISC_STATE_SCHED, &q->state);
5090                         qdisc_run(q);
5091                         if (root_lock)
5092                                 spin_unlock(root_lock);
5093                 }
5094
5095                 rcu_read_unlock();
5096         }
5097
5098         xfrm_dev_backlog(sd);
5099 }
5100
5101 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5102 /* This hook is defined here for ATM LANE */
5103 int (*br_fdb_test_addr_hook)(struct net_device *dev,
5104                              unsigned char *addr) __read_mostly;
5105 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5106 #endif
5107
5108 static inline struct sk_buff *
5109 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
5110                    struct net_device *orig_dev, bool *another)
5111 {
5112 #ifdef CONFIG_NET_CLS_ACT
5113         struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
5114         struct tcf_result cl_res;
5115
5116         /* If there's at least one ingress present somewhere (so
5117          * we get here via enabled static key), remaining devices
5118          * that are not configured with an ingress qdisc will bail
5119          * out here.
5120          */
5121         if (!miniq)
5122                 return skb;
5123
5124         if (*pt_prev) {
5125                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5126                 *pt_prev = NULL;
5127         }
5128
5129         qdisc_skb_cb(skb)->pkt_len = skb->len;
5130         tc_skb_cb(skb)->mru = 0;
5131         tc_skb_cb(skb)->post_ct = false;
5132         skb->tc_at_ingress = 1;
5133         mini_qdisc_bstats_cpu_update(miniq, skb);
5134
5135         switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
5136         case TC_ACT_OK:
5137         case TC_ACT_RECLASSIFY:
5138                 skb->tc_index = TC_H_MIN(cl_res.classid);
5139                 break;
5140         case TC_ACT_SHOT:
5141                 mini_qdisc_qstats_cpu_drop(miniq);
5142                 kfree_skb_reason(skb, SKB_DROP_REASON_TC_INGRESS);
5143                 *ret = NET_RX_DROP;
5144                 return NULL;
5145         case TC_ACT_STOLEN:
5146         case TC_ACT_QUEUED:
5147         case TC_ACT_TRAP:
5148                 consume_skb(skb);
5149                 *ret = NET_RX_SUCCESS;
5150                 return NULL;
5151         case TC_ACT_REDIRECT:
5152                 /* skb_mac_header check was done by cls/act_bpf, so
5153                  * we can safely push the L2 header back before
5154                  * redirecting to another netdev
5155                  */
5156                 __skb_push(skb, skb->mac_len);
5157                 if (skb_do_redirect(skb) == -EAGAIN) {
5158                         __skb_pull(skb, skb->mac_len);
5159                         *another = true;
5160                         break;
5161                 }
5162                 *ret = NET_RX_SUCCESS;
5163                 return NULL;
5164         case TC_ACT_CONSUMED:
5165                 *ret = NET_RX_SUCCESS;
5166                 return NULL;
5167         default:
5168                 break;
5169         }
5170 #endif /* CONFIG_NET_CLS_ACT */
5171         return skb;
5172 }
5173
5174 /**
5175  *      netdev_is_rx_handler_busy - check if receive handler is registered
5176  *      @dev: device to check
5177  *
5178  *      Check if a receive handler is already registered for a given device.
5179  *      Return true if there one.
5180  *
5181  *      The caller must hold the rtnl_mutex.
5182  */
5183 bool netdev_is_rx_handler_busy(struct net_device *dev)
5184 {
5185         ASSERT_RTNL();
5186         return dev && rtnl_dereference(dev->rx_handler);
5187 }
5188 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5189
5190 /**
5191  *      netdev_rx_handler_register - register receive handler
5192  *      @dev: device to register a handler for
5193  *      @rx_handler: receive handler to register
5194  *      @rx_handler_data: data pointer that is used by rx handler
5195  *
5196  *      Register a receive handler for a device. This handler will then be
5197  *      called from __netif_receive_skb. A negative errno code is returned
5198  *      on a failure.
5199  *
5200  *      The caller must hold the rtnl_mutex.
5201  *
5202  *      For a general description of rx_handler, see enum rx_handler_result.
5203  */
5204 int netdev_rx_handler_register(struct net_device *dev,
5205                                rx_handler_func_t *rx_handler,
5206                                void *rx_handler_data)
5207 {
5208         if (netdev_is_rx_handler_busy(dev))
5209                 return -EBUSY;
5210
5211         if (dev->priv_flags & IFF_NO_RX_HANDLER)
5212                 return -EINVAL;
5213
5214         /* Note: rx_handler_data must be set before rx_handler */
5215         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5216         rcu_assign_pointer(dev->rx_handler, rx_handler);
5217
5218         return 0;
5219 }
5220 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5221
5222 /**
5223  *      netdev_rx_handler_unregister - unregister receive handler
5224  *      @dev: device to unregister a handler from
5225  *
5226  *      Unregister a receive handler from a device.
5227  *
5228  *      The caller must hold the rtnl_mutex.
5229  */
5230 void netdev_rx_handler_unregister(struct net_device *dev)
5231 {
5232
5233         ASSERT_RTNL();
5234         RCU_INIT_POINTER(dev->rx_handler, NULL);
5235         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5236          * section has a guarantee to see a non NULL rx_handler_data
5237          * as well.
5238          */
5239         synchronize_net();
5240         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5241 }
5242 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5243
5244 /*
5245  * Limit the use of PFMEMALLOC reserves to those protocols that implement
5246  * the special handling of PFMEMALLOC skbs.
5247  */
5248 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5249 {
5250         switch (skb->protocol) {
5251         case htons(ETH_P_ARP):
5252         case htons(ETH_P_IP):
5253         case htons(ETH_P_IPV6):
5254         case htons(ETH_P_8021Q):
5255         case htons(ETH_P_8021AD):
5256                 return true;
5257         default:
5258                 return false;
5259         }
5260 }
5261
5262 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5263                              int *ret, struct net_device *orig_dev)
5264 {
5265         if (nf_hook_ingress_active(skb)) {
5266                 int ingress_retval;
5267
5268                 if (*pt_prev) {
5269                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
5270                         *pt_prev = NULL;
5271                 }
5272
5273                 rcu_read_lock();
5274                 ingress_retval = nf_hook_ingress(skb);
5275                 rcu_read_unlock();
5276                 return ingress_retval;
5277         }
5278         return 0;
5279 }
5280
5281 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5282                                     struct packet_type **ppt_prev)
5283 {
5284         struct packet_type *ptype, *pt_prev;
5285         rx_handler_func_t *rx_handler;
5286         struct sk_buff *skb = *pskb;
5287         struct net_device *orig_dev;
5288         bool deliver_exact = false;
5289         int ret = NET_RX_DROP;
5290         __be16 type;
5291
5292         net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);
5293
5294         trace_netif_receive_skb(skb);
5295
5296         orig_dev = skb->dev;
5297
5298         skb_reset_network_header(skb);
5299         if (!skb_transport_header_was_set(skb))
5300                 skb_reset_transport_header(skb);
5301         skb_reset_mac_len(skb);
5302
5303         pt_prev = NULL;
5304
5305 another_round:
5306         skb->skb_iif = skb->dev->ifindex;
5307
5308         __this_cpu_inc(softnet_data.processed);
5309
5310         if (static_branch_unlikely(&generic_xdp_needed_key)) {
5311                 int ret2;
5312
5313                 migrate_disable();
5314                 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5315                 migrate_enable();
5316
5317                 if (ret2 != XDP_PASS) {
5318                         ret = NET_RX_DROP;
5319                         goto out;
5320                 }
5321         }
5322
5323         if (eth_type_vlan(skb->protocol)) {
5324                 skb = skb_vlan_untag(skb);
5325                 if (unlikely(!skb))
5326                         goto out;
5327         }
5328
5329         if (skb_skip_tc_classify(skb))
5330                 goto skip_classify;
5331
5332         if (pfmemalloc)
5333                 goto skip_taps;
5334
5335         list_for_each_entry_rcu(ptype, &ptype_all, list) {
5336                 if (pt_prev)
5337                         ret = deliver_skb(skb, pt_prev, orig_dev);
5338                 pt_prev = ptype;
5339         }
5340
5341         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5342                 if (pt_prev)
5343                         ret = deliver_skb(skb, pt_prev, orig_dev);
5344                 pt_prev = ptype;
5345         }
5346
5347 skip_taps:
5348 #ifdef CONFIG_NET_INGRESS
5349         if (static_branch_unlikely(&ingress_needed_key)) {
5350                 bool another = false;
5351
5352                 nf_skip_egress(skb, true);
5353                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5354                                          &another);
5355                 if (another)
5356                         goto another_round;
5357                 if (!skb)
5358                         goto out;
5359
5360                 nf_skip_egress(skb, false);
5361                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5362                         goto out;
5363         }
5364 #endif
5365         skb_reset_redirect(skb);
5366 skip_classify:
5367         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5368                 goto drop;
5369
5370         if (skb_vlan_tag_present(skb)) {
5371                 if (pt_prev) {
5372                         ret = deliver_skb(skb, pt_prev, orig_dev);
5373                         pt_prev = NULL;
5374                 }
5375                 if (vlan_do_receive(&skb))
5376                         goto another_round;
5377                 else if (unlikely(!skb))
5378                         goto out;
5379         }
5380
5381         rx_handler = rcu_dereference(skb->dev->rx_handler);
5382         if (rx_handler) {
5383                 if (pt_prev) {
5384                         ret = deliver_skb(skb, pt_prev, orig_dev);
5385                         pt_prev = NULL;
5386                 }
5387                 switch (rx_handler(&skb)) {
5388                 case RX_HANDLER_CONSUMED:
5389                         ret = NET_RX_SUCCESS;
5390                         goto out;
5391                 case RX_HANDLER_ANOTHER:
5392                         goto another_round;
5393                 case RX_HANDLER_EXACT:
5394                         deliver_exact = true;
5395                         break;
5396                 case RX_HANDLER_PASS:
5397                         break;
5398                 default:
5399                         BUG();
5400                 }
5401         }
5402
5403         if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5404 check_vlan_id:
5405                 if (skb_vlan_tag_get_id(skb)) {
5406                         /* Vlan id is non 0 and vlan_do_receive() above couldn't
5407                          * find vlan device.
5408                          */
5409                         skb->pkt_type = PACKET_OTHERHOST;
5410                 } else if (eth_type_vlan(skb->protocol)) {
5411                         /* Outer header is 802.1P with vlan 0, inner header is
5412                          * 802.1Q or 802.1AD and vlan_do_receive() above could
5413                          * not find vlan dev for vlan id 0.
5414                          */
5415                         __vlan_hwaccel_clear_tag(skb);
5416                         skb = skb_vlan_untag(skb);
5417                         if (unlikely(!skb))
5418                                 goto out;
5419                         if (vlan_do_receive(&skb))
5420                                 /* After stripping off 802.1P header with vlan 0
5421                                  * vlan dev is found for inner header.
5422                                  */
5423                                 goto another_round;
5424                         else if (unlikely(!skb))
5425                                 goto out;
5426                         else
5427                                 /* We have stripped outer 802.1P vlan 0 header.
5428                                  * But could not find vlan dev.
5429                                  * check again for vlan id to set OTHERHOST.
5430                                  */
5431                                 goto check_vlan_id;
5432                 }
5433                 /* Note: we might in the future use prio bits
5434                  * and set skb->priority like in vlan_do_receive()
5435                  * For the time being, just ignore Priority Code Point
5436                  */
5437                 __vlan_hwaccel_clear_tag(skb);
5438         }
5439
5440         type = skb->protocol;
5441
5442         /* deliver only exact match when indicated */
5443         if (likely(!deliver_exact)) {
5444                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5445                                        &ptype_base[ntohs(type) &
5446                                                    PTYPE_HASH_MASK]);
5447         }
5448
5449         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5450                                &orig_dev->ptype_specific);
5451
5452         if (unlikely(skb->dev != orig_dev)) {
5453                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5454                                        &skb->dev->ptype_specific);
5455         }
5456
5457         if (pt_prev) {
5458                 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5459                         goto drop;
5460                 *ppt_prev = pt_prev;
5461         } else {
5462 drop:
5463                 if (!deliver_exact)
5464                         dev_core_stats_rx_dropped_inc(skb->dev);
5465                 else
5466                         dev_core_stats_rx_nohandler_inc(skb->dev);
5467                 kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
5468                 /* Jamal, now you will not able to escape explaining
5469                  * me how you were going to use this. :-)
5470                  */
5471                 ret = NET_RX_DROP;
5472         }
5473
5474 out:
5475         /* The invariant here is that if *ppt_prev is not NULL
5476          * then skb should also be non-NULL.
5477          *
5478          * Apparently *ppt_prev assignment above holds this invariant due to
5479          * skb dereferencing near it.
5480          */
5481         *pskb = skb;
5482         return ret;
5483 }
5484
5485 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5486 {
5487         struct net_device *orig_dev = skb->dev;
5488         struct packet_type *pt_prev = NULL;
5489         int ret;
5490
5491         ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5492         if (pt_prev)
5493                 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5494                                          skb->dev, pt_prev, orig_dev);
5495         return ret;
5496 }
5497
5498 /**
5499  *      netif_receive_skb_core - special purpose version of netif_receive_skb
5500  *      @skb: buffer to process
5501  *
5502  *      More direct receive version of netif_receive_skb().  It should
5503  *      only be used by callers that have a need to skip RPS and Generic XDP.
5504  *      Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5505  *
5506  *      This function may only be called from softirq context and interrupts
5507  *      should be enabled.
5508  *
5509  *      Return values (usually ignored):
5510  *      NET_RX_SUCCESS: no congestion
5511  *      NET_RX_DROP: packet was dropped
5512  */
5513 int netif_receive_skb_core(struct sk_buff *skb)
5514 {
5515         int ret;
5516
5517         rcu_read_lock();
5518         ret = __netif_receive_skb_one_core(skb, false);
5519         rcu_read_unlock();
5520
5521         return ret;
5522 }
5523 EXPORT_SYMBOL(netif_receive_skb_core);
5524
5525 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5526                                                   struct packet_type *pt_prev,
5527                                                   struct net_device *orig_dev)
5528 {
5529         struct sk_buff *skb, *next;
5530
5531         if (!pt_prev)
5532                 return;
5533         if (list_empty(head))
5534                 return;
5535         if (pt_prev->list_func != NULL)
5536                 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5537                                    ip_list_rcv, head, pt_prev, orig_dev);
5538         else
5539                 list_for_each_entry_safe(skb, next, head, list) {
5540                         skb_list_del_init(skb);
5541                         pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5542                 }
5543 }
5544
5545 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5546 {
5547         /* Fast-path assumptions:
5548          * - There is no RX handler.
5549          * - Only one packet_type matches.
5550          * If either of these fails, we will end up doing some per-packet
5551          * processing in-line, then handling the 'last ptype' for the whole
5552          * sublist.  This can't cause out-of-order delivery to any single ptype,
5553          * because the 'last ptype' must be constant across the sublist, and all
5554          * other ptypes are handled per-packet.
5555          */
5556         /* Current (common) ptype of sublist */
5557         struct packet_type *pt_curr = NULL;
5558         /* Current (common) orig_dev of sublist */
5559         struct net_device *od_curr = NULL;
5560         struct list_head sublist;
5561         struct sk_buff *skb, *next;
5562
5563         INIT_LIST_HEAD(&sublist);
5564         list_for_each_entry_safe(skb, next, head, list) {
5565                 struct net_device *orig_dev = skb->dev;
5566                 struct packet_type *pt_prev = NULL;
5567
5568                 skb_list_del_init(skb);
5569                 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5570                 if (!pt_prev)
5571                         continue;
5572                 if (pt_curr != pt_prev || od_curr != orig_dev) {
5573                         /* dispatch old sublist */
5574                         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5575                         /* start new sublist */
5576                         INIT_LIST_HEAD(&sublist);
5577                         pt_curr = pt_prev;
5578                         od_curr = orig_dev;
5579                 }
5580                 list_add_tail(&skb->list, &sublist);
5581         }
5582
5583         /* dispatch final sublist */
5584         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5585 }
5586
5587 static int __netif_receive_skb(struct sk_buff *skb)
5588 {
5589         int ret;
5590
5591         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5592                 unsigned int noreclaim_flag;
5593
5594                 /*
5595                  * PFMEMALLOC skbs are special, they should
5596                  * - be delivered to SOCK_MEMALLOC sockets only
5597                  * - stay away from userspace
5598                  * - have bounded memory usage
5599                  *
5600                  * Use PF_MEMALLOC as this saves us from propagating the allocation
5601                  * context down to all allocation sites.
5602                  */
5603                 noreclaim_flag = memalloc_noreclaim_save();
5604                 ret = __netif_receive_skb_one_core(skb, true);
5605                 memalloc_noreclaim_restore(noreclaim_flag);
5606         } else
5607                 ret = __netif_receive_skb_one_core(skb, false);
5608
5609         return ret;
5610 }
5611
5612 static void __netif_receive_skb_list(struct list_head *head)
5613 {
5614         unsigned long noreclaim_flag = 0;
5615         struct sk_buff *skb, *next;
5616         bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5617
5618         list_for_each_entry_safe(skb, next, head, list) {
5619                 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5620                         struct list_head sublist;
5621
5622                         /* Handle the previous sublist */
5623                         list_cut_before(&sublist, head, &skb->list);
5624                         if (!list_empty(&sublist))
5625                                 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5626                         pfmemalloc = !pfmemalloc;
5627                         /* See comments in __netif_receive_skb */
5628                         if (pfmemalloc)
5629                                 noreclaim_flag = memalloc_noreclaim_save();
5630                         else
5631                                 memalloc_noreclaim_restore(noreclaim_flag);
5632                 }
5633         }
5634         /* Handle the remaining sublist */
5635         if (!list_empty(head))
5636                 __netif_receive_skb_list_core(head, pfmemalloc);
5637         /* Restore pflags */
5638         if (pfmemalloc)
5639                 memalloc_noreclaim_restore(noreclaim_flag);
5640 }
5641
5642 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5643 {
5644         struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5645         struct bpf_prog *new = xdp->prog;
5646         int ret = 0;
5647
5648         switch (xdp->command) {
5649         case XDP_SETUP_PROG:
5650                 rcu_assign_pointer(dev->xdp_prog, new);
5651                 if (old)
5652                         bpf_prog_put(old);
5653
5654                 if (old && !new) {
5655                         static_branch_dec(&generic_xdp_needed_key);
5656                 } else if (new && !old) {
5657                         static_branch_inc(&generic_xdp_needed_key);
5658                         dev_disable_lro(dev);
5659                         dev_disable_gro_hw(dev);
5660                 }
5661                 break;
5662
5663         default:
5664                 ret = -EINVAL;
5665                 break;
5666         }
5667
5668         return ret;
5669 }
5670
5671 static int netif_receive_skb_internal(struct sk_buff *skb)
5672 {
5673         int ret;
5674
5675         net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5676
5677         if (skb_defer_rx_timestamp(skb))
5678                 return NET_RX_SUCCESS;
5679
5680         rcu_read_lock();
5681 #ifdef CONFIG_RPS
5682         if (static_branch_unlikely(&rps_needed)) {
5683                 struct rps_dev_flow voidflow, *rflow = &voidflow;
5684                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5685
5686                 if (cpu >= 0) {
5687                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5688                         rcu_read_unlock();
5689                         return ret;
5690                 }
5691         }
5692 #endif
5693         ret = __netif_receive_skb(skb);
5694         rcu_read_unlock();
5695         return ret;
5696 }
5697
5698 void netif_receive_skb_list_internal(struct list_head *head)
5699 {
5700         struct sk_buff *skb, *next;
5701         struct list_head sublist;
5702
5703         INIT_LIST_HEAD(&sublist);
5704         list_for_each_entry_safe(skb, next, head, list) {
5705                 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5706                 skb_list_del_init(skb);
5707                 if (!skb_defer_rx_timestamp(skb))
5708                         list_add_tail(&skb->list, &sublist);
5709         }
5710         list_splice_init(&sublist, head);
5711
5712         rcu_read_lock();
5713 #ifdef CONFIG_RPS
5714         if (static_branch_unlikely(&rps_needed)) {
5715                 list_for_each_entry_safe(skb, next, head, list) {
5716                         struct rps_dev_flow voidflow, *rflow = &voidflow;
5717                         int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5718
5719                         if (cpu >= 0) {
5720                                 /* Will be handled, remove from list */
5721                                 skb_list_del_init(skb);
5722                                 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5723                         }
5724                 }
5725         }
5726 #endif
5727         __netif_receive_skb_list(head);
5728         rcu_read_unlock();
5729 }
5730
5731 /**
5732  *      netif_receive_skb - process receive buffer from network
5733  *      @skb: buffer to process
5734  *
5735  *      netif_receive_skb() is the main receive data processing function.
5736  *      It always succeeds. The buffer may be dropped during processing
5737  *      for congestion control or by the protocol layers.
5738  *
5739  *      This function may only be called from softirq context and interrupts
5740  *      should be enabled.
5741  *
5742  *      Return values (usually ignored):
5743  *      NET_RX_SUCCESS: no congestion
5744  *      NET_RX_DROP: packet was dropped
5745  */
5746 int netif_receive_skb(struct sk_buff *skb)
5747 {
5748         int ret;
5749
5750         trace_netif_receive_skb_entry(skb);
5751
5752         ret = netif_receive_skb_internal(skb);
5753         trace_netif_receive_skb_exit(ret);
5754
5755         return ret;
5756 }
5757 EXPORT_SYMBOL(netif_receive_skb);
5758
5759 /**
5760  *      netif_receive_skb_list - process many receive buffers from network
5761  *      @head: list of skbs to process.
5762  *
5763  *      Since return value of netif_receive_skb() is normally ignored, and
5764  *      wouldn't be meaningful for a list, this function returns void.
5765  *
5766  *      This function may only be called from softirq context and interrupts
5767  *      should be enabled.
5768  */
5769 void netif_receive_skb_list(struct list_head *head)
5770 {
5771         struct sk_buff *skb;
5772
5773         if (list_empty(head))
5774                 return;
5775         if (trace_netif_receive_skb_list_entry_enabled()) {
5776                 list_for_each_entry(skb, head, list)
5777                         trace_netif_receive_skb_list_entry(skb);
5778         }
5779         netif_receive_skb_list_internal(head);
5780         trace_netif_receive_skb_list_exit(0);
5781 }
5782 EXPORT_SYMBOL(netif_receive_skb_list);
5783
5784 static DEFINE_PER_CPU(struct work_struct, flush_works);
5785
5786 /* Network device is going away, flush any packets still pending */
5787 static void flush_backlog(struct work_struct *work)
5788 {
5789         struct sk_buff *skb, *tmp;
5790         struct softnet_data *sd;
5791
5792         local_bh_disable();
5793         sd = this_cpu_ptr(&softnet_data);
5794
5795         rps_lock_irq_disable(sd);
5796         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5797                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5798                         __skb_unlink(skb, &sd->input_pkt_queue);
5799                         dev_kfree_skb_irq(skb);
5800                         input_queue_head_incr(sd);
5801                 }
5802         }
5803         rps_unlock_irq_enable(sd);
5804
5805         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5806                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5807                         __skb_unlink(skb, &sd->process_queue);
5808                         kfree_skb(skb);
5809                         input_queue_head_incr(sd);
5810                 }
5811         }
5812         local_bh_enable();
5813 }
5814
5815 static bool flush_required(int cpu)
5816 {
5817 #if IS_ENABLED(CONFIG_RPS)
5818         struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5819         bool do_flush;
5820
5821         rps_lock_irq_disable(sd);
5822
5823         /* as insertion into process_queue happens with the rps lock held,
5824          * process_queue access may race only with dequeue
5825          */
5826         do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5827                    !skb_queue_empty_lockless(&sd->process_queue);
5828         rps_unlock_irq_enable(sd);
5829
5830         return do_flush;
5831 #endif
5832         /* without RPS we can't safely check input_pkt_queue: during a
5833          * concurrent remote skb_queue_splice() we can detect as empty both
5834          * input_pkt_queue and process_queue even if the latter could end-up
5835          * containing a lot of packets.
5836          */
5837         return true;
5838 }
5839
5840 static void flush_all_backlogs(void)
5841 {
5842         static cpumask_t flush_cpus;
5843         unsigned int cpu;
5844
5845         /* since we are under rtnl lock protection we can use static data
5846          * for the cpumask and avoid allocating on stack the possibly
5847          * large mask
5848          */
5849         ASSERT_RTNL();
5850
5851         cpus_read_lock();
5852
5853         cpumask_clear(&flush_cpus);
5854         for_each_online_cpu(cpu) {
5855                 if (flush_required(cpu)) {
5856                         queue_work_on(cpu, system_highpri_wq,
5857                                       per_cpu_ptr(&flush_works, cpu));
5858                         cpumask_set_cpu(cpu, &flush_cpus);
5859                 }
5860         }
5861
5862         /* we can have in flight packet[s] on the cpus we are not flushing,
5863          * synchronize_net() in unregister_netdevice_many() will take care of
5864          * them
5865          */
5866         for_each_cpu(cpu, &flush_cpus)
5867                 flush_work(per_cpu_ptr(&flush_works, cpu));
5868
5869         cpus_read_unlock();
5870 }
5871
5872 static void net_rps_send_ipi(struct softnet_data *remsd)
5873 {
5874 #ifdef CONFIG_RPS
5875         while (remsd) {
5876                 struct softnet_data *next = remsd->rps_ipi_next;
5877
5878                 if (cpu_online(remsd->cpu))
5879                         smp_call_function_single_async(remsd->cpu, &remsd->csd);
5880                 remsd = next;
5881         }
5882 #endif
5883 }
5884
5885 /*
5886  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5887  * Note: called with local irq disabled, but exits with local irq enabled.
5888  */
5889 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5890 {
5891 #ifdef CONFIG_RPS
5892         struct softnet_data *remsd = sd->rps_ipi_list;
5893
5894         if (remsd) {
5895                 sd->rps_ipi_list = NULL;
5896
5897                 local_irq_enable();
5898
5899                 /* Send pending IPI's to kick RPS processing on remote cpus. */
5900                 net_rps_send_ipi(remsd);
5901         } else
5902 #endif
5903                 local_irq_enable();
5904 }
5905
5906 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5907 {
5908 #ifdef CONFIG_RPS
5909         return sd->rps_ipi_list != NULL;
5910 #else
5911         return false;
5912 #endif
5913 }
5914
5915 static int process_backlog(struct napi_struct *napi, int quota)
5916 {
5917         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5918         bool again = true;
5919         int work = 0;
5920
5921         /* Check if we have pending ipi, its better to send them now,
5922          * not waiting net_rx_action() end.
5923          */
5924         if (sd_has_rps_ipi_waiting(sd)) {
5925                 local_irq_disable();
5926                 net_rps_action_and_irq_enable(sd);
5927         }
5928
5929         napi->weight = READ_ONCE(dev_rx_weight);
5930         while (again) {
5931                 struct sk_buff *skb;
5932
5933                 while ((skb = __skb_dequeue(&sd->process_queue))) {
5934                         rcu_read_lock();
5935                         __netif_receive_skb(skb);
5936                         rcu_read_unlock();
5937                         input_queue_head_incr(sd);
5938                         if (++work >= quota)
5939                                 return work;
5940
5941                 }
5942
5943                 rps_lock_irq_disable(sd);
5944                 if (skb_queue_empty(&sd->input_pkt_queue)) {
5945                         /*
5946                          * Inline a custom version of __napi_complete().
5947                          * only current cpu owns and manipulates this napi,
5948                          * and NAPI_STATE_SCHED is the only possible flag set
5949                          * on backlog.
5950                          * We can use a plain write instead of clear_bit(),
5951                          * and we dont need an smp_mb() memory barrier.
5952                          */
5953                         napi->state = 0;
5954                         again = false;
5955                 } else {
5956                         skb_queue_splice_tail_init(&sd->input_pkt_queue,
5957                                                    &sd->process_queue);
5958                 }
5959                 rps_unlock_irq_enable(sd);
5960         }
5961
5962         return work;
5963 }
5964
5965 /**
5966  * __napi_schedule - schedule for receive
5967  * @n: entry to schedule
5968  *
5969  * The entry's receive function will be scheduled to run.
5970  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5971  */
5972 void __napi_schedule(struct napi_struct *n)
5973 {
5974         unsigned long flags;
5975
5976         local_irq_save(flags);
5977         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5978         local_irq_restore(flags);
5979 }
5980 EXPORT_SYMBOL(__napi_schedule);
5981
5982 /**
5983  *      napi_schedule_prep - check if napi can be scheduled
5984  *      @n: napi context
5985  *
5986  * Test if NAPI routine is already running, and if not mark
5987  * it as running.  This is used as a condition variable to
5988  * insure only one NAPI poll instance runs.  We also make
5989  * sure there is no pending NAPI disable.
5990  */
5991 bool napi_schedule_prep(struct napi_struct *n)
5992 {
5993         unsigned long new, val = READ_ONCE(n->state);
5994
5995         do {
5996                 if (unlikely(val & NAPIF_STATE_DISABLE))
5997                         return false;
5998                 new = val | NAPIF_STATE_SCHED;
5999
6000                 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6001                  * This was suggested by Alexander Duyck, as compiler
6002                  * emits better code than :
6003                  * if (val & NAPIF_STATE_SCHED)
6004                  *     new |= NAPIF_STATE_MISSED;
6005                  */
6006                 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6007                                                    NAPIF_STATE_MISSED;
6008         } while (!try_cmpxchg(&n->state, &val, new));
6009
6010         return !(val & NAPIF_STATE_SCHED);
6011 }
6012 EXPORT_SYMBOL(napi_schedule_prep);
6013
6014 /**
6015  * __napi_schedule_irqoff - schedule for receive
6016  * @n: entry to schedule
6017  *
6018  * Variant of __napi_schedule() assuming hard irqs are masked.
6019  *
6020  * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6021  * because the interrupt disabled assumption might not be true
6022  * due to force-threaded interrupts and spinlock substitution.
6023  */
6024 void __napi_schedule_irqoff(struct napi_struct *n)
6025 {
6026         if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6027                 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6028         else
6029                 __napi_schedule(n);
6030 }
6031 EXPORT_SYMBOL(__napi_schedule_irqoff);
6032
6033 bool napi_complete_done(struct napi_struct *n, int work_done)
6034 {
6035         unsigned long flags, val, new, timeout = 0;
6036         bool ret = true;
6037
6038         /*
6039          * 1) Don't let napi dequeue from the cpu poll list
6040          *    just in case its running on a different cpu.
6041          * 2) If we are busy polling, do nothing here, we have
6042          *    the guarantee we will be called later.
6043          */
6044         if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6045                                  NAPIF_STATE_IN_BUSY_POLL)))
6046                 return false;
6047
6048         if (work_done) {
6049                 if (n->gro_bitmask)
6050                         timeout = READ_ONCE(n->dev->gro_flush_timeout);
6051                 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
6052         }
6053         if (n->defer_hard_irqs_count > 0) {
6054                 n->defer_hard_irqs_count--;
6055                 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6056                 if (timeout)
6057                         ret = false;
6058         }
6059         if (n->gro_bitmask) {
6060                 /* When the NAPI instance uses a timeout and keeps postponing
6061                  * it, we need to bound somehow the time packets are kept in
6062                  * the GRO layer
6063                  */
6064                 napi_gro_flush(n, !!timeout);
6065         }
6066
6067         gro_normal_list(n);
6068
6069         if (unlikely(!list_empty(&n->poll_list))) {
6070                 /* If n->poll_list is not empty, we need to mask irqs */
6071                 local_irq_save(flags);
6072                 list_del_init(&n->poll_list);
6073                 local_irq_restore(flags);
6074         }
6075         WRITE_ONCE(n->list_owner, -1);
6076
6077         val = READ_ONCE(n->state);
6078         do {
6079                 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6080
6081                 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6082                               NAPIF_STATE_SCHED_THREADED |
6083                               NAPIF_STATE_PREFER_BUSY_POLL);
6084
6085                 /* If STATE_MISSED was set, leave STATE_SCHED set,
6086                  * because we will call napi->poll() one more time.
6087                  * This C code was suggested by Alexander Duyck to help gcc.
6088                  */
6089                 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6090                                                     NAPIF_STATE_SCHED;
6091         } while (!try_cmpxchg(&n->state, &val, new));
6092
6093         if (unlikely(val & NAPIF_STATE_MISSED)) {
6094                 __napi_schedule(n);
6095                 return false;
6096         }
6097
6098         if (timeout)
6099                 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6100                               HRTIMER_MODE_REL_PINNED);
6101         return ret;
6102 }
6103 EXPORT_SYMBOL(napi_complete_done);
6104
6105 /* must be called under rcu_read_lock(), as we dont take a reference */
6106 static struct napi_struct *napi_by_id(unsigned int napi_id)
6107 {
6108         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6109         struct napi_struct *napi;
6110
6111         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6112                 if (napi->napi_id == napi_id)
6113                         return napi;
6114
6115         return NULL;
6116 }
6117
6118 #if defined(CONFIG_NET_RX_BUSY_POLL)
6119
6120 static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6121 {
6122         if (!skip_schedule) {
6123                 gro_normal_list(napi);
6124                 __napi_schedule(napi);
6125                 return;
6126         }
6127
6128         if (napi->gro_bitmask) {
6129                 /* flush too old packets
6130                  * If HZ < 1000, flush all packets.
6131                  */
6132                 napi_gro_flush(napi, HZ >= 1000);
6133         }
6134
6135         gro_normal_list(napi);
6136         clear_bit(NAPI_STATE_SCHED, &napi->state);
6137 }
6138
6139 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6140                            u16 budget)
6141 {
6142         bool skip_schedule = false;
6143         unsigned long timeout;
6144         int rc;
6145
6146         /* Busy polling means there is a high chance device driver hard irq
6147          * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6148          * set in napi_schedule_prep().
6149          * Since we are about to call napi->poll() once more, we can safely
6150          * clear NAPI_STATE_MISSED.
6151          *
6152          * Note: x86 could use a single "lock and ..." instruction
6153          * to perform these two clear_bit()
6154          */
6155         clear_bit(NAPI_STATE_MISSED, &napi->state);
6156         clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6157
6158         local_bh_disable();
6159
6160         if (prefer_busy_poll) {
6161                 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6162                 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6163                 if (napi->defer_hard_irqs_count && timeout) {
6164                         hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6165                         skip_schedule = true;
6166                 }
6167         }
6168
6169         /* All we really want here is to re-enable device interrupts.
6170          * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6171          */
6172         rc = napi->poll(napi, budget);
6173         /* We can't gro_normal_list() here, because napi->poll() might have
6174          * rearmed the napi (napi_complete_done()) in which case it could
6175          * already be running on another CPU.
6176          */
6177         trace_napi_poll(napi, rc, budget);
6178         netpoll_poll_unlock(have_poll_lock);
6179         if (rc == budget)
6180                 __busy_poll_stop(napi, skip_schedule);
6181         local_bh_enable();
6182 }
6183
6184 void napi_busy_loop(unsigned int napi_id,
6185                     bool (*loop_end)(void *, unsigned long),
6186                     void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6187 {
6188         unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6189         int (*napi_poll)(struct napi_struct *napi, int budget);
6190         void *have_poll_lock = NULL;
6191         struct napi_struct *napi;
6192
6193 restart:
6194         napi_poll = NULL;
6195
6196         rcu_read_lock();
6197
6198         napi = napi_by_id(napi_id);
6199         if (!napi)
6200                 goto out;
6201
6202         preempt_disable();
6203         for (;;) {
6204                 int work = 0;
6205
6206                 local_bh_disable();
6207                 if (!napi_poll) {
6208                         unsigned long val = READ_ONCE(napi->state);
6209
6210                         /* If multiple threads are competing for this napi,
6211                          * we avoid dirtying napi->state as much as we can.
6212                          */
6213                         if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6214                                    NAPIF_STATE_IN_BUSY_POLL)) {
6215                                 if (prefer_busy_poll)
6216                                         set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6217                                 goto count;
6218                         }
6219                         if (cmpxchg(&napi->state, val,
6220                                     val | NAPIF_STATE_IN_BUSY_POLL |
6221                                           NAPIF_STATE_SCHED) != val) {
6222                                 if (prefer_busy_poll)
6223                                         set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6224                                 goto count;
6225                         }
6226                         have_poll_lock = netpoll_poll_lock(napi);
6227                         napi_poll = napi->poll;
6228                 }
6229                 work = napi_poll(napi, budget);
6230                 trace_napi_poll(napi, work, budget);
6231                 gro_normal_list(napi);
6232 count:
6233                 if (work > 0)
6234                         __NET_ADD_STATS(dev_net(napi->dev),
6235                                         LINUX_MIB_BUSYPOLLRXPACKETS, work);
6236                 local_bh_enable();
6237
6238                 if (!loop_end || loop_end(loop_end_arg, start_time))
6239                         break;
6240
6241                 if (unlikely(need_resched())) {
6242                         if (napi_poll)
6243                                 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6244                         preempt_enable();
6245                         rcu_read_unlock();
6246                         cond_resched();
6247                         if (loop_end(loop_end_arg, start_time))
6248                                 return;
6249                         goto restart;
6250                 }
6251                 cpu_relax();
6252         }
6253         if (napi_poll)
6254                 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6255         preempt_enable();
6256 out:
6257         rcu_read_unlock();
6258 }
6259 EXPORT_SYMBOL(napi_busy_loop);
6260
6261 #endif /* CONFIG_NET_RX_BUSY_POLL */
6262
6263 static void napi_hash_add(struct napi_struct *napi)
6264 {
6265         if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6266                 return;
6267
6268         spin_lock(&napi_hash_lock);
6269
6270         /* 0..NR_CPUS range is reserved for sender_cpu use */
6271         do {
6272                 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6273                         napi_gen_id = MIN_NAPI_ID;
6274         } while (napi_by_id(napi_gen_id));
6275         napi->napi_id = napi_gen_id;
6276
6277         hlist_add_head_rcu(&napi->napi_hash_node,
6278                            &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6279
6280         spin_unlock(&napi_hash_lock);
6281 }
6282
6283 /* Warning : caller is responsible to make sure rcu grace period
6284  * is respected before freeing memory containing @napi
6285  */
6286 static void napi_hash_del(struct napi_struct *napi)
6287 {
6288         spin_lock(&napi_hash_lock);
6289
6290         hlist_del_init_rcu(&napi->napi_hash_node);
6291
6292         spin_unlock(&napi_hash_lock);
6293 }
6294
6295 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6296 {
6297         struct napi_struct *napi;
6298
6299         napi = container_of(timer, struct napi_struct, timer);
6300
6301         /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6302          * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6303          */
6304         if (!napi_disable_pending(napi) &&
6305             !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6306                 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6307                 __napi_schedule_irqoff(napi);
6308         }
6309
6310         return HRTIMER_NORESTART;
6311 }
6312
6313 static void init_gro_hash(struct napi_struct *napi)
6314 {
6315         int i;
6316
6317         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6318                 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6319                 napi->gro_hash[i].count = 0;
6320         }
6321         napi->gro_bitmask = 0;
6322 }
6323
6324 int dev_set_threaded(struct net_device *dev, bool threaded)
6325 {
6326         struct napi_struct *napi;
6327         int err = 0;
6328
6329         if (dev->threaded == threaded)
6330                 return 0;
6331
6332         if (threaded) {
6333                 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6334                         if (!napi->thread) {
6335                                 err = napi_kthread_create(napi);
6336                                 if (err) {
6337                                         threaded = false;
6338                                         break;
6339                                 }
6340                         }
6341                 }
6342         }
6343
6344         dev->threaded = threaded;
6345
6346         /* Make sure kthread is created before THREADED bit
6347          * is set.
6348          */
6349         smp_mb__before_atomic();
6350
6351         /* Setting/unsetting threaded mode on a napi might not immediately
6352          * take effect, if the current napi instance is actively being
6353          * polled. In this case, the switch between threaded mode and
6354          * softirq mode will happen in the next round of napi_schedule().
6355          * This should not cause hiccups/stalls to the live traffic.
6356          */
6357         list_for_each_entry(napi, &dev->napi_list, dev_list) {
6358                 if (threaded)
6359                         set_bit(NAPI_STATE_THREADED, &napi->state);
6360                 else
6361                         clear_bit(NAPI_STATE_THREADED, &napi->state);
6362         }
6363
6364         return err;
6365 }
6366 EXPORT_SYMBOL(dev_set_threaded);
6367
6368 void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
6369                            int (*poll)(struct napi_struct *, int), int weight)
6370 {
6371         if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6372                 return;
6373
6374         INIT_LIST_HEAD(&napi->poll_list);
6375         INIT_HLIST_NODE(&napi->napi_hash_node);
6376         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6377         napi->timer.function = napi_watchdog;
6378         init_gro_hash(napi);
6379         napi->skb = NULL;
6380         INIT_LIST_HEAD(&napi->rx_list);
6381         napi->rx_count = 0;
6382         napi->poll = poll;
6383         if (weight > NAPI_POLL_WEIGHT)
6384                 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6385                                 weight);
6386         napi->weight = weight;
6387         napi->dev = dev;
6388 #ifdef CONFIG_NETPOLL
6389         napi->poll_owner = -1;
6390 #endif
6391         napi->list_owner = -1;
6392         set_bit(NAPI_STATE_SCHED, &napi->state);
6393         set_bit(NAPI_STATE_NPSVC, &napi->state);
6394         list_add_rcu(&napi->dev_list, &dev->napi_list);
6395         napi_hash_add(napi);
6396         napi_get_frags_check(napi);
6397         /* Create kthread for this napi if dev->threaded is set.
6398          * Clear dev->threaded if kthread creation failed so that
6399          * threaded mode will not be enabled in napi_enable().
6400          */
6401         if (dev->threaded && napi_kthread_create(napi))
6402                 dev->threaded = 0;
6403 }
6404 EXPORT_SYMBOL(netif_napi_add_weight);
6405
6406 void napi_disable(struct napi_struct *n)
6407 {
6408         unsigned long val, new;
6409
6410         might_sleep();
6411         set_bit(NAPI_STATE_DISABLE, &n->state);
6412
6413         val = READ_ONCE(n->state);
6414         do {
6415                 while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
6416                         usleep_range(20, 200);
6417                         val = READ_ONCE(n->state);
6418                 }
6419
6420                 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC;
6421                 new &= ~(NAPIF_STATE_THREADED | NAPIF_STATE_PREFER_BUSY_POLL);
6422         } while (!try_cmpxchg(&n->state, &val, new));
6423
6424         hrtimer_cancel(&n->timer);
6425
6426         clear_bit(NAPI_STATE_DISABLE, &n->state);
6427 }
6428 EXPORT_SYMBOL(napi_disable);
6429
6430 /**
6431  *      napi_enable - enable NAPI scheduling
6432  *      @n: NAPI context
6433  *
6434  * Resume NAPI from being scheduled on this context.
6435  * Must be paired with napi_disable.
6436  */
6437 void napi_enable(struct napi_struct *n)
6438 {
6439         unsigned long new, val = READ_ONCE(n->state);
6440
6441         do {
6442                 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
6443
6444                 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC);
6445                 if (n->dev->threaded && n->thread)
6446                         new |= NAPIF_STATE_THREADED;
6447         } while (!try_cmpxchg(&n->state, &val, new));
6448 }
6449 EXPORT_SYMBOL(napi_enable);
6450
6451 static void flush_gro_hash(struct napi_struct *napi)
6452 {
6453         int i;
6454
6455         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6456                 struct sk_buff *skb, *n;
6457
6458                 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6459                         kfree_skb(skb);
6460                 napi->gro_hash[i].count = 0;
6461         }
6462 }
6463
6464 /* Must be called in process context */
6465 void __netif_napi_del(struct napi_struct *napi)
6466 {
6467         if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6468                 return;
6469
6470         napi_hash_del(napi);
6471         list_del_rcu(&napi->dev_list);
6472         napi_free_frags(napi);
6473
6474         flush_gro_hash(napi);
6475         napi->gro_bitmask = 0;
6476
6477         if (napi->thread) {
6478                 kthread_stop(napi->thread);
6479                 napi->thread = NULL;
6480         }
6481 }
6482 EXPORT_SYMBOL(__netif_napi_del);
6483
6484 static int __napi_poll(struct napi_struct *n, bool *repoll)
6485 {
6486         int work, weight;
6487
6488         weight = n->weight;
6489
6490         /* This NAPI_STATE_SCHED test is for avoiding a race
6491          * with netpoll's poll_napi().  Only the entity which
6492          * obtains the lock and sees NAPI_STATE_SCHED set will
6493          * actually make the ->poll() call.  Therefore we avoid
6494          * accidentally calling ->poll() when NAPI is not scheduled.
6495          */
6496         work = 0;
6497         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
6498                 work = n->poll(n, weight);
6499                 trace_napi_poll(n, work, weight);
6500         }
6501
6502         if (unlikely(work > weight))
6503                 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
6504                                 n->poll, work, weight);
6505
6506         if (likely(work < weight))
6507                 return work;
6508
6509         /* Drivers must not modify the NAPI state if they
6510          * consume the entire weight.  In such cases this code
6511          * still "owns" the NAPI instance and therefore can
6512          * move the instance around on the list at-will.
6513          */
6514         if (unlikely(napi_disable_pending(n))) {
6515                 napi_complete(n);
6516                 return work;
6517         }
6518
6519         /* The NAPI context has more processing work, but busy-polling
6520          * is preferred. Exit early.
6521          */
6522         if (napi_prefer_busy_poll(n)) {
6523                 if (napi_complete_done(n, work)) {
6524                         /* If timeout is not set, we need to make sure
6525                          * that the NAPI is re-scheduled.
6526                          */
6527                         napi_schedule(n);
6528                 }
6529                 return work;
6530         }
6531
6532         if (n->gro_bitmask) {
6533                 /* flush too old packets
6534                  * If HZ < 1000, flush all packets.
6535                  */
6536                 napi_gro_flush(n, HZ >= 1000);
6537         }
6538
6539         gro_normal_list(n);
6540
6541         /* Some drivers may have called napi_schedule
6542          * prior to exhausting their budget.
6543          */
6544         if (unlikely(!list_empty(&n->poll_list))) {
6545                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6546                              n->dev ? n->dev->name : "backlog");
6547                 return work;
6548         }
6549
6550         *repoll = true;
6551
6552         return work;
6553 }
6554
6555 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6556 {
6557         bool do_repoll = false;
6558         void *have;
6559         int work;
6560
6561         list_del_init(&n->poll_list);
6562
6563         have = netpoll_poll_lock(n);
6564
6565         work = __napi_poll(n, &do_repoll);
6566
6567         if (do_repoll)
6568                 list_add_tail(&n->poll_list, repoll);
6569
6570         netpoll_poll_unlock(have);
6571
6572         return work;
6573 }
6574
6575 static int napi_thread_wait(struct napi_struct *napi)
6576 {
6577         bool woken = false;
6578
6579         set_current_state(TASK_INTERRUPTIBLE);
6580
6581         while (!kthread_should_stop()) {
6582                 /* Testing SCHED_THREADED bit here to make sure the current
6583                  * kthread owns this napi and could poll on this napi.
6584                  * Testing SCHED bit is not enough because SCHED bit might be
6585                  * set by some other busy poll thread or by napi_disable().
6586                  */
6587                 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
6588                         WARN_ON(!list_empty(&napi->poll_list));
6589                         __set_current_state(TASK_RUNNING);
6590                         return 0;
6591                 }
6592
6593                 schedule();
6594                 /* woken being true indicates this thread owns this napi. */
6595                 woken = true;
6596                 set_current_state(TASK_INTERRUPTIBLE);
6597         }
6598         __set_current_state(TASK_RUNNING);
6599
6600         return -1;
6601 }
6602
6603 static void skb_defer_free_flush(struct softnet_data *sd)
6604 {
6605         struct sk_buff *skb, *next;
6606
6607         /* Paired with WRITE_ONCE() in skb_attempt_defer_free() */
6608         if (!READ_ONCE(sd->defer_list))
6609                 return;
6610
6611         spin_lock(&sd->defer_lock);
6612         skb = sd->defer_list;
6613         sd->defer_list = NULL;
6614         sd->defer_count = 0;
6615         spin_unlock(&sd->defer_lock);
6616
6617         while (skb != NULL) {
6618                 next = skb->next;
6619                 napi_consume_skb(skb, 1);
6620                 skb = next;
6621         }
6622 }
6623
6624 static int napi_threaded_poll(void *data)
6625 {
6626         struct napi_struct *napi = data;
6627         struct softnet_data *sd;
6628         void *have;
6629
6630         while (!napi_thread_wait(napi)) {
6631                 for (;;) {
6632                         bool repoll = false;
6633
6634                         local_bh_disable();
6635                         sd = this_cpu_ptr(&softnet_data);
6636                         sd->in_napi_threaded_poll = true;
6637
6638                         have = netpoll_poll_lock(napi);
6639                         __napi_poll(napi, &repoll);
6640                         netpoll_poll_unlock(have);
6641
6642                         sd->in_napi_threaded_poll = false;
6643                         barrier();
6644
6645                         if (sd_has_rps_ipi_waiting(sd)) {
6646                                 local_irq_disable();
6647                                 net_rps_action_and_irq_enable(sd);
6648                         }
6649                         skb_defer_free_flush(sd);
6650                         local_bh_enable();
6651
6652                         if (!repoll)
6653                                 break;
6654
6655                         cond_resched();
6656                 }
6657         }
6658         return 0;
6659 }
6660
6661 static __latent_entropy void net_rx_action(struct softirq_action *h)
6662 {
6663         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6664         unsigned long time_limit = jiffies +
6665                 usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));
6666         int budget = READ_ONCE(netdev_budget);
6667         LIST_HEAD(list);
6668         LIST_HEAD(repoll);
6669
6670 start:
6671         sd->in_net_rx_action = true;
6672         local_irq_disable();
6673         list_splice_init(&sd->poll_list, &list);
6674         local_irq_enable();
6675
6676         for (;;) {
6677                 struct napi_struct *n;
6678
6679                 skb_defer_free_flush(sd);
6680
6681                 if (list_empty(&list)) {
6682                         if (list_empty(&repoll)) {
6683                                 sd->in_net_rx_action = false;
6684                                 barrier();
6685                                 /* We need to check if ____napi_schedule()
6686                                  * had refilled poll_list while
6687                                  * sd->in_net_rx_action was true.
6688                                  */
6689                                 if (!list_empty(&sd->poll_list))
6690                                         goto start;
6691                                 if (!sd_has_rps_ipi_waiting(sd))
6692                                         goto end;
6693                         }
6694                         break;
6695                 }
6696
6697                 n = list_first_entry(&list, struct napi_struct, poll_list);
6698                 budget -= napi_poll(n, &repoll);
6699
6700                 /* If softirq window is exhausted then punt.
6701                  * Allow this to run for 2 jiffies since which will allow
6702                  * an average latency of 1.5/HZ.
6703                  */
6704                 if (unlikely(budget <= 0 ||
6705                              time_after_eq(jiffies, time_limit))) {
6706                         sd->time_squeeze++;
6707                         break;
6708                 }
6709         }
6710
6711         local_irq_disable();
6712
6713         list_splice_tail_init(&sd->poll_list, &list);
6714         list_splice_tail(&repoll, &list);
6715         list_splice(&list, &sd->poll_list);
6716         if (!list_empty(&sd->poll_list))
6717                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
6718         else
6719                 sd->in_net_rx_action = false;
6720
6721         net_rps_action_and_irq_enable(sd);
6722 end:;
6723 }
6724
6725 struct netdev_adjacent {
6726         struct net_device *dev;
6727         netdevice_tracker dev_tracker;
6728
6729         /* upper master flag, there can only be one master device per list */
6730         bool master;
6731
6732         /* lookup ignore flag */
6733         bool ignore;
6734
6735         /* counter for the number of times this device was added to us */
6736         u16 ref_nr;
6737
6738         /* private field for the users */
6739         void *private;
6740
6741         struct list_head list;
6742         struct rcu_head rcu;
6743 };
6744
6745 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
6746                                                  struct list_head *adj_list)
6747 {
6748         struct netdev_adjacent *adj;
6749
6750         list_for_each_entry(adj, adj_list, list) {
6751                 if (adj->dev == adj_dev)
6752                         return adj;
6753         }
6754         return NULL;
6755 }
6756
6757 static int ____netdev_has_upper_dev(struct net_device *upper_dev,
6758                                     struct netdev_nested_priv *priv)
6759 {
6760         struct net_device *dev = (struct net_device *)priv->data;
6761
6762         return upper_dev == dev;
6763 }
6764
6765 /**
6766  * netdev_has_upper_dev - Check if device is linked to an upper device
6767  * @dev: device
6768  * @upper_dev: upper device to check
6769  *
6770  * Find out if a device is linked to specified upper device and return true
6771  * in case it is. Note that this checks only immediate upper device,
6772  * not through a complete stack of devices. The caller must hold the RTNL lock.
6773  */
6774 bool netdev_has_upper_dev(struct net_device *dev,
6775                           struct net_device *upper_dev)
6776 {
6777         struct netdev_nested_priv priv = {
6778                 .data = (void *)upper_dev,
6779         };
6780
6781         ASSERT_RTNL();
6782
6783         return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6784                                              &priv);
6785 }
6786 EXPORT_SYMBOL(netdev_has_upper_dev);
6787
6788 /**
6789  * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
6790  * @dev: device
6791  * @upper_dev: upper device to check
6792  *
6793  * Find out if a device is linked to specified upper device and return true
6794  * in case it is. Note that this checks the entire upper device chain.
6795  * The caller must hold rcu lock.
6796  */
6797
6798 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
6799                                   struct net_device *upper_dev)
6800 {
6801         struct netdev_nested_priv priv = {
6802                 .data = (void *)upper_dev,
6803         };
6804
6805         return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6806                                                &priv);
6807 }
6808 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
6809
6810 /**
6811  * netdev_has_any_upper_dev - Check if device is linked to some device
6812  * @dev: device
6813  *
6814  * Find out if a device is linked to an upper device and return true in case
6815  * it is. The caller must hold the RTNL lock.
6816  */
6817 bool netdev_has_any_upper_dev(struct net_device *dev)
6818 {
6819         ASSERT_RTNL();
6820
6821         return !list_empty(&dev->adj_list.upper);
6822 }
6823 EXPORT_SYMBOL(netdev_has_any_upper_dev);
6824
6825 /**
6826  * netdev_master_upper_dev_get - Get master upper device
6827  * @dev: device
6828  *
6829  * Find a master upper device and return pointer to it or NULL in case
6830  * it's not there. The caller must hold the RTNL lock.
6831  */
6832 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
6833 {
6834         struct netdev_adjacent *upper;
6835
6836         ASSERT_RTNL();
6837
6838         if (list_empty(&dev->adj_list.upper))
6839                 return NULL;
6840
6841         upper = list_first_entry(&dev->adj_list.upper,
6842                                  struct netdev_adjacent, list);
6843         if (likely(upper->master))
6844                 return upper->dev;
6845         return NULL;
6846 }
6847 EXPORT_SYMBOL(netdev_master_upper_dev_get);
6848
6849 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
6850 {
6851         struct netdev_adjacent *upper;
6852
6853         ASSERT_RTNL();
6854
6855         if (list_empty(&dev->adj_list.upper))
6856                 return NULL;
6857
6858         upper = list_first_entry(&dev->adj_list.upper,
6859                                  struct netdev_adjacent, list);
6860         if (likely(upper->master) && !upper->ignore)
6861                 return upper->dev;
6862         return NULL;
6863 }
6864
6865 /**
6866  * netdev_has_any_lower_dev - Check if device is linked to some device
6867  * @dev: device
6868  *
6869  * Find out if a device is linked to a lower device and return true in case
6870  * it is. The caller must hold the RTNL lock.
6871  */
6872 static bool netdev_has_any_lower_dev(struct net_device *dev)
6873 {
6874         ASSERT_RTNL();
6875
6876         return !list_empty(&dev->adj_list.lower);
6877 }
6878
6879 void *netdev_adjacent_get_private(struct list_head *adj_list)
6880 {
6881         struct netdev_adjacent *adj;
6882
6883         adj = list_entry(adj_list, struct netdev_adjacent, list);
6884
6885         return adj->private;
6886 }
6887 EXPORT_SYMBOL(netdev_adjacent_get_private);
6888
6889 /**
6890  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
6891  * @dev: device
6892  * @iter: list_head ** of the current position
6893  *
6894  * Gets the next device from the dev's upper list, starting from iter
6895  * position. The caller must hold RCU read lock.
6896  */
6897 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
6898                                                  struct list_head **iter)
6899 {
6900         struct netdev_adjacent *upper;
6901
6902         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6903
6904         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6905
6906         if (&upper->list == &dev->adj_list.upper)
6907                 return NULL;
6908
6909         *iter = &upper->list;
6910
6911         return upper->dev;
6912 }
6913 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
6914
6915 static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
6916                                                   struct list_head **iter,
6917                                                   bool *ignore)
6918 {
6919         struct netdev_adjacent *upper;
6920
6921         upper = list_entry((*iter)->next, struct netdev_adjacent, list);
6922
6923         if (&upper->list == &dev->adj_list.upper)
6924                 return NULL;
6925
6926         *iter = &upper->list;
6927         *ignore = upper->ignore;
6928
6929         return upper->dev;
6930 }
6931
6932 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
6933                                                     struct list_head **iter)
6934 {
6935         struct netdev_adjacent *upper;
6936
6937         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6938
6939         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6940
6941         if (&upper->list == &dev->adj_list.upper)
6942                 return NULL;
6943
6944         *iter = &upper->list;
6945
6946         return upper->dev;
6947 }
6948
6949 static int __netdev_walk_all_upper_dev(struct net_device *dev,
6950                                        int (*fn)(struct net_device *dev,
6951                                          struct netdev_nested_priv *priv),
6952                                        struct netdev_nested_priv *priv)
6953 {
6954         struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
6955         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
6956         int ret, cur = 0;
6957         bool ignore;
6958
6959         now = dev;
6960         iter = &dev->adj_list.upper;
6961
6962         while (1) {
6963                 if (now != dev) {
6964                         ret = fn(now, priv);
6965                         if (ret)
6966                                 return ret;
6967                 }
6968
6969                 next = NULL;
6970                 while (1) {
6971                         udev = __netdev_next_upper_dev(now, &iter, &ignore);
6972                         if (!udev)
6973                                 break;
6974                         if (ignore)
6975                                 continue;
6976
6977                         next = udev;
6978                         niter = &udev->adj_list.upper;
6979                         dev_stack[cur] = now;
6980                         iter_stack[cur++] = iter;
6981                         break;
6982                 }
6983
6984                 if (!next) {
6985                         if (!cur)
6986                                 return 0;
6987                         next = dev_stack[--cur];
6988                         niter = iter_stack[cur];
6989                 }
6990
6991                 now = next;
6992                 iter = niter;
6993         }
6994
6995         return 0;
6996 }
6997
6998 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
6999                                   int (*fn)(struct net_device *dev,
7000                                             struct netdev_nested_priv *priv),
7001                                   struct netdev_nested_priv *priv)
7002 {
7003         struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7004         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7005         int ret, cur = 0;
7006
7007         now = dev;
7008         iter = &dev->adj_list.upper;
7009
7010         while (1) {
7011                 if (now != dev) {
7012                         ret = fn(now, priv);
7013                         if (ret)
7014                                 return ret;
7015                 }
7016
7017                 next = NULL;
7018                 while (1) {
7019                         udev = netdev_next_upper_dev_rcu(now, &iter);
7020                         if (!udev)
7021                                 break;
7022
7023                         next = udev;
7024                         niter = &udev->adj_list.upper;
7025                         dev_stack[cur] = now;
7026                         iter_stack[cur++] = iter;
7027                         break;
7028                 }
7029
7030                 if (!next) {
7031                         if (!cur)
7032                                 return 0;
7033                         next = dev_stack[--cur];
7034                         niter = iter_stack[cur];
7035                 }
7036
7037                 now = next;
7038                 iter = niter;
7039         }
7040
7041         return 0;
7042 }
7043 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7044
7045 static bool __netdev_has_upper_dev(struct net_device *dev,
7046                                    struct net_device *upper_dev)
7047 {
7048         struct netdev_nested_priv priv = {
7049                 .flags = 0,
7050                 .data = (void *)upper_dev,
7051         };
7052
7053         ASSERT_RTNL();
7054
7055         return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
7056                                            &priv);
7057 }
7058
7059 /**
7060  * netdev_lower_get_next_private - Get the next ->private from the
7061  *                                 lower neighbour list
7062  * @dev: device
7063  * @iter: list_head ** of the current position
7064  *
7065  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7066  * list, starting from iter position. The caller must hold either hold the
7067  * RTNL lock or its own locking that guarantees that the neighbour lower
7068  * list will remain unchanged.
7069  */
7070 void *netdev_lower_get_next_private(struct net_device *dev,
7071                                     struct list_head **iter)
7072 {
7073         struct netdev_adjacent *lower;
7074
7075         lower = list_entry(*iter, struct netdev_adjacent, list);
7076
7077         if (&lower->list == &dev->adj_list.lower)
7078                 return NULL;
7079
7080         *iter = lower->list.next;
7081
7082         return lower->private;
7083 }
7084 EXPORT_SYMBOL(netdev_lower_get_next_private);
7085
7086 /**
7087  * netdev_lower_get_next_private_rcu - Get the next ->private from the
7088  *                                     lower neighbour list, RCU
7089  *                                     variant
7090  * @dev: device
7091  * @iter: list_head ** of the current position
7092  *
7093  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7094  * list, starting from iter position. The caller must hold RCU read lock.
7095  */
7096 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7097                                         struct list_head **iter)
7098 {
7099         struct netdev_adjacent *lower;
7100
7101         WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
7102
7103         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7104
7105         if (&lower->list == &dev->adj_list.lower)
7106                 return NULL;
7107
7108         *iter = &lower->list;
7109
7110         return lower->private;
7111 }
7112 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7113
7114 /**
7115  * netdev_lower_get_next - Get the next device from the lower neighbour
7116  *                         list
7117  * @dev: device
7118  * @iter: list_head ** of the current position
7119  *
7120  * Gets the next netdev_adjacent from the dev's lower neighbour
7121  * list, starting from iter position. The caller must hold RTNL lock or
7122  * its own locking that guarantees that the neighbour lower
7123  * list will remain unchanged.
7124  */
7125 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7126 {
7127         struct netdev_adjacent *lower;
7128
7129         lower = list_entry(*iter, struct netdev_adjacent, list);
7130
7131         if (&lower->list == &dev->adj_list.lower)
7132                 return NULL;
7133
7134         *iter = lower->list.next;
7135
7136         return lower->dev;
7137 }
7138 EXPORT_SYMBOL(netdev_lower_get_next);
7139
7140 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7141                                                 struct list_head **iter)
7142 {
7143         struct netdev_adjacent *lower;
7144
7145         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7146
7147         if (&lower->list == &dev->adj_list.lower)
7148                 return NULL;
7149
7150         *iter = &lower->list;
7151
7152         return lower->dev;
7153 }
7154
7155 static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7156                                                   struct list_head **iter,
7157                                                   bool *ignore)
7158 {
7159         struct netdev_adjacent *lower;
7160
7161         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7162
7163         if (&lower->list == &dev->adj_list.lower)
7164                 return NULL;
7165
7166         *iter = &lower->list;
7167         *ignore = lower->ignore;
7168
7169         return lower->dev;
7170 }
7171
7172 int netdev_walk_all_lower_dev(struct net_device *dev,
7173                               int (*fn)(struct net_device *dev,
7174                                         struct netdev_nested_priv *priv),
7175                               struct netdev_nested_priv *priv)
7176 {
7177         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7178         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7179         int ret, cur = 0;
7180
7181         now = dev;
7182         iter = &dev->adj_list.lower;
7183
7184         while (1) {
7185                 if (now != dev) {
7186                         ret = fn(now, priv);
7187                         if (ret)
7188                                 return ret;
7189                 }
7190
7191                 next = NULL;
7192                 while (1) {
7193                         ldev = netdev_next_lower_dev(now, &iter);
7194                         if (!ldev)
7195                                 break;
7196
7197                         next = ldev;
7198                         niter = &ldev->adj_list.lower;
7199                         dev_stack[cur] = now;
7200                         iter_stack[cur++] = iter;
7201                         break;
7202                 }
7203
7204                 if (!next) {
7205                         if (!cur)
7206                                 return 0;
7207                         next = dev_stack[--cur];
7208                         niter = iter_stack[cur];
7209                 }
7210
7211                 now = next;
7212                 iter = niter;
7213         }
7214
7215         return 0;
7216 }
7217 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7218
7219 static int __netdev_walk_all_lower_dev(struct net_device *dev,
7220                                        int (*fn)(struct net_device *dev,
7221                                          struct netdev_nested_priv *priv),
7222                                        struct netdev_nested_priv *priv)
7223 {
7224         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7225         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7226         int ret, cur = 0;
7227         bool ignore;
7228
7229         now = dev;
7230         iter = &dev->adj_list.lower;
7231
7232         while (1) {
7233                 if (now != dev) {
7234                         ret = fn(now, priv);
7235                         if (ret)
7236                                 return ret;
7237                 }
7238
7239                 next = NULL;
7240                 while (1) {
7241                         ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7242                         if (!ldev)
7243                                 break;
7244                         if (ignore)
7245                                 continue;
7246
7247                         next = ldev;
7248                         niter = &ldev->adj_list.lower;
7249                         dev_stack[cur] = now;
7250                         iter_stack[cur++] = iter;
7251                         break;
7252                 }
7253
7254                 if (!next) {
7255                         if (!cur)
7256                                 return 0;
7257                         next = dev_stack[--cur];
7258                         niter = iter_stack[cur];
7259                 }
7260
7261                 now = next;
7262                 iter = niter;
7263         }
7264
7265         return 0;
7266 }
7267
7268 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7269                                              struct list_head **iter)
7270 {
7271         struct netdev_adjacent *lower;
7272
7273         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7274         if (&lower->list == &dev->adj_list.lower)
7275                 return NULL;
7276
7277         *iter = &lower->list;
7278
7279         return lower->dev;
7280 }
7281 EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
7282
7283 static u8 __netdev_upper_depth(struct net_device *dev)
7284 {
7285         struct net_device *udev;
7286         struct list_head *iter;
7287         u8 max_depth = 0;
7288         bool ignore;
7289
7290         for (iter = &dev->adj_list.upper,
7291              udev = __netdev_next_upper_dev(dev, &iter, &ignore);
7292              udev;
7293              udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7294                 if (ignore)
7295                         continue;
7296                 if (max_depth < udev->upper_level)
7297                         max_depth = udev->upper_level;
7298         }
7299
7300         return max_depth;
7301 }
7302
7303 static u8 __netdev_lower_depth(struct net_device *dev)
7304 {
7305         struct net_device *ldev;
7306         struct list_head *iter;
7307         u8 max_depth = 0;
7308         bool ignore;
7309
7310         for (iter = &dev->adj_list.lower,
7311              ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
7312              ldev;
7313              ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7314                 if (ignore)
7315                         continue;
7316                 if (max_depth < ldev->lower_level)
7317                         max_depth = ldev->lower_level;
7318         }
7319
7320         return max_depth;
7321 }
7322
7323 static int __netdev_update_upper_level(struct net_device *dev,
7324                                        struct netdev_nested_priv *__unused)
7325 {
7326         dev->upper_level = __netdev_upper_depth(dev) + 1;
7327         return 0;
7328 }
7329
7330 #ifdef CONFIG_LOCKDEP
7331 static LIST_HEAD(net_unlink_list);
7332
7333 static void net_unlink_todo(struct net_device *dev)
7334 {
7335         if (list_empty(&dev->unlink_list))
7336                 list_add_tail(&dev->unlink_list, &net_unlink_list);
7337 }
7338 #endif
7339
7340 static int __netdev_update_lower_level(struct net_device *dev,
7341                                        struct netdev_nested_priv *priv)
7342 {
7343         dev->lower_level = __netdev_lower_depth(dev) + 1;
7344
7345 #ifdef CONFIG_LOCKDEP
7346         if (!priv)
7347                 return 0;
7348
7349         if (priv->flags & NESTED_SYNC_IMM)
7350                 dev->nested_level = dev->lower_level - 1;
7351         if (priv->flags & NESTED_SYNC_TODO)
7352                 net_unlink_todo(dev);
7353 #endif
7354         return 0;
7355 }
7356
7357 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7358                                   int (*fn)(struct net_device *dev,
7359                                             struct netdev_nested_priv *priv),
7360                                   struct netdev_nested_priv *priv)
7361 {
7362         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7363         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7364         int ret, cur = 0;
7365
7366         now = dev;
7367         iter = &dev->adj_list.lower;
7368
7369         while (1) {
7370                 if (now != dev) {
7371                         ret = fn(now, priv);
7372                         if (ret)
7373                                 return ret;
7374                 }
7375
7376                 next = NULL;
7377                 while (1) {
7378                         ldev = netdev_next_lower_dev_rcu(now, &iter);
7379                         if (!ldev)
7380                                 break;
7381
7382                         next = ldev;
7383                         niter = &ldev->adj_list.lower;
7384                         dev_stack[cur] = now;
7385                         iter_stack[cur++] = iter;
7386                         break;
7387                 }
7388
7389                 if (!next) {
7390                         if (!cur)
7391                                 return 0;
7392                         next = dev_stack[--cur];
7393                         niter = iter_stack[cur];
7394                 }
7395
7396                 now = next;
7397                 iter = niter;
7398         }
7399
7400         return 0;
7401 }
7402 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7403
7404 /**
7405  * netdev_lower_get_first_private_rcu - Get the first ->private from the
7406  *                                     lower neighbour list, RCU
7407  *                                     variant
7408  * @dev: device
7409  *
7410  * Gets the first netdev_adjacent->private from the dev's lower neighbour
7411  * list. The caller must hold RCU read lock.
7412  */
7413 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7414 {
7415         struct netdev_adjacent *lower;
7416
7417         lower = list_first_or_null_rcu(&dev->adj_list.lower,
7418                         struct netdev_adjacent, list);
7419         if (lower)
7420                 return lower->private;
7421         return NULL;
7422 }
7423 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7424
7425 /**
7426  * netdev_master_upper_dev_get_rcu - Get master upper device
7427  * @dev: device
7428  *
7429  * Find a master upper device and return pointer to it or NULL in case
7430  * it's not there. The caller must hold the RCU read lock.
7431  */
7432 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7433 {
7434         struct netdev_adjacent *upper;
7435
7436         upper = list_first_or_null_rcu(&dev->adj_list.upper,
7437                                        struct netdev_adjacent, list);
7438         if (upper && likely(upper->master))
7439                 return upper->dev;
7440         return NULL;
7441 }
7442 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7443
7444 static int netdev_adjacent_sysfs_add(struct net_device *dev,
7445                               struct net_device *adj_dev,
7446                               struct list_head *dev_list)
7447 {
7448         char linkname[IFNAMSIZ+7];
7449
7450         sprintf(linkname, dev_list == &dev->adj_list.upper ?
7451                 "upper_%s" : "lower_%s", adj_dev->name);
7452         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7453                                  linkname);
7454 }
7455 static void netdev_adjacent_sysfs_del(struct net_device *dev,
7456                                char *name,
7457                                struct list_head *dev_list)
7458 {
7459         char linkname[IFNAMSIZ+7];
7460
7461         sprintf(linkname, dev_list == &dev->adj_list.upper ?
7462                 "upper_%s" : "lower_%s", name);
7463         sysfs_remove_link(&(dev->dev.kobj), linkname);
7464 }
7465
7466 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7467                                                  struct net_device *adj_dev,
7468                                                  struct list_head *dev_list)
7469 {
7470         return (dev_list == &dev->adj_list.upper ||
7471                 dev_list == &dev->adj_list.lower) &&
7472                 net_eq(dev_net(dev), dev_net(adj_dev));
7473 }
7474
7475 static int __netdev_adjacent_dev_insert(struct net_device *dev,
7476                                         struct net_device *adj_dev,
7477                                         struct list_head *dev_list,
7478                                         void *private, bool master)
7479 {
7480         struct netdev_adjacent *adj;
7481         int ret;
7482
7483         adj = __netdev_find_adj(adj_dev, dev_list);
7484
7485         if (adj) {
7486                 adj->ref_nr += 1;
7487                 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7488                          dev->name, adj_dev->name, adj->ref_nr);
7489
7490                 return 0;
7491         }
7492
7493         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
7494         if (!adj)
7495                 return -ENOMEM;
7496
7497         adj->dev = adj_dev;
7498         adj->master = master;
7499         adj->ref_nr = 1;
7500         adj->private = private;
7501         adj->ignore = false;
7502         netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
7503
7504         pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
7505                  dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
7506
7507         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
7508                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
7509                 if (ret)
7510                         goto free_adj;
7511         }
7512
7513         /* Ensure that master link is always the first item in list. */
7514         if (master) {
7515                 ret = sysfs_create_link(&(dev->dev.kobj),
7516                                         &(adj_dev->dev.kobj), "master");
7517                 if (ret)
7518                         goto remove_symlinks;
7519
7520                 list_add_rcu(&adj->list, dev_list);
7521         } else {
7522                 list_add_tail_rcu(&adj->list, dev_list);
7523         }
7524
7525         return 0;
7526
7527 remove_symlinks:
7528         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7529                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7530 free_adj:
7531         netdev_put(adj_dev, &adj->dev_tracker);
7532         kfree(adj);
7533
7534         return ret;
7535 }
7536
7537 static void __netdev_adjacent_dev_remove(struct net_device *dev,
7538                                          struct net_device *adj_dev,
7539                                          u16 ref_nr,
7540                                          struct list_head *dev_list)
7541 {
7542         struct netdev_adjacent *adj;
7543
7544         pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
7545                  dev->name, adj_dev->name, ref_nr);
7546
7547         adj = __netdev_find_adj(adj_dev, dev_list);
7548
7549         if (!adj) {
7550                 pr_err("Adjacency does not exist for device %s from %s\n",
7551                        dev->name, adj_dev->name);
7552                 WARN_ON(1);
7553                 return;
7554         }
7555
7556         if (adj->ref_nr > ref_nr) {
7557                 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
7558                          dev->name, adj_dev->name, ref_nr,
7559                          adj->ref_nr - ref_nr);
7560                 adj->ref_nr -= ref_nr;
7561                 return;
7562         }
7563
7564         if (adj->master)
7565                 sysfs_remove_link(&(dev->dev.kobj), "master");
7566
7567         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7568                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7569
7570         list_del_rcu(&adj->list);
7571         pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
7572                  adj_dev->name, dev->name, adj_dev->name);
7573         netdev_put(adj_dev, &adj->dev_tracker);
7574         kfree_rcu(adj, rcu);
7575 }
7576
7577 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
7578                                             struct net_device *upper_dev,
7579                                             struct list_head *up_list,
7580                                             struct list_head *down_list,
7581                                             void *private, bool master)
7582 {
7583         int ret;
7584
7585         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
7586                                            private, master);
7587         if (ret)
7588                 return ret;
7589
7590         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
7591                                            private, false);
7592         if (ret) {
7593                 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
7594                 return ret;
7595         }
7596
7597         return 0;
7598 }
7599
7600 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
7601                                                struct net_device *upper_dev,
7602                                                u16 ref_nr,
7603                                                struct list_head *up_list,
7604                                                struct list_head *down_list)
7605 {
7606         __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
7607         __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
7608 }
7609
7610 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
7611                                                 struct net_device *upper_dev,
7612                                                 void *private, bool master)
7613 {
7614         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
7615                                                 &dev->adj_list.upper,
7616                                                 &upper_dev->adj_list.lower,
7617                                                 private, master);
7618 }
7619
7620 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
7621                                                    struct net_device *upper_dev)
7622 {
7623         __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
7624                                            &dev->adj_list.upper,
7625                                            &upper_dev->adj_list.lower);
7626 }
7627
7628 static int __netdev_upper_dev_link(struct net_device *dev,
7629                                    struct net_device *upper_dev, bool master,
7630                                    void *upper_priv, void *upper_info,
7631                                    struct netdev_nested_priv *priv,
7632                                    struct netlink_ext_ack *extack)
7633 {
7634         struct netdev_notifier_changeupper_info changeupper_info = {
7635                 .info = {
7636                         .dev = dev,
7637                         .extack = extack,
7638                 },
7639                 .upper_dev = upper_dev,
7640                 .master = master,
7641                 .linking = true,
7642                 .upper_info = upper_info,
7643         };
7644         struct net_device *master_dev;
7645         int ret = 0;
7646
7647         ASSERT_RTNL();
7648
7649         if (dev == upper_dev)
7650                 return -EBUSY;
7651
7652         /* To prevent loops, check if dev is not upper device to upper_dev. */
7653         if (__netdev_has_upper_dev(upper_dev, dev))
7654                 return -EBUSY;
7655
7656         if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
7657                 return -EMLINK;
7658
7659         if (!master) {
7660                 if (__netdev_has_upper_dev(dev, upper_dev))
7661                         return -EEXIST;
7662         } else {
7663                 master_dev = __netdev_master_upper_dev_get(dev);
7664                 if (master_dev)
7665                         return master_dev == upper_dev ? -EEXIST : -EBUSY;
7666         }
7667
7668         ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7669                                             &changeupper_info.info);
7670         ret = notifier_to_errno(ret);
7671         if (ret)
7672                 return ret;
7673
7674         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
7675                                                    master);
7676         if (ret)
7677                 return ret;
7678
7679         ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7680                                             &changeupper_info.info);
7681         ret = notifier_to_errno(ret);
7682         if (ret)
7683                 goto rollback;
7684
7685         __netdev_update_upper_level(dev, NULL);
7686         __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7687
7688         __netdev_update_lower_level(upper_dev, priv);
7689         __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7690                                     priv);
7691
7692         return 0;
7693
7694 rollback:
7695         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7696
7697         return ret;
7698 }
7699
7700 /**
7701  * netdev_upper_dev_link - Add a link to the upper device
7702  * @dev: device
7703  * @upper_dev: new upper device
7704  * @extack: netlink extended ack
7705  *
7706  * Adds a link to device which is upper to this one. The caller must hold
7707  * the RTNL lock. On a failure a negative errno code is returned.
7708  * On success the reference counts are adjusted and the function
7709  * returns zero.
7710  */
7711 int netdev_upper_dev_link(struct net_device *dev,
7712                           struct net_device *upper_dev,
7713                           struct netlink_ext_ack *extack)
7714 {
7715         struct netdev_nested_priv priv = {
7716                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7717                 .data = NULL,
7718         };
7719
7720         return __netdev_upper_dev_link(dev, upper_dev, false,
7721                                        NULL, NULL, &priv, extack);
7722 }
7723 EXPORT_SYMBOL(netdev_upper_dev_link);
7724
7725 /**
7726  * netdev_master_upper_dev_link - Add a master link to the upper device
7727  * @dev: device
7728  * @upper_dev: new upper device
7729  * @upper_priv: upper device private
7730  * @upper_info: upper info to be passed down via notifier
7731  * @extack: netlink extended ack
7732  *
7733  * Adds a link to device which is upper to this one. In this case, only
7734  * one master upper device can be linked, although other non-master devices
7735  * might be linked as well. The caller must hold the RTNL lock.
7736  * On a failure a negative errno code is returned. On success the reference
7737  * counts are adjusted and the function returns zero.
7738  */
7739 int netdev_master_upper_dev_link(struct net_device *dev,
7740                                  struct net_device *upper_dev,
7741                                  void *upper_priv, void *upper_info,
7742                                  struct netlink_ext_ack *extack)
7743 {
7744         struct netdev_nested_priv priv = {
7745                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7746                 .data = NULL,
7747         };
7748
7749         return __netdev_upper_dev_link(dev, upper_dev, true,
7750                                        upper_priv, upper_info, &priv, extack);
7751 }
7752 EXPORT_SYMBOL(netdev_master_upper_dev_link);
7753
7754 static void __netdev_upper_dev_unlink(struct net_device *dev,
7755                                       struct net_device *upper_dev,
7756                                       struct netdev_nested_priv *priv)
7757 {
7758         struct netdev_notifier_changeupper_info changeupper_info = {
7759                 .info = {
7760                         .dev = dev,
7761                 },
7762                 .upper_dev = upper_dev,
7763                 .linking = false,
7764         };
7765
7766         ASSERT_RTNL();
7767
7768         changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
7769
7770         call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7771                                       &changeupper_info.info);
7772
7773         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7774
7775         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7776                                       &changeupper_info.info);
7777
7778         __netdev_update_upper_level(dev, NULL);
7779         __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7780
7781         __netdev_update_lower_level(upper_dev, priv);
7782         __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7783                                     priv);
7784 }
7785
7786 /**
7787  * netdev_upper_dev_unlink - Removes a link to upper device
7788  * @dev: device
7789  * @upper_dev: new upper device
7790  *
7791  * Removes a link to device which is upper to this one. The caller must hold
7792  * the RTNL lock.
7793  */
7794 void netdev_upper_dev_unlink(struct net_device *dev,
7795                              struct net_device *upper_dev)
7796 {
7797         struct netdev_nested_priv priv = {
7798                 .flags = NESTED_SYNC_TODO,
7799                 .data = NULL,
7800         };
7801
7802         __netdev_upper_dev_unlink(dev, upper_dev, &priv);
7803 }
7804 EXPORT_SYMBOL(netdev_upper_dev_unlink);
7805
7806 static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
7807                                       struct net_device *lower_dev,
7808                                       bool val)
7809 {
7810         struct netdev_adjacent *adj;
7811
7812         adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
7813         if (adj)
7814                 adj->ignore = val;
7815
7816         adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
7817         if (adj)
7818                 adj->ignore = val;
7819 }
7820
7821 static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
7822                                         struct net_device *lower_dev)
7823 {
7824         __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
7825 }
7826
7827 static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
7828                                        struct net_device *lower_dev)
7829 {
7830         __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
7831 }
7832
7833 int netdev_adjacent_change_prepare(struct net_device *old_dev,
7834                                    struct net_device *new_dev,
7835                                    struct net_device *dev,
7836                                    struct netlink_ext_ack *extack)
7837 {
7838         struct netdev_nested_priv priv = {
7839                 .flags = 0,
7840                 .data = NULL,
7841         };
7842         int err;
7843
7844         if (!new_dev)
7845                 return 0;
7846
7847         if (old_dev && new_dev != old_dev)
7848                 netdev_adjacent_dev_disable(dev, old_dev);
7849         err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
7850                                       extack);
7851         if (err) {
7852                 if (old_dev && new_dev != old_dev)
7853                         netdev_adjacent_dev_enable(dev, old_dev);
7854                 return err;
7855         }
7856
7857         return 0;
7858 }
7859 EXPORT_SYMBOL(netdev_adjacent_change_prepare);
7860
7861 void netdev_adjacent_change_commit(struct net_device *old_dev,
7862                                    struct net_device *new_dev,
7863                                    struct net_device *dev)
7864 {
7865         struct netdev_nested_priv priv = {
7866                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7867                 .data = NULL,
7868         };
7869
7870         if (!new_dev || !old_dev)
7871                 return;
7872
7873         if (new_dev == old_dev)
7874                 return;
7875
7876         netdev_adjacent_dev_enable(dev, old_dev);
7877         __netdev_upper_dev_unlink(old_dev, dev, &priv);
7878 }
7879 EXPORT_SYMBOL(netdev_adjacent_change_commit);
7880
7881 void netdev_adjacent_change_abort(struct net_device *old_dev,
7882                                   struct net_device *new_dev,
7883                                   struct net_device *dev)
7884 {
7885         struct netdev_nested_priv priv = {
7886                 .flags = 0,
7887                 .data = NULL,
7888         };
7889
7890         if (!new_dev)
7891                 return;
7892
7893         if (old_dev && new_dev != old_dev)
7894                 netdev_adjacent_dev_enable(dev, old_dev);
7895
7896         __netdev_upper_dev_unlink(new_dev, dev, &priv);
7897 }
7898 EXPORT_SYMBOL(netdev_adjacent_change_abort);
7899
7900 /**
7901  * netdev_bonding_info_change - Dispatch event about slave change
7902  * @dev: device
7903  * @bonding_info: info to dispatch
7904  *
7905  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
7906  * The caller must hold the RTNL lock.
7907  */
7908 void netdev_bonding_info_change(struct net_device *dev,
7909                                 struct netdev_bonding_info *bonding_info)
7910 {
7911         struct netdev_notifier_bonding_info info = {
7912                 .info.dev = dev,
7913         };
7914
7915         memcpy(&info.bonding_info, bonding_info,
7916                sizeof(struct netdev_bonding_info));
7917         call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
7918                                       &info.info);
7919 }
7920 EXPORT_SYMBOL(netdev_bonding_info_change);
7921
7922 static int netdev_offload_xstats_enable_l3(struct net_device *dev,
7923                                            struct netlink_ext_ack *extack)
7924 {
7925         struct netdev_notifier_offload_xstats_info info = {
7926                 .info.dev = dev,
7927                 .info.extack = extack,
7928                 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
7929         };
7930         int err;
7931         int rc;
7932
7933         dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3),
7934                                          GFP_KERNEL);
7935         if (!dev->offload_xstats_l3)
7936                 return -ENOMEM;
7937
7938         rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
7939                                                   NETDEV_OFFLOAD_XSTATS_DISABLE,
7940                                                   &info.info);
7941         err = notifier_to_errno(rc);
7942         if (err)
7943                 goto free_stats;
7944
7945         return 0;
7946
7947 free_stats:
7948         kfree(dev->offload_xstats_l3);
7949         dev->offload_xstats_l3 = NULL;
7950         return err;
7951 }
7952
7953 int netdev_offload_xstats_enable(struct net_device *dev,
7954                                  enum netdev_offload_xstats_type type,
7955                                  struct netlink_ext_ack *extack)
7956 {
7957         ASSERT_RTNL();
7958
7959         if (netdev_offload_xstats_enabled(dev, type))
7960                 return -EALREADY;
7961
7962         switch (type) {
7963         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
7964                 return netdev_offload_xstats_enable_l3(dev, extack);
7965         }
7966
7967         WARN_ON(1);
7968         return -EINVAL;
7969 }
7970 EXPORT_SYMBOL(netdev_offload_xstats_enable);
7971
7972 static void netdev_offload_xstats_disable_l3(struct net_device *dev)
7973 {
7974         struct netdev_notifier_offload_xstats_info info = {
7975                 .info.dev = dev,
7976                 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
7977         };
7978
7979         call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
7980                                       &info.info);
7981         kfree(dev->offload_xstats_l3);
7982         dev->offload_xstats_l3 = NULL;
7983 }
7984
7985 int netdev_offload_xstats_disable(struct net_device *dev,
7986                                   enum netdev_offload_xstats_type type)
7987 {
7988         ASSERT_RTNL();
7989
7990         if (!netdev_offload_xstats_enabled(dev, type))
7991                 return -EALREADY;
7992
7993         switch (type) {
7994         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
7995                 netdev_offload_xstats_disable_l3(dev);
7996                 return 0;
7997         }
7998
7999         WARN_ON(1);
8000         return -EINVAL;
8001 }
8002 EXPORT_SYMBOL(netdev_offload_xstats_disable);
8003
8004 static void netdev_offload_xstats_disable_all(struct net_device *dev)
8005 {
8006         netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
8007 }
8008
8009 static struct rtnl_hw_stats64 *
8010 netdev_offload_xstats_get_ptr(const struct net_device *dev,
8011                               enum netdev_offload_xstats_type type)
8012 {
8013         switch (type) {
8014         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8015                 return dev->offload_xstats_l3;
8016         }
8017
8018         WARN_ON(1);
8019         return NULL;
8020 }
8021
8022 bool netdev_offload_xstats_enabled(const struct net_device *dev,
8023                                    enum netdev_offload_xstats_type type)
8024 {
8025         ASSERT_RTNL();
8026
8027         return netdev_offload_xstats_get_ptr(dev, type);
8028 }
8029 EXPORT_SYMBOL(netdev_offload_xstats_enabled);
8030
8031 struct netdev_notifier_offload_xstats_ru {
8032         bool used;
8033 };
8034
8035 struct netdev_notifier_offload_xstats_rd {
8036         struct rtnl_hw_stats64 stats;
8037         bool used;
8038 };
8039
8040 static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
8041                                   const struct rtnl_hw_stats64 *src)
8042 {
8043         dest->rx_packets          += src->rx_packets;
8044         dest->tx_packets          += src->tx_packets;
8045         dest->rx_bytes            += src->rx_bytes;
8046         dest->tx_bytes            += src->tx_bytes;
8047         dest->rx_errors           += src->rx_errors;
8048         dest->tx_errors           += src->tx_errors;
8049         dest->rx_dropped          += src->rx_dropped;
8050         dest->tx_dropped          += src->tx_dropped;
8051         dest->multicast           += src->multicast;
8052 }
8053
8054 static int netdev_offload_xstats_get_used(struct net_device *dev,
8055                                           enum netdev_offload_xstats_type type,
8056                                           bool *p_used,
8057                                           struct netlink_ext_ack *extack)
8058 {
8059         struct netdev_notifier_offload_xstats_ru report_used = {};
8060         struct netdev_notifier_offload_xstats_info info = {
8061                 .info.dev = dev,
8062                 .info.extack = extack,
8063                 .type = type,
8064                 .report_used = &report_used,
8065         };
8066         int rc;
8067
8068         WARN_ON(!netdev_offload_xstats_enabled(dev, type));
8069         rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
8070                                            &info.info);
8071         *p_used = report_used.used;
8072         return notifier_to_errno(rc);
8073 }
8074
8075 static int netdev_offload_xstats_get_stats(struct net_device *dev,
8076                                            enum netdev_offload_xstats_type type,
8077                                            struct rtnl_hw_stats64 *p_stats,
8078                                            bool *p_used,
8079                                            struct netlink_ext_ack *extack)
8080 {
8081         struct netdev_notifier_offload_xstats_rd report_delta = {};
8082         struct netdev_notifier_offload_xstats_info info = {
8083                 .info.dev = dev,
8084                 .info.extack = extack,
8085                 .type = type,
8086                 .report_delta = &report_delta,
8087         };
8088         struct rtnl_hw_stats64 *stats;
8089         int rc;
8090
8091         stats = netdev_offload_xstats_get_ptr(dev, type);
8092         if (WARN_ON(!stats))
8093                 return -EINVAL;
8094
8095         rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
8096                                            &info.info);
8097
8098         /* Cache whatever we got, even if there was an error, otherwise the
8099          * successful stats retrievals would get lost.
8100          */
8101         netdev_hw_stats64_add(stats, &report_delta.stats);
8102
8103         if (p_stats)
8104                 *p_stats = *stats;
8105         *p_used = report_delta.used;
8106
8107         return notifier_to_errno(rc);
8108 }
8109
8110 int netdev_offload_xstats_get(struct net_device *dev,
8111                               enum netdev_offload_xstats_type type,
8112                               struct rtnl_hw_stats64 *p_stats, bool *p_used,
8113                               struct netlink_ext_ack *extack)
8114 {
8115         ASSERT_RTNL();
8116
8117         if (p_stats)
8118                 return netdev_offload_xstats_get_stats(dev, type, p_stats,
8119                                                        p_used, extack);
8120         else
8121                 return netdev_offload_xstats_get_used(dev, type, p_used,
8122                                                       extack);
8123 }
8124 EXPORT_SYMBOL(netdev_offload_xstats_get);
8125
8126 void
8127 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
8128                                    const struct rtnl_hw_stats64 *stats)
8129 {
8130         report_delta->used = true;
8131         netdev_hw_stats64_add(&report_delta->stats, stats);
8132 }
8133 EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
8134
8135 void
8136 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
8137 {
8138         report_used->used = true;
8139 }
8140 EXPORT_SYMBOL(netdev_offload_xstats_report_used);
8141
8142 void netdev_offload_xstats_push_delta(struct net_device *dev,
8143                                       enum netdev_offload_xstats_type type,
8144                                       const struct rtnl_hw_stats64 *p_stats)
8145 {
8146         struct rtnl_hw_stats64 *stats;
8147
8148         ASSERT_RTNL();
8149
8150         stats = netdev_offload_xstats_get_ptr(dev, type);
8151         if (WARN_ON(!stats))
8152                 return;
8153
8154         netdev_hw_stats64_add(stats, p_stats);
8155 }
8156 EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
8157
8158 /**
8159  * netdev_get_xmit_slave - Get the xmit slave of master device
8160  * @dev: device
8161  * @skb: The packet
8162  * @all_slaves: assume all the slaves are active
8163  *
8164  * The reference counters are not incremented so the caller must be
8165  * careful with locks. The caller must hold RCU lock.
8166  * %NULL is returned if no slave is found.
8167  */
8168
8169 struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8170                                          struct sk_buff *skb,
8171                                          bool all_slaves)
8172 {
8173         const struct net_device_ops *ops = dev->netdev_ops;
8174
8175         if (!ops->ndo_get_xmit_slave)
8176                 return NULL;
8177         return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8178 }
8179 EXPORT_SYMBOL(netdev_get_xmit_slave);
8180
8181 static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8182                                                   struct sock *sk)
8183 {
8184         const struct net_device_ops *ops = dev->netdev_ops;
8185
8186         if (!ops->ndo_sk_get_lower_dev)
8187                 return NULL;
8188         return ops->ndo_sk_get_lower_dev(dev, sk);
8189 }
8190
8191 /**
8192  * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8193  * @dev: device
8194  * @sk: the socket
8195  *
8196  * %NULL is returned if no lower device is found.
8197  */
8198
8199 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8200                                             struct sock *sk)
8201 {
8202         struct net_device *lower;
8203
8204         lower = netdev_sk_get_lower_dev(dev, sk);
8205         while (lower) {
8206                 dev = lower;
8207                 lower = netdev_sk_get_lower_dev(dev, sk);
8208         }
8209
8210         return dev;
8211 }
8212 EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8213
8214 static void netdev_adjacent_add_links(struct net_device *dev)
8215 {
8216         struct netdev_adjacent *iter;
8217
8218         struct net *net = dev_net(dev);
8219
8220         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8221                 if (!net_eq(net, dev_net(iter->dev)))
8222                         continue;
8223                 netdev_adjacent_sysfs_add(iter->dev, dev,
8224                                           &iter->dev->adj_list.lower);
8225                 netdev_adjacent_sysfs_add(dev, iter->dev,
8226                                           &dev->adj_list.upper);
8227         }
8228
8229         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8230                 if (!net_eq(net, dev_net(iter->dev)))
8231                         continue;
8232                 netdev_adjacent_sysfs_add(iter->dev, dev,
8233                                           &iter->dev->adj_list.upper);
8234                 netdev_adjacent_sysfs_add(dev, iter->dev,
8235                                           &dev->adj_list.lower);
8236         }
8237 }
8238
8239 static void netdev_adjacent_del_links(struct net_device *dev)
8240 {
8241         struct netdev_adjacent *iter;
8242
8243         struct net *net = dev_net(dev);
8244
8245         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8246                 if (!net_eq(net, dev_net(iter->dev)))
8247                         continue;
8248                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8249                                           &iter->dev->adj_list.lower);
8250                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8251                                           &dev->adj_list.upper);
8252         }
8253
8254         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8255                 if (!net_eq(net, dev_net(iter->dev)))
8256                         continue;
8257                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8258                                           &iter->dev->adj_list.upper);
8259                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8260                                           &dev->adj_list.lower);
8261         }
8262 }
8263
8264 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
8265 {
8266         struct netdev_adjacent *iter;
8267
8268         struct net *net = dev_net(dev);
8269
8270         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8271                 if (!net_eq(net, dev_net(iter->dev)))
8272                         continue;
8273                 netdev_adjacent_sysfs_del(iter->dev, oldname,
8274                                           &iter->dev->adj_list.lower);
8275                 netdev_adjacent_sysfs_add(iter->dev, dev,
8276                                           &iter->dev->adj_list.lower);
8277         }
8278
8279         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8280                 if (!net_eq(net, dev_net(iter->dev)))
8281                         continue;
8282                 netdev_adjacent_sysfs_del(iter->dev, oldname,
8283                                           &iter->dev->adj_list.upper);
8284                 netdev_adjacent_sysfs_add(iter->dev, dev,
8285                                           &iter->dev->adj_list.upper);
8286         }
8287 }
8288
8289 void *netdev_lower_dev_get_private(struct net_device *dev,
8290                                    struct net_device *lower_dev)
8291 {
8292         struct netdev_adjacent *lower;
8293
8294         if (!lower_dev)
8295                 return NULL;
8296         lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
8297         if (!lower)
8298                 return NULL;
8299
8300         return lower->private;
8301 }
8302 EXPORT_SYMBOL(netdev_lower_dev_get_private);
8303
8304
8305 /**
8306  * netdev_lower_state_changed - Dispatch event about lower device state change
8307  * @lower_dev: device
8308  * @lower_state_info: state to dispatch
8309  *
8310  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8311  * The caller must hold the RTNL lock.
8312  */
8313 void netdev_lower_state_changed(struct net_device *lower_dev,
8314                                 void *lower_state_info)
8315 {
8316         struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8317                 .info.dev = lower_dev,
8318         };
8319
8320         ASSERT_RTNL();
8321         changelowerstate_info.lower_state_info = lower_state_info;
8322         call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
8323                                       &changelowerstate_info.info);
8324 }
8325 EXPORT_SYMBOL(netdev_lower_state_changed);
8326
8327 static void dev_change_rx_flags(struct net_device *dev, int flags)
8328 {
8329         const struct net_device_ops *ops = dev->netdev_ops;
8330
8331         if (ops->ndo_change_rx_flags)
8332                 ops->ndo_change_rx_flags(dev, flags);
8333 }
8334
8335 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
8336 {
8337         unsigned int old_flags = dev->flags;
8338         kuid_t uid;
8339         kgid_t gid;
8340
8341         ASSERT_RTNL();
8342
8343         dev->flags |= IFF_PROMISC;
8344         dev->promiscuity += inc;
8345         if (dev->promiscuity == 0) {
8346                 /*
8347                  * Avoid overflow.
8348                  * If inc causes overflow, untouch promisc and return error.
8349                  */
8350                 if (inc < 0)
8351                         dev->flags &= ~IFF_PROMISC;
8352                 else {
8353                         dev->promiscuity -= inc;
8354                         netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
8355                         return -EOVERFLOW;
8356                 }
8357         }
8358         if (dev->flags != old_flags) {
8359                 netdev_info(dev, "%s promiscuous mode\n",
8360                             dev->flags & IFF_PROMISC ? "entered" : "left");
8361                 if (audit_enabled) {
8362                         current_uid_gid(&uid, &gid);
8363                         audit_log(audit_context(), GFP_ATOMIC,
8364                                   AUDIT_ANOM_PROMISCUOUS,
8365                                   "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8366                                   dev->name, (dev->flags & IFF_PROMISC),
8367                                   (old_flags & IFF_PROMISC),
8368                                   from_kuid(&init_user_ns, audit_get_loginuid(current)),
8369                                   from_kuid(&init_user_ns, uid),
8370                                   from_kgid(&init_user_ns, gid),
8371                                   audit_get_sessionid(current));
8372                 }
8373
8374                 dev_change_rx_flags(dev, IFF_PROMISC);
8375         }
8376         if (notify)
8377                 __dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
8378         return 0;
8379 }
8380
8381 /**
8382  *      dev_set_promiscuity     - update promiscuity count on a device
8383  *      @dev: device
8384  *      @inc: modifier
8385  *
8386  *      Add or remove promiscuity from a device. While the count in the device
8387  *      remains above zero the interface remains promiscuous. Once it hits zero
8388  *      the device reverts back to normal filtering operation. A negative inc
8389  *      value is used to drop promiscuity on the device.
8390  *      Return 0 if successful or a negative errno code on error.
8391  */
8392 int dev_set_promiscuity(struct net_device *dev, int inc)
8393 {
8394         unsigned int old_flags = dev->flags;
8395         int err;
8396
8397         err = __dev_set_promiscuity(dev, inc, true);
8398         if (err < 0)
8399                 return err;
8400         if (dev->flags != old_flags)
8401                 dev_set_rx_mode(dev);
8402         return err;
8403 }
8404 EXPORT_SYMBOL(dev_set_promiscuity);
8405
8406 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
8407 {
8408         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
8409
8410         ASSERT_RTNL();
8411
8412         dev->flags |= IFF_ALLMULTI;
8413         dev->allmulti += inc;
8414         if (dev->allmulti == 0) {
8415                 /*
8416                  * Avoid overflow.
8417                  * If inc causes overflow, untouch allmulti and return error.
8418                  */
8419                 if (inc < 0)
8420                         dev->flags &= ~IFF_ALLMULTI;
8421                 else {
8422                         dev->allmulti -= inc;
8423                         netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
8424                         return -EOVERFLOW;
8425                 }
8426         }
8427         if (dev->flags ^ old_flags) {
8428                 netdev_info(dev, "%s allmulticast mode\n",
8429                             dev->flags & IFF_ALLMULTI ? "entered" : "left");
8430                 dev_change_rx_flags(dev, IFF_ALLMULTI);
8431                 dev_set_rx_mode(dev);
8432                 if (notify)
8433                         __dev_notify_flags(dev, old_flags,
8434                                            dev->gflags ^ old_gflags, 0, NULL);
8435         }
8436         return 0;
8437 }
8438
8439 /**
8440  *      dev_set_allmulti        - update allmulti count on a device
8441  *      @dev: device
8442  *      @inc: modifier
8443  *
8444  *      Add or remove reception of all multicast frames to a device. While the
8445  *      count in the device remains above zero the interface remains listening
8446  *      to all interfaces. Once it hits zero the device reverts back to normal
8447  *      filtering operation. A negative @inc value is used to drop the counter
8448  *      when releasing a resource needing all multicasts.
8449  *      Return 0 if successful or a negative errno code on error.
8450  */
8451
8452 int dev_set_allmulti(struct net_device *dev, int inc)
8453 {
8454         return __dev_set_allmulti(dev, inc, true);
8455 }
8456 EXPORT_SYMBOL(dev_set_allmulti);
8457
8458 /*
8459  *      Upload unicast and multicast address lists to device and
8460  *      configure RX filtering. When the device doesn't support unicast
8461  *      filtering it is put in promiscuous mode while unicast addresses
8462  *      are present.
8463  */
8464 void __dev_set_rx_mode(struct net_device *dev)
8465 {
8466         const struct net_device_ops *ops = dev->netdev_ops;
8467
8468         /* dev_open will call this function so the list will stay sane. */
8469         if (!(dev->flags&IFF_UP))
8470                 return;
8471
8472         if (!netif_device_present(dev))
8473                 return;
8474
8475         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
8476                 /* Unicast addresses changes may only happen under the rtnl,
8477                  * therefore calling __dev_set_promiscuity here is safe.
8478                  */
8479                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
8480                         __dev_set_promiscuity(dev, 1, false);
8481                         dev->uc_promisc = true;
8482                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
8483                         __dev_set_promiscuity(dev, -1, false);
8484                         dev->uc_promisc = false;
8485                 }
8486         }
8487
8488         if (ops->ndo_set_rx_mode)
8489                 ops->ndo_set_rx_mode(dev);
8490 }
8491
8492 void dev_set_rx_mode(struct net_device *dev)
8493 {
8494         netif_addr_lock_bh(dev);
8495         __dev_set_rx_mode(dev);
8496         netif_addr_unlock_bh(dev);
8497 }
8498
8499 /**
8500  *      dev_get_flags - get flags reported to userspace
8501  *      @dev: device
8502  *
8503  *      Get the combination of flag bits exported through APIs to userspace.
8504  */
8505 unsigned int dev_get_flags(const struct net_device *dev)
8506 {
8507         unsigned int flags;
8508
8509         flags = (dev->flags & ~(IFF_PROMISC |
8510                                 IFF_ALLMULTI |
8511                                 IFF_RUNNING |
8512                                 IFF_LOWER_UP |
8513                                 IFF_DORMANT)) |
8514                 (dev->gflags & (IFF_PROMISC |
8515                                 IFF_ALLMULTI));
8516
8517         if (netif_running(dev)) {
8518                 if (netif_oper_up(dev))
8519                         flags |= IFF_RUNNING;
8520                 if (netif_carrier_ok(dev))
8521                         flags |= IFF_LOWER_UP;
8522                 if (netif_dormant(dev))
8523                         flags |= IFF_DORMANT;
8524         }
8525
8526         return flags;
8527 }
8528 EXPORT_SYMBOL(dev_get_flags);
8529
8530 int __dev_change_flags(struct net_device *dev, unsigned int flags,
8531                        struct netlink_ext_ack *extack)
8532 {
8533         unsigned int old_flags = dev->flags;
8534         int ret;
8535
8536         ASSERT_RTNL();
8537
8538         /*
8539          *      Set the flags on our device.
8540          */
8541
8542         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8543                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8544                                IFF_AUTOMEDIA)) |
8545                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8546                                     IFF_ALLMULTI));
8547
8548         /*
8549          *      Load in the correct multicast list now the flags have changed.
8550          */
8551
8552         if ((old_flags ^ flags) & IFF_MULTICAST)
8553                 dev_change_rx_flags(dev, IFF_MULTICAST);
8554
8555         dev_set_rx_mode(dev);
8556
8557         /*
8558          *      Have we downed the interface. We handle IFF_UP ourselves
8559          *      according to user attempts to set it, rather than blindly
8560          *      setting it.
8561          */
8562
8563         ret = 0;
8564         if ((old_flags ^ flags) & IFF_UP) {
8565                 if (old_flags & IFF_UP)
8566                         __dev_close(dev);
8567                 else
8568                         ret = __dev_open(dev, extack);
8569         }
8570
8571         if ((flags ^ dev->gflags) & IFF_PROMISC) {
8572                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
8573                 unsigned int old_flags = dev->flags;
8574
8575                 dev->gflags ^= IFF_PROMISC;
8576
8577                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
8578                         if (dev->flags != old_flags)
8579                                 dev_set_rx_mode(dev);
8580         }
8581
8582         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
8583          * is important. Some (broken) drivers set IFF_PROMISC, when
8584          * IFF_ALLMULTI is requested not asking us and not reporting.
8585          */
8586         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
8587                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
8588
8589                 dev->gflags ^= IFF_ALLMULTI;
8590                 __dev_set_allmulti(dev, inc, false);
8591         }
8592
8593         return ret;
8594 }
8595
8596 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
8597                         unsigned int gchanges, u32 portid,
8598                         const struct nlmsghdr *nlh)
8599 {
8600         unsigned int changes = dev->flags ^ old_flags;
8601
8602         if (gchanges)
8603                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
8604
8605         if (changes & IFF_UP) {
8606                 if (dev->flags & IFF_UP)
8607                         call_netdevice_notifiers(NETDEV_UP, dev);
8608                 else
8609                         call_netdevice_notifiers(NETDEV_DOWN, dev);
8610         }
8611
8612         if (dev->flags & IFF_UP &&
8613             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
8614                 struct netdev_notifier_change_info change_info = {
8615                         .info = {
8616                                 .dev = dev,
8617                         },
8618                         .flags_changed = changes,
8619                 };
8620
8621                 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
8622         }
8623 }
8624
8625 /**
8626  *      dev_change_flags - change device settings
8627  *      @dev: device
8628  *      @flags: device state flags
8629  *      @extack: netlink extended ack
8630  *
8631  *      Change settings on device based state flags. The flags are
8632  *      in the userspace exported format.
8633  */
8634 int dev_change_flags(struct net_device *dev, unsigned int flags,
8635                      struct netlink_ext_ack *extack)
8636 {
8637         int ret;
8638         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
8639
8640         ret = __dev_change_flags(dev, flags, extack);
8641         if (ret < 0)
8642                 return ret;
8643
8644         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
8645         __dev_notify_flags(dev, old_flags, changes, 0, NULL);
8646         return ret;
8647 }
8648 EXPORT_SYMBOL(dev_change_flags);
8649
8650 int __dev_set_mtu(struct net_device *dev, int new_mtu)
8651 {
8652         const struct net_device_ops *ops = dev->netdev_ops;
8653
8654         if (ops->ndo_change_mtu)
8655                 return ops->ndo_change_mtu(dev, new_mtu);
8656
8657         /* Pairs with all the lockless reads of dev->mtu in the stack */
8658         WRITE_ONCE(dev->mtu, new_mtu);
8659         return 0;
8660 }
8661 EXPORT_SYMBOL(__dev_set_mtu);
8662
8663 int dev_validate_mtu(struct net_device *dev, int new_mtu,
8664                      struct netlink_ext_ack *extack)
8665 {
8666         /* MTU must be positive, and in range */
8667         if (new_mtu < 0 || new_mtu < dev->min_mtu) {
8668                 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
8669                 return -EINVAL;
8670         }
8671
8672         if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
8673                 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
8674                 return -EINVAL;
8675         }
8676         return 0;
8677 }
8678
8679 /**
8680  *      dev_set_mtu_ext - Change maximum transfer unit
8681  *      @dev: device
8682  *      @new_mtu: new transfer unit
8683  *      @extack: netlink extended ack
8684  *
8685  *      Change the maximum transfer size of the network device.
8686  */
8687 int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
8688                     struct netlink_ext_ack *extack)
8689 {
8690         int err, orig_mtu;
8691
8692         if (new_mtu == dev->mtu)
8693                 return 0;
8694
8695         err = dev_validate_mtu(dev, new_mtu, extack);
8696         if (err)
8697                 return err;
8698
8699         if (!netif_device_present(dev))
8700                 return -ENODEV;
8701
8702         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
8703         err = notifier_to_errno(err);
8704         if (err)
8705                 return err;
8706
8707         orig_mtu = dev->mtu;
8708         err = __dev_set_mtu(dev, new_mtu);
8709
8710         if (!err) {
8711                 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8712                                                    orig_mtu);
8713                 err = notifier_to_errno(err);
8714                 if (err) {
8715                         /* setting mtu back and notifying everyone again,
8716                          * so that they have a chance to revert changes.
8717                          */
8718                         __dev_set_mtu(dev, orig_mtu);
8719                         call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8720                                                      new_mtu);
8721                 }
8722         }
8723         return err;
8724 }
8725
8726 int dev_set_mtu(struct net_device *dev, int new_mtu)
8727 {
8728         struct netlink_ext_ack extack;
8729         int err;
8730
8731         memset(&extack, 0, sizeof(extack));
8732         err = dev_set_mtu_ext(dev, new_mtu, &extack);
8733         if (err && extack._msg)
8734                 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
8735         return err;
8736 }
8737 EXPORT_SYMBOL(dev_set_mtu);
8738
8739 /**
8740  *      dev_change_tx_queue_len - Change TX queue length of a netdevice
8741  *      @dev: device
8742  *      @new_len: new tx queue length
8743  */
8744 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
8745 {
8746         unsigned int orig_len = dev->tx_queue_len;
8747         int res;
8748
8749         if (new_len != (unsigned int)new_len)
8750                 return -ERANGE;
8751
8752         if (new_len != orig_len) {
8753                 dev->tx_queue_len = new_len;
8754                 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
8755                 res = notifier_to_errno(res);
8756                 if (res)
8757                         goto err_rollback;
8758                 res = dev_qdisc_change_tx_queue_len(dev);
8759                 if (res)
8760                         goto err_rollback;
8761         }
8762
8763         return 0;
8764
8765 err_rollback:
8766         netdev_err(dev, "refused to change device tx_queue_len\n");
8767         dev->tx_queue_len = orig_len;
8768         return res;
8769 }
8770
8771 /**
8772  *      dev_set_group - Change group this device belongs to
8773  *      @dev: device
8774  *      @new_group: group this device should belong to
8775  */
8776 void dev_set_group(struct net_device *dev, int new_group)
8777 {
8778         dev->group = new_group;
8779 }
8780
8781 /**
8782  *      dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
8783  *      @dev: device
8784  *      @addr: new address
8785  *      @extack: netlink extended ack
8786  */
8787 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
8788                               struct netlink_ext_ack *extack)
8789 {
8790         struct netdev_notifier_pre_changeaddr_info info = {
8791                 .info.dev = dev,
8792                 .info.extack = extack,
8793                 .dev_addr = addr,
8794         };
8795         int rc;
8796
8797         rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
8798         return notifier_to_errno(rc);
8799 }
8800 EXPORT_SYMBOL(dev_pre_changeaddr_notify);
8801
8802 /**
8803  *      dev_set_mac_address - Change Media Access Control Address
8804  *      @dev: device
8805  *      @sa: new address
8806  *      @extack: netlink extended ack
8807  *
8808  *      Change the hardware (MAC) address of the device
8809  */
8810 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
8811                         struct netlink_ext_ack *extack)
8812 {
8813         const struct net_device_ops *ops = dev->netdev_ops;
8814         int err;
8815
8816         if (!ops->ndo_set_mac_address)
8817                 return -EOPNOTSUPP;
8818         if (sa->sa_family != dev->type)
8819                 return -EINVAL;
8820         if (!netif_device_present(dev))
8821                 return -ENODEV;
8822         err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
8823         if (err)
8824                 return err;
8825         err = ops->ndo_set_mac_address(dev, sa);
8826         if (err)
8827                 return err;
8828         dev->addr_assign_type = NET_ADDR_SET;
8829         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
8830         add_device_randomness(dev->dev_addr, dev->addr_len);
8831         return 0;
8832 }
8833 EXPORT_SYMBOL(dev_set_mac_address);
8834
8835 static DECLARE_RWSEM(dev_addr_sem);
8836
8837 int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
8838                              struct netlink_ext_ack *extack)
8839 {
8840         int ret;
8841
8842         down_write(&dev_addr_sem);
8843         ret = dev_set_mac_address(dev, sa, extack);
8844         up_write(&dev_addr_sem);
8845         return ret;
8846 }
8847 EXPORT_SYMBOL(dev_set_mac_address_user);
8848
8849 int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
8850 {
8851         size_t size = sizeof(sa->sa_data_min);
8852         struct net_device *dev;
8853         int ret = 0;
8854
8855         down_read(&dev_addr_sem);
8856         rcu_read_lock();
8857
8858         dev = dev_get_by_name_rcu(net, dev_name);
8859         if (!dev) {
8860                 ret = -ENODEV;
8861                 goto unlock;
8862         }
8863         if (!dev->addr_len)
8864                 memset(sa->sa_data, 0, size);
8865         else
8866                 memcpy(sa->sa_data, dev->dev_addr,
8867                        min_t(size_t, size, dev->addr_len));
8868         sa->sa_family = dev->type;
8869
8870 unlock:
8871         rcu_read_unlock();
8872         up_read(&dev_addr_sem);
8873         return ret;
8874 }
8875 EXPORT_SYMBOL(dev_get_mac_address);
8876
8877 /**
8878  *      dev_change_carrier - Change device carrier
8879  *      @dev: device
8880  *      @new_carrier: new value
8881  *
8882  *      Change device carrier
8883  */
8884 int dev_change_carrier(struct net_device *dev, bool new_carrier)
8885 {
8886         const struct net_device_ops *ops = dev->netdev_ops;
8887
8888         if (!ops->ndo_change_carrier)
8889                 return -EOPNOTSUPP;
8890         if (!netif_device_present(dev))
8891                 return -ENODEV;
8892         return ops->ndo_change_carrier(dev, new_carrier);
8893 }
8894
8895 /**
8896  *      dev_get_phys_port_id - Get device physical port ID
8897  *      @dev: device
8898  *      @ppid: port ID
8899  *
8900  *      Get device physical port ID
8901  */
8902 int dev_get_phys_port_id(struct net_device *dev,
8903                          struct netdev_phys_item_id *ppid)
8904 {
8905         const struct net_device_ops *ops = dev->netdev_ops;
8906
8907         if (!ops->ndo_get_phys_port_id)
8908                 return -EOPNOTSUPP;
8909         return ops->ndo_get_phys_port_id(dev, ppid);
8910 }
8911
8912 /**
8913  *      dev_get_phys_port_name - Get device physical port name
8914  *      @dev: device
8915  *      @name: port name
8916  *      @len: limit of bytes to copy to name
8917  *
8918  *      Get device physical port name
8919  */
8920 int dev_get_phys_port_name(struct net_device *dev,
8921                            char *name, size_t len)
8922 {
8923         const struct net_device_ops *ops = dev->netdev_ops;
8924         int err;
8925
8926         if (ops->ndo_get_phys_port_name) {
8927                 err = ops->ndo_get_phys_port_name(dev, name, len);
8928                 if (err != -EOPNOTSUPP)
8929                         return err;
8930         }
8931         return devlink_compat_phys_port_name_get(dev, name, len);
8932 }
8933
8934 /**
8935  *      dev_get_port_parent_id - Get the device's port parent identifier
8936  *      @dev: network device
8937  *      @ppid: pointer to a storage for the port's parent identifier
8938  *      @recurse: allow/disallow recursion to lower devices
8939  *
8940  *      Get the devices's port parent identifier
8941  */
8942 int dev_get_port_parent_id(struct net_device *dev,
8943                            struct netdev_phys_item_id *ppid,
8944                            bool recurse)
8945 {
8946         const struct net_device_ops *ops = dev->netdev_ops;
8947         struct netdev_phys_item_id first = { };
8948         struct net_device *lower_dev;
8949         struct list_head *iter;
8950         int err;
8951
8952         if (ops->ndo_get_port_parent_id) {
8953                 err = ops->ndo_get_port_parent_id(dev, ppid);
8954                 if (err != -EOPNOTSUPP)
8955                         return err;
8956         }
8957
8958         err = devlink_compat_switch_id_get(dev, ppid);
8959         if (!recurse || err != -EOPNOTSUPP)
8960                 return err;
8961
8962         netdev_for_each_lower_dev(dev, lower_dev, iter) {
8963                 err = dev_get_port_parent_id(lower_dev, ppid, true);
8964                 if (err)
8965                         break;
8966                 if (!first.id_len)
8967                         first = *ppid;
8968                 else if (memcmp(&first, ppid, sizeof(*ppid)))
8969                         return -EOPNOTSUPP;
8970         }
8971
8972         return err;
8973 }
8974 EXPORT_SYMBOL(dev_get_port_parent_id);
8975
8976 /**
8977  *      netdev_port_same_parent_id - Indicate if two network devices have
8978  *      the same port parent identifier
8979  *      @a: first network device
8980  *      @b: second network device
8981  */
8982 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
8983 {
8984         struct netdev_phys_item_id a_id = { };
8985         struct netdev_phys_item_id b_id = { };
8986
8987         if (dev_get_port_parent_id(a, &a_id, true) ||
8988             dev_get_port_parent_id(b, &b_id, true))
8989                 return false;
8990
8991         return netdev_phys_item_id_same(&a_id, &b_id);
8992 }
8993 EXPORT_SYMBOL(netdev_port_same_parent_id);
8994
8995 /**
8996  *      dev_change_proto_down - set carrier according to proto_down.
8997  *
8998  *      @dev: device
8999  *      @proto_down: new value
9000  */
9001 int dev_change_proto_down(struct net_device *dev, bool proto_down)
9002 {
9003         if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN))
9004                 return -EOPNOTSUPP;
9005         if (!netif_device_present(dev))
9006                 return -ENODEV;
9007         if (proto_down)
9008                 netif_carrier_off(dev);
9009         else
9010                 netif_carrier_on(dev);
9011         dev->proto_down = proto_down;
9012         return 0;
9013 }
9014
9015 /**
9016  *      dev_change_proto_down_reason - proto down reason
9017  *
9018  *      @dev: device
9019  *      @mask: proto down mask
9020  *      @value: proto down value
9021  */
9022 void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9023                                   u32 value)
9024 {
9025         int b;
9026
9027         if (!mask) {
9028                 dev->proto_down_reason = value;
9029         } else {
9030                 for_each_set_bit(b, &mask, 32) {
9031                         if (value & (1 << b))
9032                                 dev->proto_down_reason |= BIT(b);
9033                         else
9034                                 dev->proto_down_reason &= ~BIT(b);
9035                 }
9036         }
9037 }
9038
9039 struct bpf_xdp_link {
9040         struct bpf_link link;
9041         struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9042         int flags;
9043 };
9044
9045 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
9046 {
9047         if (flags & XDP_FLAGS_HW_MODE)
9048                 return XDP_MODE_HW;
9049         if (flags & XDP_FLAGS_DRV_MODE)
9050                 return XDP_MODE_DRV;
9051         if (flags & XDP_FLAGS_SKB_MODE)
9052                 return XDP_MODE_SKB;
9053         return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
9054 }
9055
9056 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9057 {
9058         switch (mode) {
9059         case XDP_MODE_SKB:
9060                 return generic_xdp_install;
9061         case XDP_MODE_DRV:
9062         case XDP_MODE_HW:
9063                 return dev->netdev_ops->ndo_bpf;
9064         default:
9065                 return NULL;
9066         }
9067 }
9068
9069 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9070                                          enum bpf_xdp_mode mode)
9071 {
9072         return dev->xdp_state[mode].link;
9073 }
9074
9075 static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9076                                      enum bpf_xdp_mode mode)
9077 {
9078         struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9079
9080         if (link)
9081                 return link->link.prog;
9082         return dev->xdp_state[mode].prog;
9083 }
9084
9085 u8 dev_xdp_prog_count(struct net_device *dev)
9086 {
9087         u8 count = 0;
9088         int i;
9089
9090         for (i = 0; i < __MAX_XDP_MODE; i++)
9091                 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9092                         count++;
9093         return count;
9094 }
9095 EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
9096
9097 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9098 {
9099         struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9100
9101         return prog ? prog->aux->id : 0;
9102 }
9103
9104 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9105                              struct bpf_xdp_link *link)
9106 {
9107         dev->xdp_state[mode].link = link;
9108         dev->xdp_state[mode].prog = NULL;
9109 }
9110
9111 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9112                              struct bpf_prog *prog)
9113 {
9114         dev->xdp_state[mode].link = NULL;
9115         dev->xdp_state[mode].prog = prog;
9116 }
9117
9118 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9119                            bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9120                            u32 flags, struct bpf_prog *prog)
9121 {
9122         struct netdev_bpf xdp;
9123         int err;
9124
9125         memset(&xdp, 0, sizeof(xdp));
9126         xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9127         xdp.extack = extack;
9128         xdp.flags = flags;
9129         xdp.prog = prog;
9130
9131         /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9132          * "moved" into driver), so they don't increment it on their own, but
9133          * they do decrement refcnt when program is detached or replaced.
9134          * Given net_device also owns link/prog, we need to bump refcnt here
9135          * to prevent drivers from underflowing it.
9136          */
9137         if (prog)
9138                 bpf_prog_inc(prog);
9139         err = bpf_op(dev, &xdp);
9140         if (err) {
9141                 if (prog)
9142                         bpf_prog_put(prog);
9143                 return err;
9144         }
9145
9146         if (mode != XDP_MODE_HW)
9147                 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
9148
9149         return 0;
9150 }
9151
9152 static void dev_xdp_uninstall(struct net_device *dev)
9153 {
9154         struct bpf_xdp_link *link;
9155         struct bpf_prog *prog;
9156         enum bpf_xdp_mode mode;
9157         bpf_op_t bpf_op;
9158
9159         ASSERT_RTNL();
9160
9161         for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9162                 prog = dev_xdp_prog(dev, mode);
9163                 if (!prog)
9164                         continue;
9165
9166                 bpf_op = dev_xdp_bpf_op(dev, mode);
9167                 if (!bpf_op)
9168                         continue;
9169
9170                 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9171
9172                 /* auto-detach link from net device */
9173                 link = dev_xdp_link(dev, mode);
9174                 if (link)
9175                         link->dev = NULL;
9176                 else
9177                         bpf_prog_put(prog);
9178
9179                 dev_xdp_set_link(dev, mode, NULL);
9180         }
9181 }
9182
9183 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
9184                           struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9185                           struct bpf_prog *old_prog, u32 flags)
9186 {
9187         unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
9188         struct bpf_prog *cur_prog;
9189         struct net_device *upper;
9190         struct list_head *iter;
9191         enum bpf_xdp_mode mode;
9192         bpf_op_t bpf_op;
9193         int err;
9194
9195         ASSERT_RTNL();
9196
9197         /* either link or prog attachment, never both */
9198         if (link && (new_prog || old_prog))
9199                 return -EINVAL;
9200         /* link supports only XDP mode flags */
9201         if (link && (flags & ~XDP_FLAGS_MODES)) {
9202                 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9203                 return -EINVAL;
9204         }
9205         /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9206         if (num_modes > 1) {
9207                 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9208                 return -EINVAL;
9209         }
9210         /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9211         if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9212                 NL_SET_ERR_MSG(extack,
9213                                "More than one program loaded, unset mode is ambiguous");
9214                 return -EINVAL;
9215         }
9216         /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9217         if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9218                 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9219                 return -EINVAL;
9220         }
9221
9222         mode = dev_xdp_mode(dev, flags);
9223         /* can't replace attached link */
9224         if (dev_xdp_link(dev, mode)) {
9225                 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9226                 return -EBUSY;
9227         }
9228
9229         /* don't allow if an upper device already has a program */
9230         netdev_for_each_upper_dev_rcu(dev, upper, iter) {
9231                 if (dev_xdp_prog_count(upper) > 0) {
9232                         NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
9233                         return -EEXIST;
9234                 }
9235         }
9236
9237         cur_prog = dev_xdp_prog(dev, mode);
9238         /* can't replace attached prog with link */
9239         if (link && cur_prog) {
9240                 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9241                 return -EBUSY;
9242         }
9243         if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9244                 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9245                 return -EEXIST;
9246         }
9247
9248         /* put effective new program into new_prog */
9249         if (link)
9250                 new_prog = link->link.prog;
9251
9252         if (new_prog) {
9253                 bool offload = mode == XDP_MODE_HW;
9254                 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9255                                                ? XDP_MODE_DRV : XDP_MODE_SKB;
9256
9257                 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9258                         NL_SET_ERR_MSG(extack, "XDP program already attached");
9259                         return -EBUSY;
9260                 }
9261                 if (!offload && dev_xdp_prog(dev, other_mode)) {
9262                         NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9263                         return -EEXIST;
9264                 }
9265                 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) {
9266                         NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
9267                         return -EINVAL;
9268                 }
9269                 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) {
9270                         NL_SET_ERR_MSG(extack, "Program bound to different device");
9271                         return -EINVAL;
9272                 }
9273                 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9274                         NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9275                         return -EINVAL;
9276                 }
9277                 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9278                         NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9279                         return -EINVAL;
9280                 }
9281         }
9282
9283         /* don't call drivers if the effective program didn't change */
9284         if (new_prog != cur_prog) {
9285                 bpf_op = dev_xdp_bpf_op(dev, mode);
9286                 if (!bpf_op) {
9287                         NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9288                         return -EOPNOTSUPP;
9289                 }
9290
9291                 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9292                 if (err)
9293                         return err;
9294         }
9295
9296         if (link)
9297                 dev_xdp_set_link(dev, mode, link);
9298         else
9299                 dev_xdp_set_prog(dev, mode, new_prog);
9300         if (cur_prog)
9301                 bpf_prog_put(cur_prog);
9302
9303         return 0;
9304 }
9305
9306 static int dev_xdp_attach_link(struct net_device *dev,
9307                                struct netlink_ext_ack *extack,
9308                                struct bpf_xdp_link *link)
9309 {
9310         return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9311 }
9312
9313 static int dev_xdp_detach_link(struct net_device *dev,
9314                                struct netlink_ext_ack *extack,
9315                                struct bpf_xdp_link *link)
9316 {
9317         enum bpf_xdp_mode mode;
9318         bpf_op_t bpf_op;
9319
9320         ASSERT_RTNL();
9321
9322         mode = dev_xdp_mode(dev, link->flags);
9323         if (dev_xdp_link(dev, mode) != link)
9324                 return -EINVAL;
9325
9326         bpf_op = dev_xdp_bpf_op(dev, mode);
9327         WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9328         dev_xdp_set_link(dev, mode, NULL);
9329         return 0;
9330 }
9331
9332 static void bpf_xdp_link_release(struct bpf_link *link)
9333 {
9334         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9335
9336         rtnl_lock();
9337
9338         /* if racing with net_device's tear down, xdp_link->dev might be
9339          * already NULL, in which case link was already auto-detached
9340          */
9341         if (xdp_link->dev) {
9342                 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
9343                 xdp_link->dev = NULL;
9344         }
9345
9346         rtnl_unlock();
9347 }
9348
9349 static int bpf_xdp_link_detach(struct bpf_link *link)
9350 {
9351         bpf_xdp_link_release(link);
9352         return 0;
9353 }
9354
9355 static void bpf_xdp_link_dealloc(struct bpf_link *link)
9356 {
9357         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9358
9359         kfree(xdp_link);
9360 }
9361
9362 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9363                                      struct seq_file *seq)
9364 {
9365         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9366         u32 ifindex = 0;
9367
9368         rtnl_lock();
9369         if (xdp_link->dev)
9370                 ifindex = xdp_link->dev->ifindex;
9371         rtnl_unlock();
9372
9373         seq_printf(seq, "ifindex:\t%u\n", ifindex);
9374 }
9375
9376 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9377                                        struct bpf_link_info *info)
9378 {
9379         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9380         u32 ifindex = 0;
9381
9382         rtnl_lock();
9383         if (xdp_link->dev)
9384                 ifindex = xdp_link->dev->ifindex;
9385         rtnl_unlock();
9386
9387         info->xdp.ifindex = ifindex;
9388         return 0;
9389 }
9390
9391 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9392                                struct bpf_prog *old_prog)
9393 {
9394         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9395         enum bpf_xdp_mode mode;
9396         bpf_op_t bpf_op;
9397         int err = 0;
9398
9399         rtnl_lock();
9400
9401         /* link might have been auto-released already, so fail */
9402         if (!xdp_link->dev) {
9403                 err = -ENOLINK;
9404                 goto out_unlock;
9405         }
9406
9407         if (old_prog && link->prog != old_prog) {
9408                 err = -EPERM;
9409                 goto out_unlock;
9410         }
9411         old_prog = link->prog;
9412         if (old_prog->type != new_prog->type ||
9413             old_prog->expected_attach_type != new_prog->expected_attach_type) {
9414                 err = -EINVAL;
9415                 goto out_unlock;
9416         }
9417
9418         if (old_prog == new_prog) {
9419                 /* no-op, don't disturb drivers */
9420                 bpf_prog_put(new_prog);
9421                 goto out_unlock;
9422         }
9423
9424         mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
9425         bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9426         err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9427                               xdp_link->flags, new_prog);
9428         if (err)
9429                 goto out_unlock;
9430
9431         old_prog = xchg(&link->prog, new_prog);
9432         bpf_prog_put(old_prog);
9433
9434 out_unlock:
9435         rtnl_unlock();
9436         return err;
9437 }
9438
9439 static const struct bpf_link_ops bpf_xdp_link_lops = {
9440         .release = bpf_xdp_link_release,
9441         .dealloc = bpf_xdp_link_dealloc,
9442         .detach = bpf_xdp_link_detach,
9443         .show_fdinfo = bpf_xdp_link_show_fdinfo,
9444         .fill_link_info = bpf_xdp_link_fill_link_info,
9445         .update_prog = bpf_xdp_link_update,
9446 };
9447
9448 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9449 {
9450         struct net *net = current->nsproxy->net_ns;
9451         struct bpf_link_primer link_primer;
9452         struct bpf_xdp_link *link;
9453         struct net_device *dev;
9454         int err, fd;
9455
9456         rtnl_lock();
9457         dev = dev_get_by_index(net, attr->link_create.target_ifindex);
9458         if (!dev) {
9459                 rtnl_unlock();
9460                 return -EINVAL;
9461         }
9462
9463         link = kzalloc(sizeof(*link), GFP_USER);
9464         if (!link) {
9465                 err = -ENOMEM;
9466                 goto unlock;
9467         }
9468
9469         bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9470         link->dev = dev;
9471         link->flags = attr->link_create.flags;
9472
9473         err = bpf_link_prime(&link->link, &link_primer);
9474         if (err) {
9475                 kfree(link);
9476                 goto unlock;
9477         }
9478
9479         err = dev_xdp_attach_link(dev, NULL, link);
9480         rtnl_unlock();
9481
9482         if (err) {
9483                 link->dev = NULL;
9484                 bpf_link_cleanup(&link_primer);
9485                 goto out_put_dev;
9486         }
9487
9488         fd = bpf_link_settle(&link_primer);
9489         /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9490         dev_put(dev);
9491         return fd;
9492
9493 unlock:
9494         rtnl_unlock();
9495
9496 out_put_dev:
9497         dev_put(dev);
9498         return err;
9499 }
9500
9501 /**
9502  *      dev_change_xdp_fd - set or clear a bpf program for a device rx path
9503  *      @dev: device
9504  *      @extack: netlink extended ack
9505  *      @fd: new program fd or negative value to clear
9506  *      @expected_fd: old program fd that userspace expects to replace or clear
9507  *      @flags: xdp-related flags
9508  *
9509  *      Set or clear a bpf program for a device
9510  */
9511 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9512                       int fd, int expected_fd, u32 flags)
9513 {
9514         enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
9515         struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9516         int err;
9517
9518         ASSERT_RTNL();
9519
9520         if (fd >= 0) {
9521                 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9522                                                  mode != XDP_MODE_SKB);
9523                 if (IS_ERR(new_prog))
9524                         return PTR_ERR(new_prog);
9525         }
9526
9527         if (expected_fd >= 0) {
9528                 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9529                                                  mode != XDP_MODE_SKB);
9530                 if (IS_ERR(old_prog)) {
9531                         err = PTR_ERR(old_prog);
9532                         old_prog = NULL;
9533                         goto err_out;
9534                 }
9535         }
9536
9537         err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
9538
9539 err_out:
9540         if (err && new_prog)
9541                 bpf_prog_put(new_prog);
9542         if (old_prog)
9543                 bpf_prog_put(old_prog);
9544         return err;
9545 }
9546
9547 /**
9548  *      dev_new_index   -       allocate an ifindex
9549  *      @net: the applicable net namespace
9550  *
9551  *      Returns a suitable unique value for a new device interface
9552  *      number.  The caller must hold the rtnl semaphore or the
9553  *      dev_base_lock to be sure it remains unique.
9554  */
9555 static int dev_new_index(struct net *net)
9556 {
9557         int ifindex = net->ifindex;
9558
9559         for (;;) {
9560                 if (++ifindex <= 0)
9561                         ifindex = 1;
9562                 if (!__dev_get_by_index(net, ifindex))
9563                         return net->ifindex = ifindex;
9564         }
9565 }
9566
9567 /* Delayed registration/unregisteration */
9568 LIST_HEAD(net_todo_list);
9569 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
9570
9571 static void net_set_todo(struct net_device *dev)
9572 {
9573         list_add_tail(&dev->todo_list, &net_todo_list);
9574         atomic_inc(&dev_net(dev)->dev_unreg_count);
9575 }
9576
9577 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
9578         struct net_device *upper, netdev_features_t features)
9579 {
9580         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9581         netdev_features_t feature;
9582         int feature_bit;
9583
9584         for_each_netdev_feature(upper_disables, feature_bit) {
9585                 feature = __NETIF_F_BIT(feature_bit);
9586                 if (!(upper->wanted_features & feature)
9587                     && (features & feature)) {
9588                         netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
9589                                    &feature, upper->name);
9590                         features &= ~feature;
9591                 }
9592         }
9593
9594         return features;
9595 }
9596
9597 static void netdev_sync_lower_features(struct net_device *upper,
9598         struct net_device *lower, netdev_features_t features)
9599 {
9600         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9601         netdev_features_t feature;
9602         int feature_bit;
9603
9604         for_each_netdev_feature(upper_disables, feature_bit) {
9605                 feature = __NETIF_F_BIT(feature_bit);
9606                 if (!(features & feature) && (lower->features & feature)) {
9607                         netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
9608                                    &feature, lower->name);
9609                         lower->wanted_features &= ~feature;
9610                         __netdev_update_features(lower);
9611
9612                         if (unlikely(lower->features & feature))
9613                                 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
9614                                             &feature, lower->name);
9615                         else
9616                                 netdev_features_change(lower);
9617                 }
9618         }
9619 }
9620
9621 static netdev_features_t netdev_fix_features(struct net_device *dev,
9622         netdev_features_t features)
9623 {
9624         /* Fix illegal checksum combinations */
9625         if ((features & NETIF_F_HW_CSUM) &&
9626             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
9627                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
9628                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
9629         }
9630
9631         /* TSO requires that SG is present as well. */
9632         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
9633                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
9634                 features &= ~NETIF_F_ALL_TSO;
9635         }
9636
9637         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
9638                                         !(features & NETIF_F_IP_CSUM)) {
9639                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
9640                 features &= ~NETIF_F_TSO;
9641                 features &= ~NETIF_F_TSO_ECN;
9642         }
9643
9644         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
9645                                          !(features & NETIF_F_IPV6_CSUM)) {
9646                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
9647                 features &= ~NETIF_F_TSO6;
9648         }
9649
9650         /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
9651         if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
9652                 features &= ~NETIF_F_TSO_MANGLEID;
9653
9654         /* TSO ECN requires that TSO is present as well. */
9655         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
9656                 features &= ~NETIF_F_TSO_ECN;
9657
9658         /* Software GSO depends on SG. */
9659         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
9660                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
9661                 features &= ~NETIF_F_GSO;
9662         }
9663
9664         /* GSO partial features require GSO partial be set */
9665         if ((features & dev->gso_partial_features) &&
9666             !(features & NETIF_F_GSO_PARTIAL)) {
9667                 netdev_dbg(dev,
9668                            "Dropping partially supported GSO features since no GSO partial.\n");
9669                 features &= ~dev->gso_partial_features;
9670         }
9671
9672         if (!(features & NETIF_F_RXCSUM)) {
9673                 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
9674                  * successfully merged by hardware must also have the
9675                  * checksum verified by hardware.  If the user does not
9676                  * want to enable RXCSUM, logically, we should disable GRO_HW.
9677                  */
9678                 if (features & NETIF_F_GRO_HW) {
9679                         netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
9680                         features &= ~NETIF_F_GRO_HW;
9681                 }
9682         }
9683
9684         /* LRO/HW-GRO features cannot be combined with RX-FCS */
9685         if (features & NETIF_F_RXFCS) {
9686                 if (features & NETIF_F_LRO) {
9687                         netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
9688                         features &= ~NETIF_F_LRO;
9689                 }
9690
9691                 if (features & NETIF_F_GRO_HW) {
9692                         netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
9693                         features &= ~NETIF_F_GRO_HW;
9694                 }
9695         }
9696
9697         if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
9698                 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
9699                 features &= ~NETIF_F_LRO;
9700         }
9701
9702         if (features & NETIF_F_HW_TLS_TX) {
9703                 bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
9704                         (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
9705                 bool hw_csum = features & NETIF_F_HW_CSUM;
9706
9707                 if (!ip_csum && !hw_csum) {
9708                         netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
9709                         features &= ~NETIF_F_HW_TLS_TX;
9710                 }
9711         }
9712
9713         if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
9714                 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
9715                 features &= ~NETIF_F_HW_TLS_RX;
9716         }
9717
9718         return features;
9719 }
9720
9721 int __netdev_update_features(struct net_device *dev)
9722 {
9723         struct net_device *upper, *lower;
9724         netdev_features_t features;
9725         struct list_head *iter;
9726         int err = -1;
9727
9728         ASSERT_RTNL();
9729
9730         features = netdev_get_wanted_features(dev);
9731
9732         if (dev->netdev_ops->ndo_fix_features)
9733                 features = dev->netdev_ops->ndo_fix_features(dev, features);
9734
9735         /* driver might be less strict about feature dependencies */
9736         features = netdev_fix_features(dev, features);
9737
9738         /* some features can't be enabled if they're off on an upper device */
9739         netdev_for_each_upper_dev_rcu(dev, upper, iter)
9740                 features = netdev_sync_upper_features(dev, upper, features);
9741
9742         if (dev->features == features)
9743                 goto sync_lower;
9744
9745         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
9746                 &dev->features, &features);
9747
9748         if (dev->netdev_ops->ndo_set_features)
9749                 err = dev->netdev_ops->ndo_set_features(dev, features);
9750         else
9751                 err = 0;
9752
9753         if (unlikely(err < 0)) {
9754                 netdev_err(dev,
9755                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
9756                         err, &features, &dev->features);
9757                 /* return non-0 since some features might have changed and
9758                  * it's better to fire a spurious notification than miss it
9759                  */
9760                 return -1;
9761         }
9762
9763 sync_lower:
9764         /* some features must be disabled on lower devices when disabled
9765          * on an upper device (think: bonding master or bridge)
9766          */
9767         netdev_for_each_lower_dev(dev, lower, iter)
9768                 netdev_sync_lower_features(dev, lower, features);
9769
9770         if (!err) {
9771                 netdev_features_t diff = features ^ dev->features;
9772
9773                 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
9774                         /* udp_tunnel_{get,drop}_rx_info both need
9775                          * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
9776                          * device, or they won't do anything.
9777                          * Thus we need to update dev->features
9778                          * *before* calling udp_tunnel_get_rx_info,
9779                          * but *after* calling udp_tunnel_drop_rx_info.
9780                          */
9781                         if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
9782                                 dev->features = features;
9783                                 udp_tunnel_get_rx_info(dev);
9784                         } else {
9785                                 udp_tunnel_drop_rx_info(dev);
9786                         }
9787                 }
9788
9789                 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
9790                         if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
9791                                 dev->features = features;
9792                                 err |= vlan_get_rx_ctag_filter_info(dev);
9793                         } else {
9794                                 vlan_drop_rx_ctag_filter_info(dev);
9795                         }
9796                 }
9797
9798                 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
9799                         if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
9800                                 dev->features = features;
9801                                 err |= vlan_get_rx_stag_filter_info(dev);
9802                         } else {
9803                                 vlan_drop_rx_stag_filter_info(dev);
9804                         }
9805                 }
9806
9807                 dev->features = features;
9808         }
9809
9810         return err < 0 ? 0 : 1;
9811 }
9812
9813 /**
9814  *      netdev_update_features - recalculate device features
9815  *      @dev: the device to check
9816  *
9817  *      Recalculate dev->features set and send notifications if it
9818  *      has changed. Should be called after driver or hardware dependent
9819  *      conditions might have changed that influence the features.
9820  */
9821 void netdev_update_features(struct net_device *dev)
9822 {
9823         if (__netdev_update_features(dev))
9824                 netdev_features_change(dev);
9825 }
9826 EXPORT_SYMBOL(netdev_update_features);
9827
9828 /**
9829  *      netdev_change_features - recalculate device features
9830  *      @dev: the device to check
9831  *
9832  *      Recalculate dev->features set and send notifications even
9833  *      if they have not changed. Should be called instead of
9834  *      netdev_update_features() if also dev->vlan_features might
9835  *      have changed to allow the changes to be propagated to stacked
9836  *      VLAN devices.
9837  */
9838 void netdev_change_features(struct net_device *dev)
9839 {
9840         __netdev_update_features(dev);
9841         netdev_features_change(dev);
9842 }
9843 EXPORT_SYMBOL(netdev_change_features);
9844
9845 /**
9846  *      netif_stacked_transfer_operstate -      transfer operstate
9847  *      @rootdev: the root or lower level device to transfer state from
9848  *      @dev: the device to transfer operstate to
9849  *
9850  *      Transfer operational state from root to device. This is normally
9851  *      called when a stacking relationship exists between the root
9852  *      device and the device(a leaf device).
9853  */
9854 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
9855                                         struct net_device *dev)
9856 {
9857         if (rootdev->operstate == IF_OPER_DORMANT)
9858                 netif_dormant_on(dev);
9859         else
9860                 netif_dormant_off(dev);
9861
9862         if (rootdev->operstate == IF_OPER_TESTING)
9863                 netif_testing_on(dev);
9864         else
9865                 netif_testing_off(dev);
9866
9867         if (netif_carrier_ok(rootdev))
9868                 netif_carrier_on(dev);
9869         else
9870                 netif_carrier_off(dev);
9871 }
9872 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
9873
9874 static int netif_alloc_rx_queues(struct net_device *dev)
9875 {
9876         unsigned int i, count = dev->num_rx_queues;
9877         struct netdev_rx_queue *rx;
9878         size_t sz = count * sizeof(*rx);
9879         int err = 0;
9880
9881         BUG_ON(count < 1);
9882
9883         rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
9884         if (!rx)
9885                 return -ENOMEM;
9886
9887         dev->_rx = rx;
9888
9889         for (i = 0; i < count; i++) {
9890                 rx[i].dev = dev;
9891
9892                 /* XDP RX-queue setup */
9893                 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
9894                 if (err < 0)
9895                         goto err_rxq_info;
9896         }
9897         return 0;
9898
9899 err_rxq_info:
9900         /* Rollback successful reg's and free other resources */
9901         while (i--)
9902                 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
9903         kvfree(dev->_rx);
9904         dev->_rx = NULL;
9905         return err;
9906 }
9907
9908 static void netif_free_rx_queues(struct net_device *dev)
9909 {
9910         unsigned int i, count = dev->num_rx_queues;
9911
9912         /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
9913         if (!dev->_rx)
9914                 return;
9915
9916         for (i = 0; i < count; i++)
9917                 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
9918
9919         kvfree(dev->_rx);
9920 }
9921
9922 static void netdev_init_one_queue(struct net_device *dev,
9923                                   struct netdev_queue *queue, void *_unused)
9924 {
9925         /* Initialize queue lock */
9926         spin_lock_init(&queue->_xmit_lock);
9927         netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
9928         queue->xmit_lock_owner = -1;
9929         netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
9930         queue->dev = dev;
9931 #ifdef CONFIG_BQL
9932         dql_init(&queue->dql, HZ);
9933 #endif
9934 }
9935
9936 static void netif_free_tx_queues(struct net_device *dev)
9937 {
9938         kvfree(dev->_tx);
9939 }
9940
9941 static int netif_alloc_netdev_queues(struct net_device *dev)
9942 {
9943         unsigned int count = dev->num_tx_queues;
9944         struct netdev_queue *tx;
9945         size_t sz = count * sizeof(*tx);
9946
9947         if (count < 1 || count > 0xffff)
9948                 return -EINVAL;
9949
9950         tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
9951         if (!tx)
9952                 return -ENOMEM;
9953
9954         dev->_tx = tx;
9955
9956         netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
9957         spin_lock_init(&dev->tx_global_lock);
9958
9959         return 0;
9960 }
9961
9962 void netif_tx_stop_all_queues(struct net_device *dev)
9963 {
9964         unsigned int i;
9965
9966         for (i = 0; i < dev->num_tx_queues; i++) {
9967                 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
9968
9969                 netif_tx_stop_queue(txq);
9970         }
9971 }
9972 EXPORT_SYMBOL(netif_tx_stop_all_queues);
9973
9974 /**
9975  * register_netdevice() - register a network device
9976  * @dev: device to register
9977  *
9978  * Take a prepared network device structure and make it externally accessible.
9979  * A %NETDEV_REGISTER message is sent to the netdev notifier chain.
9980  * Callers must hold the rtnl lock - you may want register_netdev()
9981  * instead of this.
9982  */
9983 int register_netdevice(struct net_device *dev)
9984 {
9985         int ret;
9986         struct net *net = dev_net(dev);
9987
9988         BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
9989                      NETDEV_FEATURE_COUNT);
9990         BUG_ON(dev_boot_phase);
9991         ASSERT_RTNL();
9992
9993         might_sleep();
9994
9995         /* When net_device's are persistent, this will be fatal. */
9996         BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
9997         BUG_ON(!net);
9998
9999         ret = ethtool_check_ops(dev->ethtool_ops);
10000         if (ret)
10001                 return ret;
10002
10003         spin_lock_init(&dev->addr_list_lock);
10004         netdev_set_addr_lockdep_class(dev);
10005
10006         ret = dev_get_valid_name(net, dev, dev->name);
10007         if (ret < 0)
10008                 goto out;
10009
10010         ret = -ENOMEM;
10011         dev->name_node = netdev_name_node_head_alloc(dev);
10012         if (!dev->name_node)
10013                 goto out;
10014
10015         /* Init, if this function is available */
10016         if (dev->netdev_ops->ndo_init) {
10017                 ret = dev->netdev_ops->ndo_init(dev);
10018                 if (ret) {
10019                         if (ret > 0)
10020                                 ret = -EIO;
10021                         goto err_free_name;
10022                 }
10023         }
10024
10025         if (((dev->hw_features | dev->features) &
10026              NETIF_F_HW_VLAN_CTAG_FILTER) &&
10027             (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10028              !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10029                 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10030                 ret = -EINVAL;
10031                 goto err_uninit;
10032         }
10033
10034         ret = -EBUSY;
10035         if (!dev->ifindex)
10036                 dev->ifindex = dev_new_index(net);
10037         else if (__dev_get_by_index(net, dev->ifindex))
10038                 goto err_uninit;
10039
10040         /* Transfer changeable features to wanted_features and enable
10041          * software offloads (GSO and GRO).
10042          */
10043         dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
10044         dev->features |= NETIF_F_SOFT_FEATURES;
10045
10046         if (dev->udp_tunnel_nic_info) {
10047                 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10048                 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10049         }
10050
10051         dev->wanted_features = dev->features & dev->hw_features;
10052
10053         if (!(dev->flags & IFF_LOOPBACK))
10054                 dev->hw_features |= NETIF_F_NOCACHE_COPY;
10055
10056         /* If IPv4 TCP segmentation offload is supported we should also
10057          * allow the device to enable segmenting the frame with the option
10058          * of ignoring a static IP ID value.  This doesn't enable the
10059          * feature itself but allows the user to enable it later.
10060          */
10061         if (dev->hw_features & NETIF_F_TSO)
10062                 dev->hw_features |= NETIF_F_TSO_MANGLEID;
10063         if (dev->vlan_features & NETIF_F_TSO)
10064                 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10065         if (dev->mpls_features & NETIF_F_TSO)
10066                 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10067         if (dev->hw_enc_features & NETIF_F_TSO)
10068                 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
10069
10070         /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
10071          */
10072         dev->vlan_features |= NETIF_F_HIGHDMA;
10073
10074         /* Make NETIF_F_SG inheritable to tunnel devices.
10075          */
10076         dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
10077
10078         /* Make NETIF_F_SG inheritable to MPLS.
10079          */
10080         dev->mpls_features |= NETIF_F_SG;
10081
10082         ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10083         ret = notifier_to_errno(ret);
10084         if (ret)
10085                 goto err_uninit;
10086
10087         ret = netdev_register_kobject(dev);
10088         write_lock(&dev_base_lock);
10089         dev->reg_state = ret ? NETREG_UNREGISTERED : NETREG_REGISTERED;
10090         write_unlock(&dev_base_lock);
10091         if (ret)
10092                 goto err_uninit_notify;
10093
10094         __netdev_update_features(dev);
10095
10096         /*
10097          *      Default initial state at registry is that the
10098          *      device is present.
10099          */
10100
10101         set_bit(__LINK_STATE_PRESENT, &dev->state);
10102
10103         linkwatch_init_dev(dev);
10104
10105         dev_init_scheduler(dev);
10106
10107         netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
10108         list_netdevice(dev);
10109
10110         add_device_randomness(dev->dev_addr, dev->addr_len);
10111
10112         /* If the device has permanent device address, driver should
10113          * set dev_addr and also addr_assign_type should be set to
10114          * NET_ADDR_PERM (default value).
10115          */
10116         if (dev->addr_assign_type == NET_ADDR_PERM)
10117                 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10118
10119         /* Notify protocols, that a new device appeared. */
10120         ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
10121         ret = notifier_to_errno(ret);
10122         if (ret) {
10123                 /* Expect explicit free_netdev() on failure */
10124                 dev->needs_free_netdev = false;
10125                 unregister_netdevice_queue(dev, NULL);
10126                 goto out;
10127         }
10128         /*
10129          *      Prevent userspace races by waiting until the network
10130          *      device is fully setup before sending notifications.
10131          */
10132         if (!dev->rtnl_link_ops ||
10133             dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10134                 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
10135
10136 out:
10137         return ret;
10138
10139 err_uninit_notify:
10140         call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
10141 err_uninit:
10142         if (dev->netdev_ops->ndo_uninit)
10143                 dev->netdev_ops->ndo_uninit(dev);
10144         if (dev->priv_destructor)
10145                 dev->priv_destructor(dev);
10146 err_free_name:
10147         netdev_name_node_free(dev->name_node);
10148         goto out;
10149 }
10150 EXPORT_SYMBOL(register_netdevice);
10151
10152 /**
10153  *      init_dummy_netdev       - init a dummy network device for NAPI
10154  *      @dev: device to init
10155  *
10156  *      This takes a network device structure and initialize the minimum
10157  *      amount of fields so it can be used to schedule NAPI polls without
10158  *      registering a full blown interface. This is to be used by drivers
10159  *      that need to tie several hardware interfaces to a single NAPI
10160  *      poll scheduler due to HW limitations.
10161  */
10162 int init_dummy_netdev(struct net_device *dev)
10163 {
10164         /* Clear everything. Note we don't initialize spinlocks
10165          * are they aren't supposed to be taken by any of the
10166          * NAPI code and this dummy netdev is supposed to be
10167          * only ever used for NAPI polls
10168          */
10169         memset(dev, 0, sizeof(struct net_device));
10170
10171         /* make sure we BUG if trying to hit standard
10172          * register/unregister code path
10173          */
10174         dev->reg_state = NETREG_DUMMY;
10175
10176         /* NAPI wants this */
10177         INIT_LIST_HEAD(&dev->napi_list);
10178
10179         /* a dummy interface is started by default */
10180         set_bit(__LINK_STATE_PRESENT, &dev->state);
10181         set_bit(__LINK_STATE_START, &dev->state);
10182
10183         /* napi_busy_loop stats accounting wants this */
10184         dev_net_set(dev, &init_net);
10185
10186         /* Note : We dont allocate pcpu_refcnt for dummy devices,
10187          * because users of this 'device' dont need to change
10188          * its refcount.
10189          */
10190
10191         return 0;
10192 }
10193 EXPORT_SYMBOL_GPL(init_dummy_netdev);
10194
10195
10196 /**
10197  *      register_netdev - register a network device
10198  *      @dev: device to register
10199  *
10200  *      Take a completed network device structure and add it to the kernel
10201  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10202  *      chain. 0 is returned on success. A negative errno code is returned
10203  *      on a failure to set up the device, or if the name is a duplicate.
10204  *
10205  *      This is a wrapper around register_netdevice that takes the rtnl semaphore
10206  *      and expands the device name if you passed a format string to
10207  *      alloc_netdev.
10208  */
10209 int register_netdev(struct net_device *dev)
10210 {
10211         int err;
10212
10213         if (rtnl_lock_killable())
10214                 return -EINTR;
10215         err = register_netdevice(dev);
10216         rtnl_unlock();
10217         return err;
10218 }
10219 EXPORT_SYMBOL(register_netdev);
10220
10221 int netdev_refcnt_read(const struct net_device *dev)
10222 {
10223 #ifdef CONFIG_PCPU_DEV_REFCNT
10224         int i, refcnt = 0;
10225
10226         for_each_possible_cpu(i)
10227                 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10228         return refcnt;
10229 #else
10230         return refcount_read(&dev->dev_refcnt);
10231 #endif
10232 }
10233 EXPORT_SYMBOL(netdev_refcnt_read);
10234
10235 int netdev_unregister_timeout_secs __read_mostly = 10;
10236
10237 #define WAIT_REFS_MIN_MSECS 1
10238 #define WAIT_REFS_MAX_MSECS 250
10239 /**
10240  * netdev_wait_allrefs_any - wait until all references are gone.
10241  * @list: list of net_devices to wait on
10242  *
10243  * This is called when unregistering network devices.
10244  *
10245  * Any protocol or device that holds a reference should register
10246  * for netdevice notification, and cleanup and put back the
10247  * reference if they receive an UNREGISTER event.
10248  * We can get stuck here if buggy protocols don't correctly
10249  * call dev_put.
10250  */
10251 static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
10252 {
10253         unsigned long rebroadcast_time, warning_time;
10254         struct net_device *dev;
10255         int wait = 0;
10256
10257         rebroadcast_time = warning_time = jiffies;
10258
10259         list_for_each_entry(dev, list, todo_list)
10260                 if (netdev_refcnt_read(dev) == 1)
10261                         return dev;
10262
10263         while (true) {
10264                 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10265                         rtnl_lock();
10266
10267                         /* Rebroadcast unregister notification */
10268                         list_for_each_entry(dev, list, todo_list)
10269                                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10270
10271                         __rtnl_unlock();
10272                         rcu_barrier();
10273                         rtnl_lock();
10274
10275                         list_for_each_entry(dev, list, todo_list)
10276                                 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10277                                              &dev->state)) {
10278                                         /* We must not have linkwatch events
10279                                          * pending on unregister. If this
10280                                          * happens, we simply run the queue
10281                                          * unscheduled, resulting in a noop
10282                                          * for this device.
10283                                          */
10284                                         linkwatch_run_queue();
10285                                         break;
10286                                 }
10287
10288                         __rtnl_unlock();
10289
10290                         rebroadcast_time = jiffies;
10291                 }
10292
10293                 if (!wait) {
10294                         rcu_barrier();
10295                         wait = WAIT_REFS_MIN_MSECS;
10296                 } else {
10297                         msleep(wait);
10298                         wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10299                 }
10300
10301                 list_for_each_entry(dev, list, todo_list)
10302                         if (netdev_refcnt_read(dev) == 1)
10303                                 return dev;
10304
10305                 if (time_after(jiffies, warning_time +
10306                                READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
10307                         list_for_each_entry(dev, list, todo_list) {
10308                                 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10309                                          dev->name, netdev_refcnt_read(dev));
10310                                 ref_tracker_dir_print(&dev->refcnt_tracker, 10);
10311                         }
10312
10313                         warning_time = jiffies;
10314                 }
10315         }
10316 }
10317
10318 /* The sequence is:
10319  *
10320  *      rtnl_lock();
10321  *      ...
10322  *      register_netdevice(x1);
10323  *      register_netdevice(x2);
10324  *      ...
10325  *      unregister_netdevice(y1);
10326  *      unregister_netdevice(y2);
10327  *      ...
10328  *      rtnl_unlock();
10329  *      free_netdev(y1);
10330  *      free_netdev(y2);
10331  *
10332  * We are invoked by rtnl_unlock().
10333  * This allows us to deal with problems:
10334  * 1) We can delete sysfs objects which invoke hotplug
10335  *    without deadlocking with linkwatch via keventd.
10336  * 2) Since we run with the RTNL semaphore not held, we can sleep
10337  *    safely in order to wait for the netdev refcnt to drop to zero.
10338  *
10339  * We must not return until all unregister events added during
10340  * the interval the lock was held have been completed.
10341  */
10342 void netdev_run_todo(void)
10343 {
10344         struct net_device *dev, *tmp;
10345         struct list_head list;
10346 #ifdef CONFIG_LOCKDEP
10347         struct list_head unlink_list;
10348
10349         list_replace_init(&net_unlink_list, &unlink_list);
10350
10351         while (!list_empty(&unlink_list)) {
10352                 struct net_device *dev = list_first_entry(&unlink_list,
10353                                                           struct net_device,
10354                                                           unlink_list);
10355                 list_del_init(&dev->unlink_list);
10356                 dev->nested_level = dev->lower_level - 1;
10357         }
10358 #endif
10359
10360         /* Snapshot list, allow later requests */
10361         list_replace_init(&net_todo_list, &list);
10362
10363         __rtnl_unlock();
10364
10365         /* Wait for rcu callbacks to finish before next phase */
10366         if (!list_empty(&list))
10367                 rcu_barrier();
10368
10369         list_for_each_entry_safe(dev, tmp, &list, todo_list) {
10370                 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
10371                         netdev_WARN(dev, "run_todo but not unregistering\n");
10372                         list_del(&dev->todo_list);
10373                         continue;
10374                 }
10375
10376                 write_lock(&dev_base_lock);
10377                 dev->reg_state = NETREG_UNREGISTERED;
10378                 write_unlock(&dev_base_lock);
10379                 linkwatch_forget_dev(dev);
10380         }
10381
10382         while (!list_empty(&list)) {
10383                 dev = netdev_wait_allrefs_any(&list);
10384                 list_del(&dev->todo_list);
10385
10386                 /* paranoia */
10387                 BUG_ON(netdev_refcnt_read(dev) != 1);
10388                 BUG_ON(!list_empty(&dev->ptype_all));
10389                 BUG_ON(!list_empty(&dev->ptype_specific));
10390                 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10391                 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10392
10393                 if (dev->priv_destructor)
10394                         dev->priv_destructor(dev);
10395                 if (dev->needs_free_netdev)
10396                         free_netdev(dev);
10397
10398                 if (atomic_dec_and_test(&dev_net(dev)->dev_unreg_count))
10399                         wake_up(&netdev_unregistering_wq);
10400
10401                 /* Free network device */
10402                 kobject_put(&dev->dev.kobj);
10403         }
10404 }
10405
10406 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
10407  * all the same fields in the same order as net_device_stats, with only
10408  * the type differing, but rtnl_link_stats64 may have additional fields
10409  * at the end for newer counters.
10410  */
10411 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
10412                              const struct net_device_stats *netdev_stats)
10413 {
10414         size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
10415         const atomic_long_t *src = (atomic_long_t *)netdev_stats;
10416         u64 *dst = (u64 *)stats64;
10417
10418         BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
10419         for (i = 0; i < n; i++)
10420                 dst[i] = (unsigned long)atomic_long_read(&src[i]);
10421         /* zero out counters that only exist in rtnl_link_stats64 */
10422         memset((char *)stats64 + n * sizeof(u64), 0,
10423                sizeof(*stats64) - n * sizeof(u64));
10424 }
10425 EXPORT_SYMBOL(netdev_stats_to_stats64);
10426
10427 struct net_device_core_stats __percpu *netdev_core_stats_alloc(struct net_device *dev)
10428 {
10429         struct net_device_core_stats __percpu *p;
10430
10431         p = alloc_percpu_gfp(struct net_device_core_stats,
10432                              GFP_ATOMIC | __GFP_NOWARN);
10433
10434         if (p && cmpxchg(&dev->core_stats, NULL, p))
10435                 free_percpu(p);
10436
10437         /* This READ_ONCE() pairs with the cmpxchg() above */
10438         return READ_ONCE(dev->core_stats);
10439 }
10440 EXPORT_SYMBOL(netdev_core_stats_alloc);
10441
10442 /**
10443  *      dev_get_stats   - get network device statistics
10444  *      @dev: device to get statistics from
10445  *      @storage: place to store stats
10446  *
10447  *      Get network statistics from device. Return @storage.
10448  *      The device driver may provide its own method by setting
10449  *      dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
10450  *      otherwise the internal statistics structure is used.
10451  */
10452 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
10453                                         struct rtnl_link_stats64 *storage)
10454 {
10455         const struct net_device_ops *ops = dev->netdev_ops;
10456         const struct net_device_core_stats __percpu *p;
10457
10458         if (ops->ndo_get_stats64) {
10459                 memset(storage, 0, sizeof(*storage));
10460                 ops->ndo_get_stats64(dev, storage);
10461         } else if (ops->ndo_get_stats) {
10462                 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
10463         } else {
10464                 netdev_stats_to_stats64(storage, &dev->stats);
10465         }
10466
10467         /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
10468         p = READ_ONCE(dev->core_stats);
10469         if (p) {
10470                 const struct net_device_core_stats *core_stats;
10471                 int i;
10472
10473                 for_each_possible_cpu(i) {
10474                         core_stats = per_cpu_ptr(p, i);
10475                         storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
10476                         storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
10477                         storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
10478                         storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
10479                 }
10480         }
10481         return storage;
10482 }
10483 EXPORT_SYMBOL(dev_get_stats);
10484
10485 /**
10486  *      dev_fetch_sw_netstats - get per-cpu network device statistics
10487  *      @s: place to store stats
10488  *      @netstats: per-cpu network stats to read from
10489  *
10490  *      Read per-cpu network statistics and populate the related fields in @s.
10491  */
10492 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
10493                            const struct pcpu_sw_netstats __percpu *netstats)
10494 {
10495         int cpu;
10496
10497         for_each_possible_cpu(cpu) {
10498                 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
10499                 const struct pcpu_sw_netstats *stats;
10500                 unsigned int start;
10501
10502                 stats = per_cpu_ptr(netstats, cpu);
10503                 do {
10504                         start = u64_stats_fetch_begin(&stats->syncp);
10505                         rx_packets = u64_stats_read(&stats->rx_packets);
10506                         rx_bytes   = u64_stats_read(&stats->rx_bytes);
10507                         tx_packets = u64_stats_read(&stats->tx_packets);
10508                         tx_bytes   = u64_stats_read(&stats->tx_bytes);
10509                 } while (u64_stats_fetch_retry(&stats->syncp, start));
10510
10511                 s->rx_packets += rx_packets;
10512                 s->rx_bytes   += rx_bytes;
10513                 s->tx_packets += tx_packets;
10514                 s->tx_bytes   += tx_bytes;
10515         }
10516 }
10517 EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
10518
10519 /**
10520  *      dev_get_tstats64 - ndo_get_stats64 implementation
10521  *      @dev: device to get statistics from
10522  *      @s: place to store stats
10523  *
10524  *      Populate @s from dev->stats and dev->tstats. Can be used as
10525  *      ndo_get_stats64() callback.
10526  */
10527 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
10528 {
10529         netdev_stats_to_stats64(s, &dev->stats);
10530         dev_fetch_sw_netstats(s, dev->tstats);
10531 }
10532 EXPORT_SYMBOL_GPL(dev_get_tstats64);
10533
10534 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
10535 {
10536         struct netdev_queue *queue = dev_ingress_queue(dev);
10537
10538 #ifdef CONFIG_NET_CLS_ACT
10539         if (queue)
10540                 return queue;
10541         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
10542         if (!queue)
10543                 return NULL;
10544         netdev_init_one_queue(dev, queue, NULL);
10545         RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
10546         queue->qdisc_sleeping = &noop_qdisc;
10547         rcu_assign_pointer(dev->ingress_queue, queue);
10548 #endif
10549         return queue;
10550 }
10551
10552 static const struct ethtool_ops default_ethtool_ops;
10553
10554 void netdev_set_default_ethtool_ops(struct net_device *dev,
10555                                     const struct ethtool_ops *ops)
10556 {
10557         if (dev->ethtool_ops == &default_ethtool_ops)
10558                 dev->ethtool_ops = ops;
10559 }
10560 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
10561
10562 /**
10563  * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
10564  * @dev: netdev to enable the IRQ coalescing on
10565  *
10566  * Sets a conservative default for SW IRQ coalescing. Users can use
10567  * sysfs attributes to override the default values.
10568  */
10569 void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
10570 {
10571         WARN_ON(dev->reg_state == NETREG_REGISTERED);
10572
10573         dev->gro_flush_timeout = 20000;
10574         dev->napi_defer_hard_irqs = 1;
10575 }
10576 EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
10577
10578 void netdev_freemem(struct net_device *dev)
10579 {
10580         char *addr = (char *)dev - dev->padded;
10581
10582         kvfree(addr);
10583 }
10584
10585 /**
10586  * alloc_netdev_mqs - allocate network device
10587  * @sizeof_priv: size of private data to allocate space for
10588  * @name: device name format string
10589  * @name_assign_type: origin of device name
10590  * @setup: callback to initialize device
10591  * @txqs: the number of TX subqueues to allocate
10592  * @rxqs: the number of RX subqueues to allocate
10593  *
10594  * Allocates a struct net_device with private data area for driver use
10595  * and performs basic initialization.  Also allocates subqueue structs
10596  * for each queue on the device.
10597  */
10598 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
10599                 unsigned char name_assign_type,
10600                 void (*setup)(struct net_device *),
10601                 unsigned int txqs, unsigned int rxqs)
10602 {
10603         struct net_device *dev;
10604         unsigned int alloc_size;
10605         struct net_device *p;
10606
10607         BUG_ON(strlen(name) >= sizeof(dev->name));
10608
10609         if (txqs < 1) {
10610                 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
10611                 return NULL;
10612         }
10613
10614         if (rxqs < 1) {
10615                 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
10616                 return NULL;
10617         }
10618
10619         alloc_size = sizeof(struct net_device);
10620         if (sizeof_priv) {
10621                 /* ensure 32-byte alignment of private area */
10622                 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
10623                 alloc_size += sizeof_priv;
10624         }
10625         /* ensure 32-byte alignment of whole construct */
10626         alloc_size += NETDEV_ALIGN - 1;
10627
10628         p = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10629         if (!p)
10630                 return NULL;
10631
10632         dev = PTR_ALIGN(p, NETDEV_ALIGN);
10633         dev->padded = (char *)dev - (char *)p;
10634
10635         ref_tracker_dir_init(&dev->refcnt_tracker, 128);
10636 #ifdef CONFIG_PCPU_DEV_REFCNT
10637         dev->pcpu_refcnt = alloc_percpu(int);
10638         if (!dev->pcpu_refcnt)
10639                 goto free_dev;
10640         __dev_hold(dev);
10641 #else
10642         refcount_set(&dev->dev_refcnt, 1);
10643 #endif
10644
10645         if (dev_addr_init(dev))
10646                 goto free_pcpu;
10647
10648         dev_mc_init(dev);
10649         dev_uc_init(dev);
10650
10651         dev_net_set(dev, &init_net);
10652
10653         dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
10654         dev->gso_max_segs = GSO_MAX_SEGS;
10655         dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
10656         dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
10657         dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
10658         dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
10659         dev->tso_max_segs = TSO_MAX_SEGS;
10660         dev->upper_level = 1;
10661         dev->lower_level = 1;
10662 #ifdef CONFIG_LOCKDEP
10663         dev->nested_level = 0;
10664         INIT_LIST_HEAD(&dev->unlink_list);
10665 #endif
10666
10667         INIT_LIST_HEAD(&dev->napi_list);
10668         INIT_LIST_HEAD(&dev->unreg_list);
10669         INIT_LIST_HEAD(&dev->close_list);
10670         INIT_LIST_HEAD(&dev->link_watch_list);
10671         INIT_LIST_HEAD(&dev->adj_list.upper);
10672         INIT_LIST_HEAD(&dev->adj_list.lower);
10673         INIT_LIST_HEAD(&dev->ptype_all);
10674         INIT_LIST_HEAD(&dev->ptype_specific);
10675         INIT_LIST_HEAD(&dev->net_notifier_list);
10676 #ifdef CONFIG_NET_SCHED
10677         hash_init(dev->qdisc_hash);
10678 #endif
10679         dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
10680         setup(dev);
10681
10682         if (!dev->tx_queue_len) {
10683                 dev->priv_flags |= IFF_NO_QUEUE;
10684                 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
10685         }
10686
10687         dev->num_tx_queues = txqs;
10688         dev->real_num_tx_queues = txqs;
10689         if (netif_alloc_netdev_queues(dev))
10690                 goto free_all;
10691
10692         dev->num_rx_queues = rxqs;
10693         dev->real_num_rx_queues = rxqs;
10694         if (netif_alloc_rx_queues(dev))
10695                 goto free_all;
10696
10697         strcpy(dev->name, name);
10698         dev->name_assign_type = name_assign_type;
10699         dev->group = INIT_NETDEV_GROUP;
10700         if (!dev->ethtool_ops)
10701                 dev->ethtool_ops = &default_ethtool_ops;
10702
10703         nf_hook_netdev_init(dev);
10704
10705         return dev;
10706
10707 free_all:
10708         free_netdev(dev);
10709         return NULL;
10710
10711 free_pcpu:
10712 #ifdef CONFIG_PCPU_DEV_REFCNT
10713         free_percpu(dev->pcpu_refcnt);
10714 free_dev:
10715 #endif
10716         netdev_freemem(dev);
10717         return NULL;
10718 }
10719 EXPORT_SYMBOL(alloc_netdev_mqs);
10720
10721 /**
10722  * free_netdev - free network device
10723  * @dev: device
10724  *
10725  * This function does the last stage of destroying an allocated device
10726  * interface. The reference to the device object is released. If this
10727  * is the last reference then it will be freed.Must be called in process
10728  * context.
10729  */
10730 void free_netdev(struct net_device *dev)
10731 {
10732         struct napi_struct *p, *n;
10733
10734         might_sleep();
10735
10736         /* When called immediately after register_netdevice() failed the unwind
10737          * handling may still be dismantling the device. Handle that case by
10738          * deferring the free.
10739          */
10740         if (dev->reg_state == NETREG_UNREGISTERING) {
10741                 ASSERT_RTNL();
10742                 dev->needs_free_netdev = true;
10743                 return;
10744         }
10745
10746         netif_free_tx_queues(dev);
10747         netif_free_rx_queues(dev);
10748
10749         kfree(rcu_dereference_protected(dev->ingress_queue, 1));
10750
10751         /* Flush device addresses */
10752         dev_addr_flush(dev);
10753
10754         list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10755                 netif_napi_del(p);
10756
10757         ref_tracker_dir_exit(&dev->refcnt_tracker);
10758 #ifdef CONFIG_PCPU_DEV_REFCNT
10759         free_percpu(dev->pcpu_refcnt);
10760         dev->pcpu_refcnt = NULL;
10761 #endif
10762         free_percpu(dev->core_stats);
10763         dev->core_stats = NULL;
10764         free_percpu(dev->xdp_bulkq);
10765         dev->xdp_bulkq = NULL;
10766
10767         /*  Compatibility with error handling in drivers */
10768         if (dev->reg_state == NETREG_UNINITIALIZED) {
10769                 netdev_freemem(dev);
10770                 return;
10771         }
10772
10773         BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
10774         dev->reg_state = NETREG_RELEASED;
10775
10776         /* will free via device release */
10777         put_device(&dev->dev);
10778 }
10779 EXPORT_SYMBOL(free_netdev);
10780
10781 /**
10782  *      synchronize_net -  Synchronize with packet receive processing
10783  *
10784  *      Wait for packets currently being received to be done.
10785  *      Does not block later packets from starting.
10786  */
10787 void synchronize_net(void)
10788 {
10789         might_sleep();
10790         if (rtnl_is_locked())
10791                 synchronize_rcu_expedited();
10792         else
10793                 synchronize_rcu();
10794 }
10795 EXPORT_SYMBOL(synchronize_net);
10796
10797 /**
10798  *      unregister_netdevice_queue - remove device from the kernel
10799  *      @dev: device
10800  *      @head: list
10801  *
10802  *      This function shuts down a device interface and removes it
10803  *      from the kernel tables.
10804  *      If head not NULL, device is queued to be unregistered later.
10805  *
10806  *      Callers must hold the rtnl semaphore.  You may want
10807  *      unregister_netdev() instead of this.
10808  */
10809
10810 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
10811 {
10812         ASSERT_RTNL();
10813
10814         if (head) {
10815                 list_move_tail(&dev->unreg_list, head);
10816         } else {
10817                 LIST_HEAD(single);
10818
10819                 list_add(&dev->unreg_list, &single);
10820                 unregister_netdevice_many(&single);
10821         }
10822 }
10823 EXPORT_SYMBOL(unregister_netdevice_queue);
10824
10825 void unregister_netdevice_many_notify(struct list_head *head,
10826                                       u32 portid, const struct nlmsghdr *nlh)
10827 {
10828         struct net_device *dev, *tmp;
10829         LIST_HEAD(close_head);
10830
10831         BUG_ON(dev_boot_phase);
10832         ASSERT_RTNL();
10833
10834         if (list_empty(head))
10835                 return;
10836
10837         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
10838                 /* Some devices call without registering
10839                  * for initialization unwind. Remove those
10840                  * devices and proceed with the remaining.
10841                  */
10842                 if (dev->reg_state == NETREG_UNINITIALIZED) {
10843                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
10844                                  dev->name, dev);
10845
10846                         WARN_ON(1);
10847                         list_del(&dev->unreg_list);
10848                         continue;
10849                 }
10850                 dev->dismantle = true;
10851                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
10852         }
10853
10854         /* If device is running, close it first. */
10855         list_for_each_entry(dev, head, unreg_list)
10856                 list_add_tail(&dev->close_list, &close_head);
10857         dev_close_many(&close_head, true);
10858
10859         list_for_each_entry(dev, head, unreg_list) {
10860                 /* And unlink it from device chain. */
10861                 write_lock(&dev_base_lock);
10862                 unlist_netdevice(dev, false);
10863                 dev->reg_state = NETREG_UNREGISTERING;
10864                 write_unlock(&dev_base_lock);
10865         }
10866         flush_all_backlogs();
10867
10868         synchronize_net();
10869
10870         list_for_each_entry(dev, head, unreg_list) {
10871                 struct sk_buff *skb = NULL;
10872
10873                 /* Shutdown queueing discipline. */
10874                 dev_shutdown(dev);
10875
10876                 dev_xdp_uninstall(dev);
10877                 bpf_dev_bound_netdev_unregister(dev);
10878
10879                 netdev_offload_xstats_disable_all(dev);
10880
10881                 /* Notify protocols, that we are about to destroy
10882                  * this device. They should clean all the things.
10883                  */
10884                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10885
10886                 if (!dev->rtnl_link_ops ||
10887                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10888                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
10889                                                      GFP_KERNEL, NULL, 0,
10890                                                      portid, nlh);
10891
10892                 /*
10893                  *      Flush the unicast and multicast chains
10894                  */
10895                 dev_uc_flush(dev);
10896                 dev_mc_flush(dev);
10897
10898                 netdev_name_node_alt_flush(dev);
10899                 netdev_name_node_free(dev->name_node);
10900
10901                 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
10902
10903                 if (dev->netdev_ops->ndo_uninit)
10904                         dev->netdev_ops->ndo_uninit(dev);
10905
10906                 if (skb)
10907                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
10908
10909                 /* Notifier chain MUST detach us all upper devices. */
10910                 WARN_ON(netdev_has_any_upper_dev(dev));
10911                 WARN_ON(netdev_has_any_lower_dev(dev));
10912
10913                 /* Remove entries from kobject tree */
10914                 netdev_unregister_kobject(dev);
10915 #ifdef CONFIG_XPS
10916                 /* Remove XPS queueing entries */
10917                 netif_reset_xps_queues_gt(dev, 0);
10918 #endif
10919         }
10920
10921         synchronize_net();
10922
10923         list_for_each_entry(dev, head, unreg_list) {
10924                 netdev_put(dev, &dev->dev_registered_tracker);
10925                 net_set_todo(dev);
10926         }
10927
10928         list_del(head);
10929 }
10930
10931 /**
10932  *      unregister_netdevice_many - unregister many devices
10933  *      @head: list of devices
10934  *
10935  *  Note: As most callers use a stack allocated list_head,
10936  *  we force a list_del() to make sure stack wont be corrupted later.
10937  */
10938 void unregister_netdevice_many(struct list_head *head)
10939 {
10940         unregister_netdevice_many_notify(head, 0, NULL);
10941 }
10942 EXPORT_SYMBOL(unregister_netdevice_many);
10943
10944 /**
10945  *      unregister_netdev - remove device from the kernel
10946  *      @dev: device
10947  *
10948  *      This function shuts down a device interface and removes it
10949  *      from the kernel tables.
10950  *
10951  *      This is just a wrapper for unregister_netdevice that takes
10952  *      the rtnl semaphore.  In general you want to use this and not
10953  *      unregister_netdevice.
10954  */
10955 void unregister_netdev(struct net_device *dev)
10956 {
10957         rtnl_lock();
10958         unregister_netdevice(dev);
10959         rtnl_unlock();
10960 }
10961 EXPORT_SYMBOL(unregister_netdev);
10962
10963 /**
10964  *      __dev_change_net_namespace - move device to different nethost namespace
10965  *      @dev: device
10966  *      @net: network namespace
10967  *      @pat: If not NULL name pattern to try if the current device name
10968  *            is already taken in the destination network namespace.
10969  *      @new_ifindex: If not zero, specifies device index in the target
10970  *                    namespace.
10971  *
10972  *      This function shuts down a device interface and moves it
10973  *      to a new network namespace. On success 0 is returned, on
10974  *      a failure a netagive errno code is returned.
10975  *
10976  *      Callers must hold the rtnl semaphore.
10977  */
10978
10979 int __dev_change_net_namespace(struct net_device *dev, struct net *net,
10980                                const char *pat, int new_ifindex)
10981 {
10982         struct net *net_old = dev_net(dev);
10983         int err, new_nsid;
10984
10985         ASSERT_RTNL();
10986
10987         /* Don't allow namespace local devices to be moved. */
10988         err = -EINVAL;
10989         if (dev->features & NETIF_F_NETNS_LOCAL)
10990                 goto out;
10991
10992         /* Ensure the device has been registrered */
10993         if (dev->reg_state != NETREG_REGISTERED)
10994                 goto out;
10995
10996         /* Get out if there is nothing todo */
10997         err = 0;
10998         if (net_eq(net_old, net))
10999                 goto out;
11000
11001         /* Pick the destination device name, and ensure
11002          * we can use it in the destination network namespace.
11003          */
11004         err = -EEXIST;
11005         if (netdev_name_in_use(net, dev->name)) {
11006                 /* We get here if we can't use the current device name */
11007                 if (!pat)
11008                         goto out;
11009                 err = dev_get_valid_name(net, dev, pat);
11010                 if (err < 0)
11011                         goto out;
11012         }
11013
11014         /* Check that new_ifindex isn't used yet. */
11015         err = -EBUSY;
11016         if (new_ifindex && __dev_get_by_index(net, new_ifindex))
11017                 goto out;
11018
11019         /*
11020          * And now a mini version of register_netdevice unregister_netdevice.
11021          */
11022
11023         /* If device is running close it first. */
11024         dev_close(dev);
11025
11026         /* And unlink it from device chain */
11027         unlist_netdevice(dev, true);
11028
11029         synchronize_net();
11030
11031         /* Shutdown queueing discipline. */
11032         dev_shutdown(dev);
11033
11034         /* Notify protocols, that we are about to destroy
11035          * this device. They should clean all the things.
11036          *
11037          * Note that dev->reg_state stays at NETREG_REGISTERED.
11038          * This is wanted because this way 8021q and macvlan know
11039          * the device is just moving and can keep their slaves up.
11040          */
11041         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11042         rcu_barrier();
11043
11044         new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11045         /* If there is an ifindex conflict assign a new one */
11046         if (!new_ifindex) {
11047                 if (__dev_get_by_index(net, dev->ifindex))
11048                         new_ifindex = dev_new_index(net);
11049                 else
11050                         new_ifindex = dev->ifindex;
11051         }
11052
11053         rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11054                             new_ifindex);
11055
11056         /*
11057          *      Flush the unicast and multicast chains
11058          */
11059         dev_uc_flush(dev);
11060         dev_mc_flush(dev);
11061
11062         /* Send a netdev-removed uevent to the old namespace */
11063         kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
11064         netdev_adjacent_del_links(dev);
11065
11066         /* Move per-net netdevice notifiers that are following the netdevice */
11067         move_netdevice_notifiers_dev_net(dev, net);
11068
11069         /* Actually switch the network namespace */
11070         dev_net_set(dev, net);
11071         dev->ifindex = new_ifindex;
11072
11073         /* Send a netdev-add uevent to the new namespace */
11074         kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
11075         netdev_adjacent_add_links(dev);
11076
11077         /* Fixup kobjects */
11078         err = device_rename(&dev->dev, dev->name);
11079         WARN_ON(err);
11080
11081         /* Adapt owner in case owning user namespace of target network
11082          * namespace is different from the original one.
11083          */
11084         err = netdev_change_owner(dev, net_old, net);
11085         WARN_ON(err);
11086
11087         /* Add the device back in the hashes */
11088         list_netdevice(dev);
11089
11090         /* Notify protocols, that a new device appeared. */
11091         call_netdevice_notifiers(NETDEV_REGISTER, dev);
11092
11093         /*
11094          *      Prevent userspace races by waiting until the network
11095          *      device is fully setup before sending notifications.
11096          */
11097         rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
11098
11099         synchronize_net();
11100         err = 0;
11101 out:
11102         return err;
11103 }
11104 EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
11105
11106 static int dev_cpu_dead(unsigned int oldcpu)
11107 {
11108         struct sk_buff **list_skb;
11109         struct sk_buff *skb;
11110         unsigned int cpu;
11111         struct softnet_data *sd, *oldsd, *remsd = NULL;
11112
11113         local_irq_disable();
11114         cpu = smp_processor_id();
11115         sd = &per_cpu(softnet_data, cpu);
11116         oldsd = &per_cpu(softnet_data, oldcpu);
11117
11118         /* Find end of our completion_queue. */
11119         list_skb = &sd->completion_queue;
11120         while (*list_skb)
11121                 list_skb = &(*list_skb)->next;
11122         /* Append completion queue from offline CPU. */
11123         *list_skb = oldsd->completion_queue;
11124         oldsd->completion_queue = NULL;
11125
11126         /* Append output queue from offline CPU. */
11127         if (oldsd->output_queue) {
11128                 *sd->output_queue_tailp = oldsd->output_queue;
11129                 sd->output_queue_tailp = oldsd->output_queue_tailp;
11130                 oldsd->output_queue = NULL;
11131                 oldsd->output_queue_tailp = &oldsd->output_queue;
11132         }
11133         /* Append NAPI poll list from offline CPU, with one exception :
11134          * process_backlog() must be called by cpu owning percpu backlog.
11135          * We properly handle process_queue & input_pkt_queue later.
11136          */
11137         while (!list_empty(&oldsd->poll_list)) {
11138                 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11139                                                             struct napi_struct,
11140                                                             poll_list);
11141
11142                 list_del_init(&napi->poll_list);
11143                 if (napi->poll == process_backlog)
11144                         napi->state = 0;
11145                 else
11146                         ____napi_schedule(sd, napi);
11147         }
11148
11149         raise_softirq_irqoff(NET_TX_SOFTIRQ);
11150         local_irq_enable();
11151
11152 #ifdef CONFIG_RPS
11153         remsd = oldsd->rps_ipi_list;
11154         oldsd->rps_ipi_list = NULL;
11155 #endif
11156         /* send out pending IPI's on offline CPU */
11157         net_rps_send_ipi(remsd);
11158
11159         /* Process offline CPU's input_pkt_queue */
11160         while ((skb = __skb_dequeue(&oldsd->process_queue))) {
11161                 netif_rx(skb);
11162                 input_queue_head_incr(oldsd);
11163         }
11164         while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
11165                 netif_rx(skb);
11166                 input_queue_head_incr(oldsd);
11167         }
11168
11169         return 0;
11170 }
11171
11172 /**
11173  *      netdev_increment_features - increment feature set by one
11174  *      @all: current feature set
11175  *      @one: new feature set
11176  *      @mask: mask feature set
11177  *
11178  *      Computes a new feature set after adding a device with feature set
11179  *      @one to the master device with current feature set @all.  Will not
11180  *      enable anything that is off in @mask. Returns the new feature set.
11181  */
11182 netdev_features_t netdev_increment_features(netdev_features_t all,
11183         netdev_features_t one, netdev_features_t mask)
11184 {
11185         if (mask & NETIF_F_HW_CSUM)
11186                 mask |= NETIF_F_CSUM_MASK;
11187         mask |= NETIF_F_VLAN_CHALLENGED;
11188
11189         all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
11190         all &= one | ~NETIF_F_ALL_FOR_ALL;
11191
11192         /* If one device supports hw checksumming, set for all. */
11193         if (all & NETIF_F_HW_CSUM)
11194                 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
11195
11196         return all;
11197 }
11198 EXPORT_SYMBOL(netdev_increment_features);
11199
11200 static struct hlist_head * __net_init netdev_create_hash(void)
11201 {
11202         int i;
11203         struct hlist_head *hash;
11204
11205         hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
11206         if (hash != NULL)
11207                 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11208                         INIT_HLIST_HEAD(&hash[i]);
11209
11210         return hash;
11211 }
11212
11213 /* Initialize per network namespace state */
11214 static int __net_init netdev_init(struct net *net)
11215 {
11216         BUILD_BUG_ON(GRO_HASH_BUCKETS >
11217                      8 * sizeof_field(struct napi_struct, gro_bitmask));
11218
11219         INIT_LIST_HEAD(&net->dev_base_head);
11220
11221         net->dev_name_head = netdev_create_hash();
11222         if (net->dev_name_head == NULL)
11223                 goto err_name;
11224
11225         net->dev_index_head = netdev_create_hash();
11226         if (net->dev_index_head == NULL)
11227                 goto err_idx;
11228
11229         RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11230
11231         return 0;
11232
11233 err_idx:
11234         kfree(net->dev_name_head);
11235 err_name:
11236         return -ENOMEM;
11237 }
11238
11239 /**
11240  *      netdev_drivername - network driver for the device
11241  *      @dev: network device
11242  *
11243  *      Determine network driver for device.
11244  */
11245 const char *netdev_drivername(const struct net_device *dev)
11246 {
11247         const struct device_driver *driver;
11248         const struct device *parent;
11249         const char *empty = "";
11250
11251         parent = dev->dev.parent;
11252         if (!parent)
11253                 return empty;
11254
11255         driver = parent->driver;
11256         if (driver && driver->name)
11257                 return driver->name;
11258         return empty;
11259 }
11260
11261 static void __netdev_printk(const char *level, const struct net_device *dev,
11262                             struct va_format *vaf)
11263 {
11264         if (dev && dev->dev.parent) {
11265                 dev_printk_emit(level[1] - '0',
11266                                 dev->dev.parent,
11267                                 "%s %s %s%s: %pV",
11268                                 dev_driver_string(dev->dev.parent),
11269                                 dev_name(dev->dev.parent),
11270                                 netdev_name(dev), netdev_reg_state(dev),
11271                                 vaf);
11272         } else if (dev) {
11273                 printk("%s%s%s: %pV",
11274                        level, netdev_name(dev), netdev_reg_state(dev), vaf);
11275         } else {
11276                 printk("%s(NULL net_device): %pV", level, vaf);
11277         }
11278 }
11279
11280 void netdev_printk(const char *level, const struct net_device *dev,
11281                    const char *format, ...)
11282 {
11283         struct va_format vaf;
11284         va_list args;
11285
11286         va_start(args, format);
11287
11288         vaf.fmt = format;
11289         vaf.va = &args;
11290
11291         __netdev_printk(level, dev, &vaf);
11292
11293         va_end(args);
11294 }
11295 EXPORT_SYMBOL(netdev_printk);
11296
11297 #define define_netdev_printk_level(func, level)                 \
11298 void func(const struct net_device *dev, const char *fmt, ...)   \
11299 {                                                               \
11300         struct va_format vaf;                                   \
11301         va_list args;                                           \
11302                                                                 \
11303         va_start(args, fmt);                                    \
11304                                                                 \
11305         vaf.fmt = fmt;                                          \
11306         vaf.va = &args;                                         \
11307                                                                 \
11308         __netdev_printk(level, dev, &vaf);                      \
11309                                                                 \
11310         va_end(args);                                           \
11311 }                                                               \
11312 EXPORT_SYMBOL(func);
11313
11314 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
11315 define_netdev_printk_level(netdev_alert, KERN_ALERT);
11316 define_netdev_printk_level(netdev_crit, KERN_CRIT);
11317 define_netdev_printk_level(netdev_err, KERN_ERR);
11318 define_netdev_printk_level(netdev_warn, KERN_WARNING);
11319 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
11320 define_netdev_printk_level(netdev_info, KERN_INFO);
11321
11322 static void __net_exit netdev_exit(struct net *net)
11323 {
11324         kfree(net->dev_name_head);
11325         kfree(net->dev_index_head);
11326         if (net != &init_net)
11327                 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
11328 }
11329
11330 static struct pernet_operations __net_initdata netdev_net_ops = {
11331         .init = netdev_init,
11332         .exit = netdev_exit,
11333 };
11334
11335 static void __net_exit default_device_exit_net(struct net *net)
11336 {
11337         struct net_device *dev, *aux;
11338         /*
11339          * Push all migratable network devices back to the
11340          * initial network namespace
11341          */
11342         ASSERT_RTNL();
11343         for_each_netdev_safe(net, dev, aux) {
11344                 int err;
11345                 char fb_name[IFNAMSIZ];
11346
11347                 /* Ignore unmoveable devices (i.e. loopback) */
11348                 if (dev->features & NETIF_F_NETNS_LOCAL)
11349                         continue;
11350
11351                 /* Leave virtual devices for the generic cleanup */
11352                 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
11353                         continue;
11354
11355                 /* Push remaining network devices to init_net */
11356                 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
11357                 if (netdev_name_in_use(&init_net, fb_name))
11358                         snprintf(fb_name, IFNAMSIZ, "dev%%d");
11359                 err = dev_change_net_namespace(dev, &init_net, fb_name);
11360                 if (err) {
11361                         pr_emerg("%s: failed to move %s to init_net: %d\n",
11362                                  __func__, dev->name, err);
11363                         BUG();
11364                 }
11365         }
11366 }
11367
11368 static void __net_exit default_device_exit_batch(struct list_head *net_list)
11369 {
11370         /* At exit all network devices most be removed from a network
11371          * namespace.  Do this in the reverse order of registration.
11372          * Do this across as many network namespaces as possible to
11373          * improve batching efficiency.
11374          */
11375         struct net_device *dev;
11376         struct net *net;
11377         LIST_HEAD(dev_kill_list);
11378
11379         rtnl_lock();
11380         list_for_each_entry(net, net_list, exit_list) {
11381                 default_device_exit_net(net);
11382                 cond_resched();
11383         }
11384
11385         list_for_each_entry(net, net_list, exit_list) {
11386                 for_each_netdev_reverse(net, dev) {
11387                         if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
11388                                 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
11389                         else
11390                                 unregister_netdevice_queue(dev, &dev_kill_list);
11391                 }
11392         }
11393         unregister_netdevice_many(&dev_kill_list);
11394         rtnl_unlock();
11395 }
11396
11397 static struct pernet_operations __net_initdata default_device_ops = {
11398         .exit_batch = default_device_exit_batch,
11399 };
11400
11401 /*
11402  *      Initialize the DEV module. At boot time this walks the device list and
11403  *      unhooks any devices that fail to initialise (normally hardware not
11404  *      present) and leaves us with a valid list of present and active devices.
11405  *
11406  */
11407
11408 /*
11409  *       This is called single threaded during boot, so no need
11410  *       to take the rtnl semaphore.
11411  */
11412 static int __init net_dev_init(void)
11413 {
11414         int i, rc = -ENOMEM;
11415
11416         BUG_ON(!dev_boot_phase);
11417
11418         if (dev_proc_init())
11419                 goto out;
11420
11421         if (netdev_kobject_init())
11422                 goto out;
11423
11424         INIT_LIST_HEAD(&ptype_all);
11425         for (i = 0; i < PTYPE_HASH_SIZE; i++)
11426                 INIT_LIST_HEAD(&ptype_base[i]);
11427
11428         if (register_pernet_subsys(&netdev_net_ops))
11429                 goto out;
11430
11431         /*
11432          *      Initialise the packet receive queues.
11433          */
11434
11435         for_each_possible_cpu(i) {
11436                 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
11437                 struct softnet_data *sd = &per_cpu(softnet_data, i);
11438
11439                 INIT_WORK(flush, flush_backlog);
11440
11441                 skb_queue_head_init(&sd->input_pkt_queue);
11442                 skb_queue_head_init(&sd->process_queue);
11443 #ifdef CONFIG_XFRM_OFFLOAD
11444                 skb_queue_head_init(&sd->xfrm_backlog);
11445 #endif
11446                 INIT_LIST_HEAD(&sd->poll_list);
11447                 sd->output_queue_tailp = &sd->output_queue;
11448 #ifdef CONFIG_RPS
11449                 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
11450                 sd->cpu = i;
11451 #endif
11452                 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
11453                 spin_lock_init(&sd->defer_lock);
11454
11455                 init_gro_hash(&sd->backlog);
11456                 sd->backlog.poll = process_backlog;
11457                 sd->backlog.weight = weight_p;
11458         }
11459
11460         dev_boot_phase = 0;
11461
11462         /* The loopback device is special if any other network devices
11463          * is present in a network namespace the loopback device must
11464          * be present. Since we now dynamically allocate and free the
11465          * loopback device ensure this invariant is maintained by
11466          * keeping the loopback device as the first device on the
11467          * list of network devices.  Ensuring the loopback devices
11468          * is the first device that appears and the last network device
11469          * that disappears.
11470          */
11471         if (register_pernet_device(&loopback_net_ops))
11472                 goto out;
11473
11474         if (register_pernet_device(&default_device_ops))
11475                 goto out;
11476
11477         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11478         open_softirq(NET_RX_SOFTIRQ, net_rx_action);
11479
11480         rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11481                                        NULL, dev_cpu_dead);
11482         WARN_ON(rc < 0);
11483         rc = 0;
11484 out:
11485         return rc;
11486 }
11487
11488 subsys_initcall(net_dev_init);