net/ipv4/tcp.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   4  *              operating system.  INET is implemented using the  BSD Socket
   5  *              interface as the means of communication with the user level.
   6  *
   7  *              Implementation of the Transmission Control Protocol(TCP).
   8  *
   9  * Authors:     Ross Biro
  10  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  11  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  12  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  13  *              Florian La Roche, <flla@stud.uni-sb.de>
  14  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  15  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  16  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  17  *              Matthew Dillon, <dillon@apollo.west.oic.com>
  18  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  19  *              Jorge Cwik, <jorge@laser.satlink.net>
  20  *
  21  * Fixes:
  22  *              Alan Cox        :       Numerous verify_area() calls
  23  *              Alan Cox        :       Set the ACK bit on a reset
  24  *              Alan Cox        :       Stopped it crashing if it closed while
  25  *                                      sk->inuse=1 and was trying to connect
  26  *                                      (tcp_err()).
  27  *              Alan Cox        :       All icmp error handling was broken
  28  *                                      pointers passed where wrong and the
  29  *                                      socket was looked up backwards. Nobody
  30  *                                      tested any icmp error code obviously.
  31  *              Alan Cox        :       tcp_err() now handled properly. It
  32  *                                      wakes people on errors. poll
  33  *                                      behaves and the icmp error race
  34  *                                      has gone by moving it into sock.c
  35  *              Alan Cox        :       tcp_send_reset() fixed to work for
  36  *                                      everything not just packets for
  37  *                                      unknown sockets.
  38  *              Alan Cox        :       tcp option processing.
  39  *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  40  *                                      syn rule wrong]
  41  *              Herp Rosmanith  :       More reset fixes
  42  *              Alan Cox        :       No longer acks invalid rst frames.
  43  *                                      Acking any kind of RST is right out.
  44  *              Alan Cox        :       Sets an ignore me flag on an rst
  45  *                                      receive otherwise odd bits of prattle
  46  *                                      escape still
  47  *              Alan Cox        :       Fixed another acking RST frame bug.
  48  *                                      Should stop LAN workplace lockups.
  49  *              Alan Cox        :       Some tidyups using the new skb list
  50  *                                      facilities
  51  *              Alan Cox        :       sk->keepopen now seems to work
  52  *              Alan Cox        :       Pulls options out correctly on accepts
  53  *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  54  *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  55  *                                      bit to skb ops.
  56  *              Alan Cox        :       Tidied tcp_data to avoid a potential
  57  *                                      nasty.
  58  *              Alan Cox        :       Added some better commenting, as the
  59  *                                      tcp is hard to follow
  60  *              Alan Cox        :       Removed incorrect check for 20 * psh
  61  *      Michael O'Reilly        :       ack < copied bug fix.
  62  *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  63  *              Alan Cox        :       FIN with no memory -> CRASH
  64  *              Alan Cox        :       Added socket option proto entries.
  65  *                                      Also added awareness of them to accept.
  66  *              Alan Cox        :       Added TCP options (SOL_TCP)
  67  *              Alan Cox        :       Switched wakeup calls to callbacks,
  68  *                                      so the kernel can layer network
  69  *                                      sockets.
  70  *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  71  *              Alan Cox        :       Handle FIN (more) properly (we hope).
  72  *              Alan Cox        :       RST frames sent on unsynchronised
  73  *                                      state ack error.
  74  *              Alan Cox        :       Put in missing check for SYN bit.
  75  *              Alan Cox        :       Added tcp_select_window() aka NET2E
  76  *                                      window non shrink trick.
  77  *              Alan Cox        :       Added a couple of small NET2E timer
  78  *                                      fixes
  79  *              Charles Hedrick :       TCP fixes
  80  *              Toomas Tamm     :       TCP window fixes
  81  *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  82  *              Charles Hedrick :       Rewrote most of it to actually work
  83  *              Linus           :       Rewrote tcp_read() and URG handling
  84  *                                      completely
  85  *              Gerhard Koerting:       Fixed some missing timer handling
  86  *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  87  *              Gerhard Koerting:       PC/TCP workarounds
  88  *              Adam Caldwell   :       Assorted timer/timing errors
  89  *              Matthew Dillon  :       Fixed another RST bug
  90  *              Alan Cox        :       Move to kernel side addressing changes.
  91  *              Alan Cox        :       Beginning work on TCP fastpathing
  92  *                                      (not yet usable)
  93  *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  94  *              Alan Cox        :       TCP fast path debugging
  95  *              Alan Cox        :       Window clamping
  96  *              Michael Riepe   :       Bug in tcp_check()
  97  *              Matt Dillon     :       More TCP improvements and RST bug fixes
  98  *              Matt Dillon     :       Yet more small nasties remove from the
  99  *                                      TCP code (Be very nice to this man if
 100  *                                      tcp finally works 100%) 8)
 101  *              Alan Cox        :       BSD accept semantics.
 102  *              Alan Cox        :       Reset on closedown bug.
 103  *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 104  *              Michael Pall    :       Handle poll() after URG properly in
 105  *                                      all cases.
 106  *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 107  *                                      (multi URG PUSH broke rlogin).
 108  *              Michael Pall    :       Fix the multi URG PUSH problem in
 109  *                                      tcp_readable(), poll() after URG
 110  *                                      works now.
 111  *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 112  *                                      BSD api.
 113  *              Alan Cox        :       Changed the semantics of sk->socket to
 114  *                                      fix a race and a signal problem with
 115  *                                      accept() and async I/O.
 116  *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 117  *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 118  *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 119  *                                      clients/servers which listen in on
 120  *                                      fixed ports.
 121  *              Alan Cox        :       Cleaned the above up and shrank it to
 122  *                                      a sensible code size.
 123  *              Alan Cox        :       Self connect lockup fix.
 124  *              Alan Cox        :       No connect to multicast.
 125  *              Ross Biro       :       Close unaccepted children on master
 126  *                                      socket close.
 127  *              Alan Cox        :       Reset tracing code.
 128  *              Alan Cox        :       Spurious resets on shutdown.
 129  *              Alan Cox        :       Giant 15 minute/60 second timer error
 130  *              Alan Cox        :       Small whoops in polling before an
 131  *                                      accept.
 132  *              Alan Cox        :       Kept the state trace facility since
 133  *                                      it's handy for debugging.
 134  *              Alan Cox        :       More reset handler fixes.
 135  *              Alan Cox        :       Started rewriting the code based on
 136  *                                      the RFC's for other useful protocol
 137  *                                      references see: Comer, KA9Q NOS, and
 138  *                                      for a reference on the difference
 139  *                                      between specifications and how BSD
 140  *                                      works see the 4.4lite source.
 141  *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 142  *                                      close.
 143  *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 144  *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 145  *              Alan Cox        :       Reimplemented timers as per the RFC
 146  *                                      and using multiple timers for sanity.
 147  *              Alan Cox        :       Small bug fixes, and a lot of new
 148  *                                      comments.
 149  *              Alan Cox        :       Fixed dual reader crash by locking
 150  *                                      the buffers (much like datagram.c)
 151  *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 152  *                                      now gets fed up of retrying without
 153  *                                      (even a no space) answer.
 154  *              Alan Cox        :       Extracted closing code better
 155  *              Alan Cox        :       Fixed the closing state machine to
 156  *                                      resemble the RFC.
 157  *              Alan Cox        :       More 'per spec' fixes.
 158  *              Jorge Cwik      :       Even faster checksumming.
 159  *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 160  *                                      only frames. At least one pc tcp stack
 161  *                                      generates them.
 162  *              Alan Cox        :       Cache last socket.
 163  *              Alan Cox        :       Per route irtt.
 164  *              Matt Day        :       poll()->select() match BSD precisely on error
 165  *              Alan Cox        :       New buffers
 166  *              Marc Tamsky     :       Various sk->prot->retransmits and
 167  *                                      sk->retransmits misupdating fixed.
 168  *                                      Fixed tcp_write_timeout: stuck close,
 169  *                                      and TCP syn retries gets used now.
 170  *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 171  *                                      ack if state is TCP_CLOSED.
 172  *              Alan Cox        :       Look up device on a retransmit - routes may
 173  *                                      change. Doesn't yet cope with MSS shrink right
 174  *                                      but it's a start!
 175  *              Marc Tamsky     :       Closing in closing fixes.
 176  *              Mike Shaver     :       RFC1122 verifications.
 177  *              Alan Cox        :       rcv_saddr errors.
 178  *              Alan Cox        :       Block double connect().
 179  *              Alan Cox        :       Small hooks for enSKIP.
 180  *              Alexey Kuznetsov:       Path MTU discovery.
 181  *              Alan Cox        :       Support soft errors.
 182  *              Alan Cox        :       Fix MTU discovery pathological case
 183  *                                      when the remote claims no mtu!
 184  *              Marc Tamsky     :       TCP_CLOSE fix.
 185  *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 186  *                                      window but wrong (fixes NT lpd problems)
 187  *              Pedro Roque     :       Better TCP window handling, delayed ack.
 188  *              Joerg Reuter    :       No modification of locked buffers in
 189  *                                      tcp_do_retransmit()
 190  *              Eric Schenk     :       Changed receiver side silly window
 191  *                                      avoidance algorithm to BSD style
 192  *                                      algorithm. This doubles throughput
 193  *                                      against machines running Solaris,
 194  *                                      and seems to result in general
 195  *                                      improvement.
 196  *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 197  *      Willy Konynenberg       :       Transparent proxying support.
 198  *      Mike McLagan            :       Routing by source
 199  *              Keith Owens     :       Do proper merging with partial SKB's in
 200  *                                      tcp_do_sendmsg to avoid burstiness.
 201  *              Eric Schenk     :       Fix fast close down bug with
 202  *                                      shutdown() followed by close().
 203  *              Andi Kleen      :       Make poll agree with SIGIO
 204  *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 205  *                                      lingertime == 0 (RFC 793 ABORT Call)
 206  *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 207  *                                      csum_and_copy_from_user() if possible.
 208  *
 209  * Description of States:
 210  *
 211  *      TCP_SYN_SENT            sent a connection request, waiting for ack
 212  *
 213  *      TCP_SYN_RECV            received a connection request, sent ack,
 214  *                              waiting for final ack in three-way handshake.
 215  *
 216  *      TCP_ESTABLISHED         connection established
 217  *
 218  *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 219  *                              transmission of remaining buffered data
 220  *
 221  *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 222  *                              to shutdown
 223  *
 224  *      TCP_CLOSING             both sides have shutdown but we still have
 225  *                              data we have to finish sending
 226  *
 227  *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 228  *                              closed, can only be entered from FIN_WAIT2
 229  *                              or CLOSING.  Required because the other end
 230  *                              may not have gotten our last ACK causing it
 231  *                              to retransmit the data packet (which we ignore)
 232  *
 233  *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 234  *                              us to finish writing our data and to shutdown
 235  *                              (we have to close() to move on to LAST_ACK)
 236  *
 237  *      TCP_LAST_ACK            out side has shutdown after remote has
 238  *                              shutdown.  There may still be data in our
 239  *                              buffer that we have to finish sending
 240  *
 241  *      TCP_CLOSE               socket is finished
 242  */
 243
 244 #define pr_fmt(fmt) "TCP: " fmt
 245
 246 #include <crypto/hash.h>
 247 #include <linux/kernel.h>
 248 #include <linux/module.h>
 249 #include <linux/types.h>
 250 #include <linux/fcntl.h>
 251 #include <linux/poll.h>
 252 #include <linux/inet_diag.h>
 253 #include <linux/init.h>
 254 #include <linux/fs.h>
 255 #include <linux/skbuff.h>
 256 #include <linux/scatterlist.h>
 257 #include <linux/splice.h>
 258 #include <linux/net.h>
 259 #include <linux/socket.h>
 260 #include <linux/random.h>
 261 #include <linux/memblock.h>
 262 #include <linux/highmem.h>
 263 #include <linux/swap.h>
 264 #include <linux/cache.h>
 265 #include <linux/err.h>
 266 #include <linux/time.h>
 267 #include <linux/slab.h>
 268 #include <linux/errqueue.h>
 269 #include <linux/static_key.h>
 270
 271 #include <net/icmp.h>
 272 #include <net/inet_common.h>
 273 #include <net/tcp.h>
 274 #include <net/mptcp.h>
 275 #include <net/xfrm.h>
 276 #include <net/ip.h>
 277 #include <net/sock.h>
 278
 279 #include <linux/uaccess.h>
 280 #include <asm/ioctls.h>
 281 #include <net/busy_poll.h>
 282
 283 struct percpu_counter tcp_orphan_count;
 284 EXPORT_SYMBOL_GPL(tcp_orphan_count);
 285
 286 long sysctl_tcp_mem[3] __read_mostly;
 287 EXPORT_SYMBOL(sysctl_tcp_mem);
 288
 289 atomic_long_t tcp_memory_allocated;     /* Current allocated memory. */
 290 EXPORT_SYMBOL(tcp_memory_allocated);
 291
 292 #if IS_ENABLED(CONFIG_SMC)
 293 DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
 294 EXPORT_SYMBOL(tcp_have_smc);
 295 #endif
 296
 297 /*
 298  * Current number of TCP sockets.
 299  */
 300 struct percpu_counter tcp_sockets_allocated;
 301 EXPORT_SYMBOL(tcp_sockets_allocated);
 302
 303 /*
 304  * TCP splice context
 305  */
 306 struct tcp_splice_state {
 307         struct pipe_inode_info *pipe;
 308         size_t len;
 309         unsigned int flags;
 310 };
 311
 312 /*
 313  * Pressure flag: try to collapse.
 314  * Technical note: it is used by multiple contexts non atomically.
 315  * All the __sk_mem_schedule() is of this nature: accounting
 316  * is strict, actions are advisory and have some latency.
 317  */
 318 unsigned long tcp_memory_pressure __read_mostly;
 319 EXPORT_SYMBOL_GPL(tcp_memory_pressure);
 320
 321 DEFINE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
 322 EXPORT_SYMBOL(tcp_rx_skb_cache_key);
 323
 324 DEFINE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
 325
 326 void tcp_enter_memory_pressure(struct sock *sk)
 327 {
 328         unsigned long val;
 329
 330         if (READ_ONCE(tcp_memory_pressure))
 331                 return;
 332         val = jiffies;
 333
 334         if (!val)
 335                 val--;
 336         if (!cmpxchg(&tcp_memory_pressure, 0, val))
 337                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
 338 }
 339 EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure);
 340
 341 void tcp_leave_memory_pressure(struct sock *sk)
 342 {
 343         unsigned long val;
 344
 345         if (!READ_ONCE(tcp_memory_pressure))
 346                 return;
 347         val = xchg(&tcp_memory_pressure, 0);
 348         if (val)
 349                 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
 350                               jiffies_to_msecs(jiffies - val));
 351 }
 352 EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure);
 353
 354 /* Convert seconds to retransmits based on initial and max timeout */
 355 static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
 356 {
 357         u8 res = 0;
 358
 359         if (seconds > 0) {
 360                 int period = timeout;
 361
 362                 res = 1;
 363                 while (seconds > period && res < 255) {
 364                         res++;
 365                         timeout <<= 1;
 366                         if (timeout > rto_max)
 367                                 timeout = rto_max;
 368                         period += timeout;
 369                 }
 370         }
 371         return res;
 372 }
 373
 374 /* Convert retransmits to seconds based on initial and max timeout */
 375 static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
 376 {
 377         int period = 0;
 378
 379         if (retrans > 0) {
 380                 period = timeout;
 381                 while (--retrans) {
 382                         timeout <<= 1;
 383                         if (timeout > rto_max)
 384                                 timeout = rto_max;
 385                         period += timeout;
 386                 }
 387         }
 388         return period;
 389 }
 390
 391 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
 392 {
 393         u32 rate = READ_ONCE(tp->rate_delivered);
 394         u32 intv = READ_ONCE(tp->rate_interval_us);
 395         u64 rate64 = 0;
 396
 397         if (rate && intv) {
 398                 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
 399                 do_div(rate64, intv);
 400         }
 401         return rate64;
 402 }
 403
 404 /* Address-family independent initialization for a tcp_sock.
 405  *
 406  * NOTE: A lot of things set to zero explicitly by call to
 407  *       sk_alloc() so need not be done here.
 408  */
 409 void tcp_init_sock(struct sock *sk)
 410 {
 411         struct inet_connection_sock *icsk = inet_csk(sk);
 412         struct tcp_sock *tp = tcp_sk(sk);
 413
 414         tp->out_of_order_queue = RB_ROOT;
 415         sk->tcp_rtx_queue = RB_ROOT;
 416         tcp_init_xmit_timers(sk);
 417         INIT_LIST_HEAD(&tp->tsq_node);
 418         INIT_LIST_HEAD(&tp->tsorted_sent_queue);
 419
 420         icsk->icsk_rto = TCP_TIMEOUT_INIT;
 421         icsk->icsk_rto_min = TCP_RTO_MIN;
 422         icsk->icsk_delack_max = TCP_DELACK_MAX;
 423         tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
 424         minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
 425
 426         /* So many TCP implementations out there (incorrectly) count the
 427          * initial SYN frame in their delayed-ACK and congestion control
 428          * algorithms that we must have the following bandaid to talk
 429          * efficiently to them.  -DaveM
 430          */
 431         tp->snd_cwnd = TCP_INIT_CWND;
 432
 433         /* There's a bubble in the pipe until at least the first ACK. */
 434         tp->app_limited = ~0U;
 435
 436         /* See draft-stevens-tcpca-spec-01 for discussion of the
 437          * initialization of these values.
 438          */
 439         tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
 440         tp->snd_cwnd_clamp = ~0;
 441         tp->mss_cache = TCP_MSS_DEFAULT;
 442
 443         tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering;
 444         tcp_assign_congestion_control(sk);
 445
 446         tp->tsoffset = 0;
 447         tp->rack.reo_wnd_steps = 1;
 448
 449         sk->sk_write_space = sk_stream_write_space;
 450         sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
 451
 452         icsk->icsk_sync_mss = tcp_sync_mss;
 453
 454         WRITE_ONCE(sk->sk_sndbuf, sock_net(sk)->ipv4.sysctl_tcp_wmem[1]);
 455         WRITE_ONCE(sk->sk_rcvbuf, sock_net(sk)->ipv4.sysctl_tcp_rmem[1]);
 456
 457         sk_sockets_allocated_inc(sk);
 458         sk->sk_route_forced_caps = NETIF_F_GSO;
 459 }
 460 EXPORT_SYMBOL(tcp_init_sock);
 461
 462 static void tcp_tx_timestamp(struct sock *sk, u16 tsflags)
 463 {
 464         struct sk_buff *skb = tcp_write_queue_tail(sk);
 465
 466         if (tsflags && skb) {
 467                 struct skb_shared_info *shinfo = skb_shinfo(skb);
 468                 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 469
 470                 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags);
 471                 if (tsflags & SOF_TIMESTAMPING_TX_ACK)
 472                         tcb->txstamp_ack = 1;
 473                 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
 474                         shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
 475         }
 476 }
 477
 478 static inline bool tcp_stream_is_readable(const struct tcp_sock *tp,
 479                                           int target, struct sock *sk)
 480 {
 481         int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
 482
 483         if (avail > 0) {
 484                 if (avail >= target)
 485                         return true;
 486                 if (tcp_rmem_pressure(sk))
 487                         return true;
 488         }
 489         if (sk->sk_prot->stream_memory_read)
 490                 return sk->sk_prot->stream_memory_read(sk);
 491         return false;
 492 }
 493
 494 /*
 495  *      Wait for a TCP event.
 496  *
 497  *      Note that we don't need to lock the socket, as the upper poll layers
 498  *      take care of normal races (between the test and the event) and we don't
 499  *      go look at any of the socket buffers directly.
 500  */
 501 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 502 {
 503         __poll_t mask;
 504         struct sock *sk = sock->sk;
 505         const struct tcp_sock *tp = tcp_sk(sk);
 506         int state;
 507
 508         sock_poll_wait(file, sock, wait);
 509
 510         state = inet_sk_state_load(sk);
 511         if (state == TCP_LISTEN)
 512                 return inet_csk_listen_poll(sk);
 513
 514         /* Socket is not locked. We are protected from async events
 515          * by poll logic and correct handling of state changes
 516          * made by other threads is impossible in any case.
 517          */
 518
 519         mask = 0;
 520
 521         /*
 522          * EPOLLHUP is certainly not done right. But poll() doesn't
 523          * have a notion of HUP in just one direction, and for a
 524          * socket the read side is more interesting.
 525          *
 526          * Some poll() documentation says that EPOLLHUP is incompatible
 527          * with the EPOLLOUT/POLLWR flags, so somebody should check this
 528          * all. But careful, it tends to be safer to return too many
 529          * bits than too few, and you can easily break real applications
 530          * if you don't tell them that something has hung up!
 531          *
 532          * Check-me.
 533          *
 534          * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and
 535          * our fs/select.c). It means that after we received EOF,
 536          * poll always returns immediately, making impossible poll() on write()
 537          * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP
 538          * if and only if shutdown has been made in both directions.
 539          * Actually, it is interesting to look how Solaris and DUX
 540          * solve this dilemma. I would prefer, if EPOLLHUP were maskable,
 541          * then we could set it on SND_SHUTDOWN. BTW examples given
 542          * in Stevens' books assume exactly this behaviour, it explains
 543          * why EPOLLHUP is incompatible with EPOLLOUT.  --ANK
 544          *
 545          * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 546          * blocking on fresh not-connected or disconnected socket. --ANK
 547          */
 548         if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
 549                 mask |= EPOLLHUP;
 550         if (sk->sk_shutdown & RCV_SHUTDOWN)
 551                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
 552
 553         /* Connected or passive Fast Open socket? */
 554         if (state != TCP_SYN_SENT &&
 555             (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
 556                 int target = sock_rcvlowat(sk, 0, INT_MAX);
 557
 558                 if (READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
 559                     !sock_flag(sk, SOCK_URGINLINE) &&
 560                     tp->urg_data)
 561                         target++;
 562
 563                 if (tcp_stream_is_readable(tp, target, sk))
 564                         mask |= EPOLLIN | EPOLLRDNORM;
 565
 566                 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 567                         if (__sk_stream_is_writeable(sk, 1)) {
 568                                 mask |= EPOLLOUT | EPOLLWRNORM;
 569                         } else {  /* send SIGIO later */
 570                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 571                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 572
 573                                 /* Race breaker. If space is freed after
 574                                  * wspace test but before the flags are set,
 575                                  * IO signal will be lost. Memory barrier
 576                                  * pairs with the input side.
 577                                  */
 578                                 smp_mb__after_atomic();
 579                                 if (__sk_stream_is_writeable(sk, 1))
 580                                         mask |= EPOLLOUT | EPOLLWRNORM;
 581                         }
 582                 } else
 583                         mask |= EPOLLOUT | EPOLLWRNORM;
 584
 585                 if (tp->urg_data & TCP_URG_VALID)
 586                         mask |= EPOLLPRI;
 587         } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) {
 588                 /* Active TCP fastopen socket with defer_connect
 589                  * Return EPOLLOUT so application can call write()
 590                  * in order for kernel to generate SYN+data
 591                  */
 592                 mask |= EPOLLOUT | EPOLLWRNORM;
 593         }
 594         /* This barrier is coupled with smp_wmb() in tcp_reset() */
 595         smp_rmb();
 596         if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue))
 597                 mask |= EPOLLERR;
 598
 599         return mask;
 600 }
 601 EXPORT_SYMBOL(tcp_poll);
 602
 603 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 604 {
 605         struct tcp_sock *tp = tcp_sk(sk);
 606         int answ;
 607         bool slow;
 608
 609         switch (cmd) {
 610         case SIOCINQ:
 611                 if (sk->sk_state == TCP_LISTEN)
 612                         return -EINVAL;
 613
 614                 slow = lock_sock_fast(sk);
 615                 answ = tcp_inq(sk);
 616                 unlock_sock_fast(sk, slow);
 617                 break;
 618         case SIOCATMARK:
 619                 answ = tp->urg_data &&
 620                        READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
 621                 break;
 622         case SIOCOUTQ:
 623                 if (sk->sk_state == TCP_LISTEN)
 624                         return -EINVAL;
 625
 626                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 627                         answ = 0;
 628                 else
 629                         answ = READ_ONCE(tp->write_seq) - tp->snd_una;
 630                 break;
 631         case SIOCOUTQNSD:
 632                 if (sk->sk_state == TCP_LISTEN)
 633                         return -EINVAL;
 634
 635                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 636                         answ = 0;
 637                 else
 638                         answ = READ_ONCE(tp->write_seq) -
 639                                READ_ONCE(tp->snd_nxt);
 640                 break;
 641         default:
 642                 return -ENOIOCTLCMD;
 643         }
 644
 645         return put_user(answ, (int __user *)arg);
 646 }
 647 EXPORT_SYMBOL(tcp_ioctl);
 648
 649 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 650 {
 651         TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
 652         tp->pushed_seq = tp->write_seq;
 653 }
 654
 655 static inline bool forced_push(const struct tcp_sock *tp)
 656 {
 657         return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 658 }
 659
 660 static void skb_entail(struct sock *sk, struct sk_buff *skb)
 661 {
 662         struct tcp_sock *tp = tcp_sk(sk);
 663         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 664
 665         skb->csum    = 0;
 666         tcb->seq     = tcb->end_seq = tp->write_seq;
 667         tcb->tcp_flags = TCPHDR_ACK;
 668         tcb->sacked  = 0;
 669         __skb_header_release(skb);
 670         tcp_add_write_queue_tail(sk, skb);
 671         sk_wmem_queued_add(sk, skb->truesize);
 672         sk_mem_charge(sk, skb->truesize);
 673         if (tp->nonagle & TCP_NAGLE_PUSH)
 674                 tp->nonagle &= ~TCP_NAGLE_PUSH;
 675
 676         tcp_slow_start_after_idle_check(sk);
 677 }
 678
 679 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
 680 {
 681         if (flags & MSG_OOB)
 682                 tp->snd_up = tp->write_seq;
 683 }
 684
 685 /* If a not yet filled skb is pushed, do not send it if
 686  * we have data packets in Qdisc or NIC queues :
 687  * Because TX completion will happen shortly, it gives a chance
 688  * to coalesce future sendmsg() payload into this skb, without
 689  * need for a timer, and with no latency trade off.
 690  * As packets containing data payload have a bigger truesize
 691  * than pure acks (dataless) packets, the last checks prevent
 692  * autocorking if we only have an ACK in Qdisc/NIC queues,
 693  * or if TX completion was delayed after we processed ACK packet.
 694  */
 695 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
 696                                 int size_goal)
 697 {
 698         return skb->len < size_goal &&
 699                sock_net(sk)->ipv4.sysctl_tcp_autocorking &&
 700                !tcp_rtx_queue_empty(sk) &&
 701                refcount_read(&sk->sk_wmem_alloc) > skb->truesize;
 702 }
 703
 704 void tcp_push(struct sock *sk, int flags, int mss_now,
 705               int nonagle, int size_goal)
 706 {
 707         struct tcp_sock *tp = tcp_sk(sk);
 708         struct sk_buff *skb;
 709
 710         skb = tcp_write_queue_tail(sk);
 711         if (!skb)
 712                 return;
 713         if (!(flags & MSG_MORE) || forced_push(tp))
 714                 tcp_mark_push(tp, skb);
 715
 716         tcp_mark_urg(tp, flags);
 717
 718         if (tcp_should_autocork(sk, skb, size_goal)) {
 719
 720                 /* avoid atomic op if TSQ_THROTTLED bit is already set */
 721                 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
 722                         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
 723                         set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
 724                 }
 725                 /* It is possible TX completion already happened
 726                  * before we set TSQ_THROTTLED.
 727                  */
 728                 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize)
 729                         return;
 730         }
 731
 732         if (flags & MSG_MORE)
 733                 nonagle = TCP_NAGLE_CORK;
 734
 735         __tcp_push_pending_frames(sk, mss_now, nonagle);
 736 }
 737
 738 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
 739                                 unsigned int offset, size_t len)
 740 {
 741         struct tcp_splice_state *tss = rd_desc->arg.data;
 742         int ret;
 743
 744         ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
 745                               min(rd_desc->count, len), tss->flags);
 746         if (ret > 0)
 747                 rd_desc->count -= ret;
 748         return ret;
 749 }
 750
 751 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
 752 {
 753         /* Store TCP splice context information in read_descriptor_t. */
 754         read_descriptor_t rd_desc = {
 755                 .arg.data = tss,
 756                 .count    = tss->len,
 757         };
 758
 759         return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
 760 }
 761
 762 /**
 763  *  tcp_splice_read - splice data from TCP socket to a pipe
 764  * @sock:       socket to splice from
 765  * @ppos:       position (not valid)
 766  * @pipe:       pipe to splice to
 767  * @len:        number of bytes to splice
 768  * @flags:      splice modifier flags
 769  *
 770  * Description:
 771  *    Will read pages from given socket and fill them into a pipe.
 772  *
 773  **/
 774 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 775                         struct pipe_inode_info *pipe, size_t len,
 776                         unsigned int flags)
 777 {
 778         struct sock *sk = sock->sk;
 779         struct tcp_splice_state tss = {
 780                 .pipe = pipe,
 781                 .len = len,
 782                 .flags = flags,
 783         };
 784         long timeo;
 785         ssize_t spliced;
 786         int ret;
 787
 788         sock_rps_record_flow(sk);
 789         /*
 790          * We can't seek on a socket input
 791          */
 792         if (unlikely(*ppos))
 793                 return -ESPIPE;
 794
 795         ret = spliced = 0;
 796
 797         lock_sock(sk);
 798
 799         timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
 800         while (tss.len) {
 801                 ret = __tcp_splice_read(sk, &tss);
 802                 if (ret < 0)
 803                         break;
 804                 else if (!ret) {
 805                         if (spliced)
 806                                 break;
 807                         if (sock_flag(sk, SOCK_DONE))
 808                                 break;
 809                         if (sk->sk_err) {
 810                                 ret = sock_error(sk);
 811                                 break;
 812                         }
 813                         if (sk->sk_shutdown & RCV_SHUTDOWN)
 814                                 break;
 815                         if (sk->sk_state == TCP_CLOSE) {
 816                                 /*
 817                                  * This occurs when user tries to read
 818                                  * from never connected socket.
 819                                  */
 820                                 ret = -ENOTCONN;
 821                                 break;
 822                         }
 823                         if (!timeo) {
 824                                 ret = -EAGAIN;
 825                                 break;
 826                         }
 827                         /* if __tcp_splice_read() got nothing while we have
 828                          * an skb in receive queue, we do not want to loop.
 829                          * This might happen with URG data.
 830                          */
 831                         if (!skb_queue_empty(&sk->sk_receive_queue))
 832                                 break;
 833                         sk_wait_data(sk, &timeo, NULL);
 834                         if (signal_pending(current)) {
 835                                 ret = sock_intr_errno(timeo);
 836                                 break;
 837                         }
 838                         continue;
 839                 }
 840                 tss.len -= ret;
 841                 spliced += ret;
 842
 843                 if (!timeo)
 844                         break;
 845                 release_sock(sk);
 846                 lock_sock(sk);
 847
 848                 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
 849                     (sk->sk_shutdown & RCV_SHUTDOWN) ||
 850                     signal_pending(current))
 851                         break;
 852         }
 853
 854         release_sock(sk);
 855
 856         if (spliced)
 857                 return spliced;
 858
 859         return ret;
 860 }
 861 EXPORT_SYMBOL(tcp_splice_read);
 862
 863 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
 864                                     bool force_schedule)
 865 {
 866         struct sk_buff *skb;
 867
 868         if (likely(!size)) {
 869                 skb = sk->sk_tx_skb_cache;
 870                 if (skb) {
 871                         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
 872                         sk->sk_tx_skb_cache = NULL;
 873                         pskb_trim(skb, 0);
 874                         INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
 875                         skb_shinfo(skb)->tx_flags = 0;
 876                         memset(TCP_SKB_CB(skb), 0, sizeof(struct tcp_skb_cb));
 877                         return skb;
 878                 }
 879         }
 880         /* The TCP header must be at least 32-bit aligned.  */
 881         size = ALIGN(size, 4);
 882
 883         if (unlikely(tcp_under_memory_pressure(sk)))
 884                 sk_mem_reclaim_partial(sk);
 885
 886         skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
 887         if (likely(skb)) {
 888                 bool mem_scheduled;
 889
 890                 if (force_schedule) {
 891                         mem_scheduled = true;
 892                         sk_forced_mem_schedule(sk, skb->truesize);
 893                 } else {
 894                         mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
 895                 }
 896                 if (likely(mem_scheduled)) {
 897                         skb_reserve(skb, sk->sk_prot->max_header);
 898                         /*
 899                          * Make sure that we have exactly size bytes
 900                          * available to the caller, no more, no less.
 901                          */
 902                         skb->reserved_tailroom = skb->end - skb->tail - size;
 903                         INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
 904                         return skb;
 905                 }
 906                 __kfree_skb(skb);
 907         } else {
 908                 sk->sk_prot->enter_memory_pressure(sk);
 909                 sk_stream_moderate_sndbuf(sk);
 910         }
 911         return NULL;
 912 }
 913
 914 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
 915                                        int large_allowed)
 916 {
 917         struct tcp_sock *tp = tcp_sk(sk);
 918         u32 new_size_goal, size_goal;
 919
 920         if (!large_allowed)
 921                 return mss_now;
 922
 923         /* Note : tcp_tso_autosize() will eventually split this later */
 924         new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER;
 925         new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);
 926
 927         /* We try hard to avoid divides here */
 928         size_goal = tp->gso_segs * mss_now;
 929         if (unlikely(new_size_goal < size_goal ||
 930                      new_size_goal >= size_goal + mss_now)) {
 931                 tp->gso_segs = min_t(u16, new_size_goal / mss_now,
 932                                      sk->sk_gso_max_segs);
 933                 size_goal = tp->gso_segs * mss_now;
 934         }
 935
 936         return max(size_goal, mss_now);
 937 }
 938
 939 int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
 940 {
 941         int mss_now;
 942
 943         mss_now = tcp_current_mss(sk);
 944         *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
 945
 946         return mss_now;
 947 }
 948
 949 /* In some cases, both sendpage() and sendmsg() could have added
 950  * an skb to the write queue, but failed adding payload on it.
 951  * We need to remove it to consume less memory, but more
 952  * importantly be able to generate EPOLLOUT for Edge Trigger epoll()
 953  * users.
 954  */
 955 static void tcp_remove_empty_skb(struct sock *sk, struct sk_buff *skb)
 956 {
 957         if (skb && !skb->len) {
 958                 tcp_unlink_write_queue(skb, sk);
 959                 if (tcp_write_queue_empty(sk))
 960                         tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
 961                 sk_wmem_free_skb(sk, skb);
 962         }
 963 }
 964
 965 ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 966                          size_t size, int flags)
 967 {
 968         struct tcp_sock *tp = tcp_sk(sk);
 969         int mss_now, size_goal;
 970         int err;
 971         ssize_t copied;
 972         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 973
 974         if (IS_ENABLED(CONFIG_DEBUG_VM) &&
 975             WARN_ONCE(PageSlab(page), "page must not be a Slab one"))
 976                 return -EINVAL;
 977
 978         /* Wait for a connection to finish. One exception is TCP Fast Open
 979          * (passive side) where data is allowed to be sent before a connection
 980          * is fully established.
 981          */
 982         if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
 983             !tcp_passive_fastopen(sk)) {
 984                 err = sk_stream_wait_connect(sk, &timeo);
 985                 if (err != 0)
 986                         goto out_err;
 987         }
 988
 989         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 990
 991         mss_now = tcp_send_mss(sk, &size_goal, flags);
 992         copied = 0;
 993
 994         err = -EPIPE;
 995         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 996                 goto out_err;
 997
 998         while (size > 0) {
 999                 struct sk_buff *skb = tcp_write_queue_tail(sk);
1000                 int copy, i;
1001                 bool can_coalesce;
1002
1003                 if (!skb || (copy = size_goal - skb->len) <= 0 ||
1004                     !tcp_skb_can_collapse_to(skb)) {
1005 new_segment:
1006                         if (!sk_stream_memory_free(sk))
1007                                 goto wait_for_space;
1008
1009                         skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1010                                         tcp_rtx_and_write_queues_empty(sk));
1011                         if (!skb)
1012                                 goto wait_for_space;
1013
1014 #ifdef CONFIG_TLS_DEVICE
1015                         skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
1016 #endif
1017                         skb_entail(sk, skb);
1018                         copy = size_goal;
1019                 }
1020
1021                 if (copy > size)
1022                         copy = size;
1023
1024                 i = skb_shinfo(skb)->nr_frags;
1025                 can_coalesce = skb_can_coalesce(skb, i, page, offset);
1026                 if (!can_coalesce && i >= sysctl_max_skb_frags) {
1027                         tcp_mark_push(tp, skb);
1028                         goto new_segment;
1029                 }
1030                 if (!sk_wmem_schedule(sk, copy))
1031                         goto wait_for_space;
1032
1033                 if (can_coalesce) {
1034                         skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1035                 } else {
1036                         get_page(page);
1037                         skb_fill_page_desc(skb, i, page, offset, copy);
1038                 }
1039
1040                 if (!(flags & MSG_NO_SHARED_FRAGS))
1041                         skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
1042
1043                 skb->len += copy;
1044                 skb->data_len += copy;
1045                 skb->truesize += copy;
1046                 sk_wmem_queued_add(sk, copy);
1047                 sk_mem_charge(sk, copy);
1048                 skb->ip_summed = CHECKSUM_PARTIAL;
1049                 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1050                 TCP_SKB_CB(skb)->end_seq += copy;
1051                 tcp_skb_pcount_set(skb, 0);
1052
1053                 if (!copied)
1054                         TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1055
1056                 copied += copy;
1057                 offset += copy;
1058                 size -= copy;
1059                 if (!size)
1060                         goto out;
1061
1062                 if (skb->len < size_goal || (flags & MSG_OOB))
1063                         continue;
1064
1065                 if (forced_push(tp)) {
1066                         tcp_mark_push(tp, skb);
1067                         __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1068                 } else if (skb == tcp_send_head(sk))
1069                         tcp_push_one(sk, mss_now);
1070                 continue;
1071
1072 wait_for_space:
1073                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1074                 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1075                          TCP_NAGLE_PUSH, size_goal);
1076
1077                 err = sk_stream_wait_memory(sk, &timeo);
1078                 if (err != 0)
1079                         goto do_error;
1080
1081                 mss_now = tcp_send_mss(sk, &size_goal, flags);
1082         }
1083
1084 out:
1085         if (copied) {
1086                 tcp_tx_timestamp(sk, sk->sk_tsflags);
1087                 if (!(flags & MSG_SENDPAGE_NOTLAST))
1088                         tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1089         }
1090         return copied;
1091
1092 do_error:
1093         tcp_remove_empty_skb(sk, tcp_write_queue_tail(sk));
1094         if (copied)
1095                 goto out;
1096 out_err:
1097         /* make sure we wake any epoll edge trigger waiter */
1098         if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1099                 sk->sk_write_space(sk);
1100                 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1101         }
1102         return sk_stream_error(sk, flags, err);
1103 }
1104 EXPORT_SYMBOL_GPL(do_tcp_sendpages);
1105
1106 int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
1107                         size_t size, int flags)
1108 {
1109         if (!(sk->sk_route_caps & NETIF_F_SG))
1110                 return sock_no_sendpage_locked(sk, page, offset, size, flags);
1111
1112         tcp_rate_check_app_limited(sk);  /* is sending application-limited? */
1113
1114         return do_tcp_sendpages(sk, page, offset, size, flags);
1115 }
1116 EXPORT_SYMBOL_GPL(tcp_sendpage_locked);
1117
1118 int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1119                  size_t size, int flags)
1120 {
1121         int ret;
1122
1123         lock_sock(sk);
1124         ret = tcp_sendpage_locked(sk, page, offset, size, flags);
1125         release_sock(sk);
1126
1127         return ret;
1128 }
1129 EXPORT_SYMBOL(tcp_sendpage);
1130
1131 void tcp_free_fastopen_req(struct tcp_sock *tp)
1132 {
1133         if (tp->fastopen_req) {
1134                 kfree(tp->fastopen_req);
1135                 tp->fastopen_req = NULL;
1136         }
1137 }
1138
1139 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1140                                 int *copied, size_t size,
1141                                 struct ubuf_info *uarg)
1142 {
1143         struct tcp_sock *tp = tcp_sk(sk);
1144         struct inet_sock *inet = inet_sk(sk);
1145         struct sockaddr *uaddr = msg->msg_name;
1146         int err, flags;
1147
1148         if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) ||
1149             (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1150              uaddr->sa_family == AF_UNSPEC))
1151                 return -EOPNOTSUPP;
1152         if (tp->fastopen_req)
1153                 return -EALREADY; /* Another Fast Open is in progress */
1154
1155         tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1156                                    sk->sk_allocation);
1157         if (unlikely(!tp->fastopen_req))
1158                 return -ENOBUFS;
1159         tp->fastopen_req->data = msg;
1160         tp->fastopen_req->size = size;
1161         tp->fastopen_req->uarg = uarg;
1162
1163         if (inet->defer_connect) {
1164                 err = tcp_connect(sk);
1165                 /* Same failure procedure as in tcp_v4/6_connect */
1166                 if (err) {
1167                         tcp_set_state(sk, TCP_CLOSE);
1168                         inet->inet_dport = 0;
1169                         sk->sk_route_caps = 0;
1170                 }
1171         }
1172         flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1173         err = __inet_stream_connect(sk->sk_socket, uaddr,
1174                                     msg->msg_namelen, flags, 1);
1175         /* fastopen_req could already be freed in __inet_stream_connect
1176          * if the connection times out or gets rst
1177          */
1178         if (tp->fastopen_req) {
1179                 *copied = tp->fastopen_req->copied;
1180                 tcp_free_fastopen_req(tp);
1181                 inet->defer_connect = 0;
1182         }
1183         return err;
1184 }
1185
1186 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
1187 {
1188         struct tcp_sock *tp = tcp_sk(sk);
1189         struct ubuf_info *uarg = NULL;
1190         struct sk_buff *skb;
1191         struct sockcm_cookie sockc;
1192         int flags, err, copied = 0;
1193         int mss_now = 0, size_goal, copied_syn = 0;
1194         int process_backlog = 0;
1195         bool zc = false;
1196         long timeo;
1197
1198         flags = msg->msg_flags;
1199
1200         if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) {
1201                 skb = tcp_write_queue_tail(sk);
1202                 uarg = sock_zerocopy_realloc(sk, size, skb_zcopy(skb));
1203                 if (!uarg) {
1204                         err = -ENOBUFS;
1205                         goto out_err;
1206                 }
1207
1208                 zc = sk->sk_route_caps & NETIF_F_SG;
1209                 if (!zc)
1210                         uarg->zerocopy = 0;
1211         }
1212
1213         if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) &&
1214             !tp->repair) {
1215                 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1216                 if (err == -EINPROGRESS && copied_syn > 0)
1217                         goto out;
1218                 else if (err)
1219                         goto out_err;
1220         }
1221
1222         timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1223
1224         tcp_rate_check_app_limited(sk);  /* is sending application-limited? */
1225
1226         /* Wait for a connection to finish. One exception is TCP Fast Open
1227          * (passive side) where data is allowed to be sent before a connection
1228          * is fully established.
1229          */
1230         if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1231             !tcp_passive_fastopen(sk)) {
1232                 err = sk_stream_wait_connect(sk, &timeo);
1233                 if (err != 0)
1234                         goto do_error;
1235         }
1236
1237         if (unlikely(tp->repair)) {
1238                 if (tp->repair_queue == TCP_RECV_QUEUE) {
1239                         copied = tcp_send_rcvq(sk, msg, size);
1240                         goto out_nopush;
1241                 }
1242
1243                 err = -EINVAL;
1244                 if (tp->repair_queue == TCP_NO_QUEUE)
1245                         goto out_err;
1246
1247                 /* 'common' sending to sendq */
1248         }
1249
1250         sockcm_init(&sockc, sk);
1251         if (msg->msg_controllen) {
1252                 err = sock_cmsg_send(sk, msg, &sockc);
1253                 if (unlikely(err)) {
1254                         err = -EINVAL;
1255                         goto out_err;
1256                 }
1257         }
1258
1259         /* This should be in poll */
1260         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1261
1262         /* Ok commence sending. */
1263         copied = 0;
1264
1265 restart:
1266         mss_now = tcp_send_mss(sk, &size_goal, flags);
1267
1268         err = -EPIPE;
1269         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1270                 goto do_error;
1271
1272         while (msg_data_left(msg)) {
1273                 int copy = 0;
1274
1275                 skb = tcp_write_queue_tail(sk);
1276                 if (skb)
1277                         copy = size_goal - skb->len;
1278
1279                 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1280                         bool first_skb;
1281
1282 new_segment:
1283                         if (!sk_stream_memory_free(sk))
1284                                 goto wait_for_space;
1285
1286                         if (unlikely(process_backlog >= 16)) {
1287                                 process_backlog = 0;
1288                                 if (sk_flush_backlog(sk))
1289                                         goto restart;
1290                         }
1291                         first_skb = tcp_rtx_and_write_queues_empty(sk);
1292                         skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1293                                                   first_skb);
1294                         if (!skb)
1295                                 goto wait_for_space;
1296
1297                         process_backlog++;
1298                         skb->ip_summed = CHECKSUM_PARTIAL;
1299
1300                         skb_entail(sk, skb);
1301                         copy = size_goal;
1302
1303                         /* All packets are restored as if they have
1304                          * already been sent. skb_mstamp_ns isn't set to
1305                          * avoid wrong rtt estimation.
1306                          */
1307                         if (tp->repair)
1308                                 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1309                 }
1310
1311                 /* Try to append data to the end of skb. */
1312                 if (copy > msg_data_left(msg))
1313                         copy = msg_data_left(msg);
1314
1315                 /* Where to copy to? */
1316                 if (skb_availroom(skb) > 0 && !zc) {
1317                         /* We have some space in skb head. Superb! */
1318                         copy = min_t(int, copy, skb_availroom(skb));
1319                         err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy);
1320                         if (err)
1321                                 goto do_fault;
1322                 } else if (!zc) {
1323                         bool merge = true;
1324                         int i = skb_shinfo(skb)->nr_frags;
1325                         struct page_frag *pfrag = sk_page_frag(sk);
1326
1327                         if (!sk_page_frag_refill(sk, pfrag))
1328                                 goto wait_for_space;
1329
1330                         if (!skb_can_coalesce(skb, i, pfrag->page,
1331                                               pfrag->offset)) {
1332                                 if (i >= sysctl_max_skb_frags) {
1333                                         tcp_mark_push(tp, skb);
1334                                         goto new_segment;
1335                                 }
1336                                 merge = false;
1337                         }
1338
1339                         copy = min_t(int, copy, pfrag->size - pfrag->offset);
1340
1341                         if (!sk_wmem_schedule(sk, copy))
1342                                 goto wait_for_space;
1343
1344                         err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1345                                                        pfrag->page,
1346                                                        pfrag->offset,
1347                                                        copy);
1348                         if (err)
1349                                 goto do_error;
1350
1351                         /* Update the skb. */
1352                         if (merge) {
1353                                 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1354                         } else {
1355                                 skb_fill_page_desc(skb, i, pfrag->page,
1356                                                    pfrag->offset, copy);
1357                                 page_ref_inc(pfrag->page);
1358                         }
1359                         pfrag->offset += copy;
1360                 } else {
1361                         err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
1362                         if (err == -EMSGSIZE || err == -EEXIST) {
1363                                 tcp_mark_push(tp, skb);
1364                                 goto new_segment;
1365                         }
1366                         if (err < 0)
1367                                 goto do_error;
1368                         copy = err;
1369                 }
1370
1371                 if (!copied)
1372                         TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1373
1374                 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1375                 TCP_SKB_CB(skb)->end_seq += copy;
1376                 tcp_skb_pcount_set(skb, 0);
1377
1378                 copied += copy;
1379                 if (!msg_data_left(msg)) {
1380                         if (unlikely(flags & MSG_EOR))
1381                                 TCP_SKB_CB(skb)->eor = 1;
1382                         goto out;
1383                 }
1384
1385                 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
1386                         continue;
1387
1388                 if (forced_push(tp)) {
1389                         tcp_mark_push(tp, skb);
1390                         __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1391                 } else if (skb == tcp_send_head(sk))
1392                         tcp_push_one(sk, mss_now);
1393                 continue;
1394
1395 wait_for_space:
1396                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1397                 if (copied)
1398                         tcp_push(sk, flags & ~MSG_MORE, mss_now,
1399                                  TCP_NAGLE_PUSH, size_goal);
1400
1401                 err = sk_stream_wait_memory(sk, &timeo);
1402                 if (err != 0)
1403                         goto do_error;
1404
1405                 mss_now = tcp_send_mss(sk, &size_goal, flags);
1406         }
1407
1408 out:
1409         if (copied) {
1410                 tcp_tx_timestamp(sk, sockc.tsflags);
1411                 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1412         }
1413 out_nopush:
1414         sock_zerocopy_put(uarg);
1415         return copied + copied_syn;
1416
1417 do_error:
1418         skb = tcp_write_queue_tail(sk);
1419 do_fault:
1420         tcp_remove_empty_skb(sk, skb);
1421
1422         if (copied + copied_syn)
1423                 goto out;
1424 out_err:
1425         sock_zerocopy_put_abort(uarg, true);
1426         err = sk_stream_error(sk, flags, err);
1427         /* make sure we wake any epoll edge trigger waiter */
1428         if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1429                 sk->sk_write_space(sk);
1430                 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1431         }
1432         return err;
1433 }
1434 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
1435
1436 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1437 {
1438         int ret;
1439
1440         lock_sock(sk);
1441         ret = tcp_sendmsg_locked(sk, msg, size);
1442         release_sock(sk);
1443
1444         return ret;
1445 }
1446 EXPORT_SYMBOL(tcp_sendmsg);
1447
1448 /*
1449  *      Handle reading urgent data. BSD has very simple semantics for
1450  *      this, no blocking and very strange errors 8)
1451  */
1452
1453 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1454 {
1455         struct tcp_sock *tp = tcp_sk(sk);
1456
1457         /* No URG data to read. */
1458         if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1459             tp->urg_data == TCP_URG_READ)
1460                 return -EINVAL; /* Yes this is right ! */
1461
1462         if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1463                 return -ENOTCONN;
1464
1465         if (tp->urg_data & TCP_URG_VALID) {
1466                 int err = 0;
1467                 char c = tp->urg_data;
1468
1469                 if (!(flags & MSG_PEEK))
1470                         tp->urg_data = TCP_URG_READ;
1471
1472                 /* Read urgent data. */
1473                 msg->msg_flags |= MSG_OOB;
1474
1475                 if (len > 0) {
1476                         if (!(flags & MSG_TRUNC))
1477                                 err = memcpy_to_msg(msg, &c, 1);
1478                         len = 1;
1479                 } else
1480                         msg->msg_flags |= MSG_TRUNC;
1481
1482                 return err ? -EFAULT : len;
1483         }
1484
1485         if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1486                 return 0;
1487
1488         /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1489          * the available implementations agree in this case:
1490          * this call should never block, independent of the
1491          * blocking state of the socket.
1492          * Mike <pall@rz.uni-karlsruhe.de>
1493          */
1494         return -EAGAIN;
1495 }
1496
1497 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1498 {
1499         struct sk_buff *skb;
1500         int copied = 0, err = 0;
1501
1502         /* XXX -- need to support SO_PEEK_OFF */
1503
1504         skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
1505                 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1506                 if (err)
1507                         return err;
1508                 copied += skb->len;
1509         }
1510
1511         skb_queue_walk(&sk->sk_write_queue, skb) {
1512                 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1513                 if (err)
1514                         break;
1515
1516                 copied += skb->len;
1517         }
1518
1519         return err ?: copied;
1520 }
1521
1522 /* Clean up the receive buffer for full frames taken by the user,
1523  * then send an ACK if necessary.  COPIED is the number of bytes
1524  * tcp_recvmsg has given to the user so far, it speeds up the
1525  * calculation of whether or not we must ACK for the sake of
1526  * a window update.
1527  */
1528 void tcp_cleanup_rbuf(struct sock *sk, int copied)
1529 {
1530         struct tcp_sock *tp = tcp_sk(sk);
1531         bool time_to_ack = false;
1532
1533         struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1534
1535         WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1536              "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1537              tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1538
1539         if (inet_csk_ack_scheduled(sk)) {
1540                 const struct inet_connection_sock *icsk = inet_csk(sk);
1541                    /* Delayed ACKs frequently hit locked sockets during bulk
1542                     * receive. */
1543                 if (icsk->icsk_ack.blocked ||
1544                     /* Once-per-two-segments ACK was not sent by tcp_input.c */
1545                     tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1546                     /*
1547                      * If this read emptied read buffer, we send ACK, if
1548                      * connection is not bidirectional, user drained
1549                      * receive buffer and there was a small segment
1550                      * in queue.
1551                      */
1552                     (copied > 0 &&
1553                      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1554                       ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1555                        !inet_csk_in_pingpong_mode(sk))) &&
1556                       !atomic_read(&sk->sk_rmem_alloc)))
1557                         time_to_ack = true;
1558         }
1559
1560         /* We send an ACK if we can now advertise a non-zero window
1561          * which has been raised "significantly".
1562          *
1563          * Even if window raised up to infinity, do not send window open ACK
1564          * in states, where we will not receive more. It is useless.
1565          */
1566         if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1567                 __u32 rcv_window_now = tcp_receive_window(tp);
1568
1569                 /* Optimize, __tcp_select_window() is not cheap. */
1570                 if (2*rcv_window_now <= tp->window_clamp) {
1571                         __u32 new_window = __tcp_select_window(sk);
1572
1573                         /* Send ACK now, if this read freed lots of space
1574                          * in our buffer. Certainly, new_window is new window.
1575                          * We can advertise it now, if it is not less than current one.
1576                          * "Lots" means "at least twice" here.
1577                          */
1578                         if (new_window && new_window >= 2 * rcv_window_now)
1579                                 time_to_ack = true;
1580                 }
1581         }
1582         if (time_to_ack)
1583                 tcp_send_ack(sk);
1584 }
1585
1586 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1587 {
1588         struct sk_buff *skb;
1589         u32 offset;
1590
1591         while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1592                 offset = seq - TCP_SKB_CB(skb)->seq;
1593                 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1594                         pr_err_once("%s: found a SYN, please report !\n", __func__);
1595                         offset--;
1596                 }
1597                 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1598                         *off = offset;
1599                         return skb;
1600                 }
1601                 /* This looks weird, but this can happen if TCP collapsing
1602                  * splitted a fat GRO packet, while we released socket lock
1603                  * in skb_splice_bits()
1604                  */
1605                 sk_eat_skb(sk, skb);
1606         }
1607         return NULL;
1608 }
1609
1610 /*
1611  * This routine provides an alternative to tcp_recvmsg() for routines
1612  * that would like to handle copying from skbuffs directly in 'sendfile'
1613  * fashion.
1614  * Note:
1615  *      - It is assumed that the socket was locked by the caller.
1616  *      - The routine does not block.
1617  *      - At present, there is no support for reading OOB data
1618  *        or for 'peeking' the socket using this routine
1619  *        (although both would be easy to implement).
1620  */
1621 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1622                   sk_read_actor_t recv_actor)
1623 {
1624         struct sk_buff *skb;
1625         struct tcp_sock *tp = tcp_sk(sk);
1626         u32 seq = tp->copied_seq;
1627         u32 offset;
1628         int copied = 0;
1629
1630         if (sk->sk_state == TCP_LISTEN)
1631                 return -ENOTCONN;
1632         while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1633                 if (offset < skb->len) {
1634                         int used;
1635                         size_t len;
1636
1637                         len = skb->len - offset;
1638                         /* Stop reading if we hit a patch of urgent data */
1639                         if (tp->urg_data) {
1640                                 u32 urg_offset = tp->urg_seq - seq;
1641                                 if (urg_offset < len)
1642                                         len = urg_offset;
1643                                 if (!len)
1644                                         break;
1645                         }
1646                         used = recv_actor(desc, skb, offset, len);
1647                         if (used <= 0) {
1648                                 if (!copied)
1649                                         copied = used;
1650                                 break;
1651                         } else if (used <= len) {
1652                                 seq += used;
1653                                 copied += used;
1654                                 offset += used;
1655                         }
1656                         /* If recv_actor drops the lock (e.g. TCP splice
1657                          * receive) the skb pointer might be invalid when
1658                          * getting here: tcp_collapse might have deleted it
1659                          * while aggregating skbs from the socket queue.
1660                          */
1661                         skb = tcp_recv_skb(sk, seq - 1, &offset);
1662                         if (!skb)
1663                                 break;
1664                         /* TCP coalescing might have appended data to the skb.
1665                          * Try to splice more frags
1666                          */
1667                         if (offset + 1 != skb->len)
1668                                 continue;
1669                 }
1670                 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1671                         sk_eat_skb(sk, skb);
1672                         ++seq;
1673                         break;
1674                 }
1675                 sk_eat_skb(sk, skb);
1676                 if (!desc->count)
1677                         break;
1678                 WRITE_ONCE(tp->copied_seq, seq);
1679         }
1680         WRITE_ONCE(tp->copied_seq, seq);
1681
1682         tcp_rcv_space_adjust(sk);
1683
1684         /* Clean up data we have read: This will do ACK frames. */
1685         if (copied > 0) {
1686                 tcp_recv_skb(sk, seq, &offset);
1687                 tcp_cleanup_rbuf(sk, copied);
1688         }
1689         return copied;
1690 }
1691 EXPORT_SYMBOL(tcp_read_sock);
1692
1693 int tcp_peek_len(struct socket *sock)
1694 {
1695         return tcp_inq(sock->sk);
1696 }
1697 EXPORT_SYMBOL(tcp_peek_len);
1698
1699 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */
1700 int tcp_set_rcvlowat(struct sock *sk, int val)
1701 {
1702         int cap;
1703
1704         if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1705                 cap = sk->sk_rcvbuf >> 1;
1706         else
1707                 cap = sock_net(sk)->ipv4.sysctl_tcp_rmem[2] >> 1;
1708         val = min(val, cap);
1709         WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1710
1711         /* Check if we need to signal EPOLLIN right now */
1712         tcp_data_ready(sk);
1713
1714         if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1715                 return 0;
1716
1717         val <<= 1;
1718         if (val > sk->sk_rcvbuf) {
1719                 WRITE_ONCE(sk->sk_rcvbuf, val);
1720                 tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val);
1721         }
1722         return 0;
1723 }
1724 EXPORT_SYMBOL(tcp_set_rcvlowat);
1725
1726 #ifdef CONFIG_MMU
1727 static const struct vm_operations_struct tcp_vm_ops = {
1728 };
1729
1730 int tcp_mmap(struct file *file, struct socket *sock,
1731              struct vm_area_struct *vma)
1732 {
1733         if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1734                 return -EPERM;
1735         vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC);
1736
1737         /* Instruct vm_insert_page() to not mmap_read_lock(mm) */
1738         vma->vm_flags |= VM_MIXEDMAP;
1739
1740         vma->vm_ops = &tcp_vm_ops;
1741         return 0;
1742 }
1743 EXPORT_SYMBOL(tcp_mmap);
1744
1745 static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
1746                                         struct page **pages,
1747                                         unsigned long pages_to_map,
1748                                         unsigned long *insert_addr,
1749                                         u32 *length_with_pending,
1750                                         u32 *seq,
1751                                         struct tcp_zerocopy_receive *zc)
1752 {
1753         unsigned long pages_remaining = pages_to_map;
1754         int bytes_mapped;
1755         int ret;
1756
1757         ret = vm_insert_pages(vma, *insert_addr, pages, &pages_remaining);
1758         bytes_mapped = PAGE_SIZE * (pages_to_map - pages_remaining);
1759         /* Even if vm_insert_pages fails, it may have partially succeeded in
1760          * mapping (some but not all of the pages).
1761          */
1762         *seq += bytes_mapped;
1763         *insert_addr += bytes_mapped;
1764         if (ret) {
1765                 /* But if vm_insert_pages did fail, we have to unroll some state
1766                  * we speculatively touched before.
1767                  */
1768                 const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
1769                 *length_with_pending -= bytes_not_mapped;
1770                 zc->recv_skip_hint += bytes_not_mapped;
1771         }
1772         return ret;
1773 }
1774
1775 static int tcp_zerocopy_receive(struct sock *sk,
1776                                 struct tcp_zerocopy_receive *zc)
1777 {
1778         unsigned long address = (unsigned long)zc->address;
1779         u32 length = 0, seq, offset, zap_len;
1780         #define PAGE_BATCH_SIZE 8
1781         struct page *pages[PAGE_BATCH_SIZE];
1782         const skb_frag_t *frags = NULL;
1783         struct vm_area_struct *vma;
1784         struct sk_buff *skb = NULL;
1785         unsigned long pg_idx = 0;
1786         unsigned long curr_addr;
1787         struct tcp_sock *tp;
1788         int inq;
1789         int ret;
1790
1791         if (address & (PAGE_SIZE - 1) || address != zc->address)
1792                 return -EINVAL;
1793
1794         if (sk->sk_state == TCP_LISTEN)
1795                 return -ENOTCONN;
1796
1797         sock_rps_record_flow(sk);
1798
1799         tp = tcp_sk(sk);
1800
1801         mmap_read_lock(current->mm);
1802
1803         vma = find_vma(current->mm, address);
1804         if (!vma || vma->vm_start > address || vma->vm_ops != &tcp_vm_ops) {
1805                 mmap_read_unlock(current->mm);
1806                 return -EINVAL;
1807         }
1808         zc->length = min_t(unsigned long, zc->length, vma->vm_end - address);
1809
1810         seq = tp->copied_seq;
1811         inq = tcp_inq(sk);
1812         zc->length = min_t(u32, zc->length, inq);
1813         zap_len = zc->length & ~(PAGE_SIZE - 1);
1814         if (zap_len) {
1815                 zap_page_range(vma, address, zap_len);
1816                 zc->recv_skip_hint = 0;
1817         } else {
1818                 zc->recv_skip_hint = zc->length;
1819         }
1820         ret = 0;
1821         curr_addr = address;
1822         while (length + PAGE_SIZE <= zc->length) {
1823                 if (zc->recv_skip_hint < PAGE_SIZE) {
1824                         /* If we're here, finish the current batch. */
1825                         if (pg_idx) {
1826                                 ret = tcp_zerocopy_vm_insert_batch(vma, pages,
1827                                                                    pg_idx,
1828                                                                    &curr_addr,
1829                                                                    &length,
1830                                                                    &seq, zc);
1831                                 if (ret)
1832                                         goto out;
1833                                 pg_idx = 0;
1834                         }
1835                         if (skb) {
1836                                 if (zc->recv_skip_hint > 0)
1837                                         break;
1838                                 skb = skb->next;
1839                                 offset = seq - TCP_SKB_CB(skb)->seq;
1840                         } else {
1841                                 skb = tcp_recv_skb(sk, seq, &offset);
1842                         }
1843                         zc->recv_skip_hint = skb->len - offset;
1844                         offset -= skb_headlen(skb);
1845                         if ((int)offset < 0 || skb_has_frag_list(skb))
1846                                 break;
1847                         frags = skb_shinfo(skb)->frags;
1848                         while (offset) {
1849                                 if (skb_frag_size(frags) > offset)
1850                                         goto out;
1851                                 offset -= skb_frag_size(frags);
1852                                 frags++;
1853                         }
1854                 }
1855                 if (skb_frag_size(frags) != PAGE_SIZE || skb_frag_off(frags)) {
1856                         int remaining = zc->recv_skip_hint;
1857
1858                         while (remaining && (skb_frag_size(frags) != PAGE_SIZE ||
1859                                              skb_frag_off(frags))) {
1860                                 remaining -= skb_frag_size(frags);
1861                                 frags++;
1862                         }
1863                         zc->recv_skip_hint -= remaining;
1864                         break;
1865                 }
1866                 pages[pg_idx] = skb_frag_page(frags);
1867                 pg_idx++;
1868                 length += PAGE_SIZE;
1869                 zc->recv_skip_hint -= PAGE_SIZE;
1870                 frags++;
1871                 if (pg_idx == PAGE_BATCH_SIZE) {
1872                         ret = tcp_zerocopy_vm_insert_batch(vma, pages, pg_idx,
1873                                                            &curr_addr, &length,
1874                                                            &seq, zc);
1875                         if (ret)
1876                                 goto out;
1877                         pg_idx = 0;
1878                 }
1879         }
1880         if (pg_idx) {
1881                 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pg_idx,
1882                                                    &curr_addr, &length, &seq,
1883                                                    zc);
1884         }
1885 out:
1886         mmap_read_unlock(current->mm);
1887         if (length) {
1888                 WRITE_ONCE(tp->copied_seq, seq);
1889                 tcp_rcv_space_adjust(sk);
1890
1891                 /* Clean up data we have read: This will do ACK frames. */
1892                 tcp_recv_skb(sk, seq, &offset);
1893                 tcp_cleanup_rbuf(sk, length);
1894                 ret = 0;
1895                 if (length == zc->length)
1896                         zc->recv_skip_hint = 0;
1897         } else {
1898                 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
1899                         ret = -EIO;
1900         }
1901         zc->length = length;
1902         return ret;
1903 }
1904 #endif
1905
1906 static void tcp_update_recv_tstamps(struct sk_buff *skb,
1907                                     struct scm_timestamping_internal *tss)
1908 {
1909         if (skb->tstamp)
1910                 tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1911         else
1912                 tss->ts[0] = (struct timespec64) {0};
1913
1914         if (skb_hwtstamps(skb)->hwtstamp)
1915                 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1916         else
1917                 tss->ts[2] = (struct timespec64) {0};
1918 }
1919
1920 /* Similar to __sock_recv_timestamp, but does not require an skb */
1921 static void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
1922                                struct scm_timestamping_internal *tss)
1923 {
1924         int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
1925         bool has_timestamping = false;
1926
1927         if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
1928                 if (sock_flag(sk, SOCK_RCVTSTAMP)) {
1929                         if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
1930                                 if (new_tstamp) {
1931                                         struct __kernel_timespec kts = {
1932                                                 .tv_sec = tss->ts[0].tv_sec,
1933                                                 .tv_nsec = tss->ts[0].tv_nsec,
1934                                         };
1935                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
1936                                                  sizeof(kts), &kts);
1937                                 } else {
1938                                         struct __kernel_old_timespec ts_old = {
1939                                                 .tv_sec = tss->ts[0].tv_sec,
1940                                                 .tv_nsec = tss->ts[0].tv_nsec,
1941                                         };
1942                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
1943                                                  sizeof(ts_old), &ts_old);
1944                                 }
1945                         } else {
1946                                 if (new_tstamp) {
1947                                         struct __kernel_sock_timeval stv = {
1948                                                 .tv_sec = tss->ts[0].tv_sec,
1949                                                 .tv_usec = tss->ts[0].tv_nsec / 1000,
1950                                         };
1951                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
1952                                                  sizeof(stv), &stv);
1953                                 } else {
1954                                         struct __kernel_old_timeval tv = {
1955                                                 .tv_sec = tss->ts[0].tv_sec,
1956                                                 .tv_usec = tss->ts[0].tv_nsec / 1000,
1957                                         };
1958                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
1959                                                  sizeof(tv), &tv);
1960                                 }
1961                         }
1962                 }
1963
1964                 if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE)
1965                         has_timestamping = true;
1966                 else
1967                         tss->ts[0] = (struct timespec64) {0};
1968         }
1969
1970         if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
1971                 if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)
1972                         has_timestamping = true;
1973                 else
1974                         tss->ts[2] = (struct timespec64) {0};
1975         }
1976
1977         if (has_timestamping) {
1978                 tss->ts[1] = (struct timespec64) {0};
1979                 if (sock_flag(sk, SOCK_TSTAMP_NEW))
1980                         put_cmsg_scm_timestamping64(msg, tss);
1981                 else
1982                         put_cmsg_scm_timestamping(msg, tss);
1983         }
1984 }
1985
1986 static int tcp_inq_hint(struct sock *sk)
1987 {
1988         const struct tcp_sock *tp = tcp_sk(sk);
1989         u32 copied_seq = READ_ONCE(tp->copied_seq);
1990         u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
1991         int inq;
1992
1993         inq = rcv_nxt - copied_seq;
1994         if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
1995                 lock_sock(sk);
1996                 inq = tp->rcv_nxt - tp->copied_seq;
1997                 release_sock(sk);
1998         }
1999         /* After receiving a FIN, tell the user-space to continue reading
2000          * by returning a non-zero inq.
2001          */
2002         if (inq == 0 && sock_flag(sk, SOCK_DONE))
2003                 inq = 1;
2004         return inq;
2005 }
2006
2007 /*
2008  *      This routine copies from a sock struct into the user buffer.
2009  *
2010  *      Technical note: in 2.3 we work on _locked_ socket, so that
2011  *      tricks with *seq access order and skb->users are not required.
2012  *      Probably, code can be easily improved even more.
2013  */
2014
2015 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
2016                 int flags, int *addr_len)
2017 {
2018         struct tcp_sock *tp = tcp_sk(sk);
2019         int copied = 0;
2020         u32 peek_seq;
2021         u32 *seq;
2022         unsigned long used;
2023         int err, inq;
2024         int target;             /* Read at least this many bytes */
2025         long timeo;
2026         struct sk_buff *skb, *last;
2027         u32 urg_hole = 0;
2028         struct scm_timestamping_internal tss;
2029         int cmsg_flags;
2030
2031         if (unlikely(flags & MSG_ERRQUEUE))
2032                 return inet_recv_error(sk, msg, len, addr_len);
2033
2034         if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue) &&
2035             (sk->sk_state == TCP_ESTABLISHED))
2036                 sk_busy_loop(sk, nonblock);
2037
2038         lock_sock(sk);
2039
2040         err = -ENOTCONN;
2041         if (sk->sk_state == TCP_LISTEN)
2042                 goto out;
2043
2044         cmsg_flags = tp->recvmsg_inq ? 1 : 0;
2045         timeo = sock_rcvtimeo(sk, nonblock);
2046
2047         /* Urgent data needs to be handled specially. */
2048         if (flags & MSG_OOB)
2049                 goto recv_urg;
2050
2051         if (unlikely(tp->repair)) {
2052                 err = -EPERM;
2053                 if (!(flags & MSG_PEEK))
2054                         goto out;
2055
2056                 if (tp->repair_queue == TCP_SEND_QUEUE)
2057                         goto recv_sndq;
2058
2059                 err = -EINVAL;
2060                 if (tp->repair_queue == TCP_NO_QUEUE)
2061                         goto out;
2062
2063                 /* 'common' recv queue MSG_PEEK-ing */
2064         }
2065
2066         seq = &tp->copied_seq;
2067         if (flags & MSG_PEEK) {
2068                 peek_seq = tp->copied_seq;
2069                 seq = &peek_seq;
2070         }
2071
2072         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2073
2074         do {
2075                 u32 offset;
2076
2077                 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
2078                 if (tp->urg_data && tp->urg_seq == *seq) {
2079                         if (copied)
2080                                 break;
2081                         if (signal_pending(current)) {
2082                                 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
2083                                 break;
2084                         }
2085                 }
2086
2087                 /* Next get a buffer. */
2088
2089                 last = skb_peek_tail(&sk->sk_receive_queue);
2090                 skb_queue_walk(&sk->sk_receive_queue, skb) {
2091                         last = skb;
2092                         /* Now that we have two receive queues this
2093                          * shouldn't happen.
2094                          */
2095                         if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
2096                                  "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
2097                                  *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
2098                                  flags))
2099                                 break;
2100
2101                         offset = *seq - TCP_SKB_CB(skb)->seq;
2102                         if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2103                                 pr_err_once("%s: found a SYN, please report !\n", __func__);
2104                                 offset--;
2105                         }
2106                         if (offset < skb->len)
2107                                 goto found_ok_skb;
2108                         if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2109                                 goto found_fin_ok;
2110                         WARN(!(flags & MSG_PEEK),
2111                              "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
2112                              *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
2113                 }
2114
2115                 /* Well, if we have backlog, try to process it now yet. */
2116
2117                 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
2118                         break;
2119
2120                 if (copied) {
2121                         if (sk->sk_err ||
2122                             sk->sk_state == TCP_CLOSE ||
2123                             (sk->sk_shutdown & RCV_SHUTDOWN) ||
2124                             !timeo ||
2125                             signal_pending(current))
2126                                 break;
2127                 } else {
2128                         if (sock_flag(sk, SOCK_DONE))
2129                                 break;
2130
2131                         if (sk->sk_err) {
2132                                 copied = sock_error(sk);
2133                                 break;
2134                         }
2135
2136                         if (sk->sk_shutdown & RCV_SHUTDOWN)
2137                                 break;
2138
2139                         if (sk->sk_state == TCP_CLOSE) {
2140                                 /* This occurs when user tries to read
2141                                  * from never connected socket.
2142                                  */
2143                                 copied = -ENOTCONN;
2144                                 break;
2145                         }
2146
2147                         if (!timeo) {
2148                                 copied = -EAGAIN;
2149                                 break;
2150                         }
2151
2152                         if (signal_pending(current)) {
2153                                 copied = sock_intr_errno(timeo);
2154                                 break;
2155                         }
2156                 }
2157
2158                 tcp_cleanup_rbuf(sk, copied);
2159
2160                 if (copied >= target) {
2161                         /* Do not sleep, just process backlog. */
2162                         release_sock(sk);
2163                         lock_sock(sk);
2164                 } else {
2165                         sk_wait_data(sk, &timeo, last);
2166                 }
2167
2168                 if ((flags & MSG_PEEK) &&
2169                     (peek_seq - copied - urg_hole != tp->copied_seq)) {
2170                         net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
2171                                             current->comm,
2172                                             task_pid_nr(current));
2173                         peek_seq = tp->copied_seq;
2174                 }
2175                 continue;
2176
2177 found_ok_skb:
2178                 /* Ok so how much can we use? */
2179                 used = skb->len - offset;
2180                 if (len < used)
2181                         used = len;
2182
2183                 /* Do we have urgent data here? */
2184                 if (tp->urg_data) {
2185                         u32 urg_offset = tp->urg_seq - *seq;
2186                         if (urg_offset < used) {
2187                                 if (!urg_offset) {
2188                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
2189                                                 WRITE_ONCE(*seq, *seq + 1);
2190                                                 urg_hole++;
2191                                                 offset++;
2192                                                 used--;
2193                                                 if (!used)
2194                                                         goto skip_copy;
2195                                         }
2196                                 } else
2197                                         used = urg_offset;
2198                         }
2199                 }
2200
2201                 if (!(flags & MSG_TRUNC)) {
2202                         err = skb_copy_datagram_msg(skb, offset, msg, used);
2203                         if (err) {
2204                                 /* Exception. Bailout! */
2205                                 if (!copied)
2206                                         copied = -EFAULT;
2207                                 break;
2208                         }
2209                 }
2210
2211                 WRITE_ONCE(*seq, *seq + used);
2212                 copied += used;
2213                 len -= used;
2214
2215                 tcp_rcv_space_adjust(sk);
2216
2217 skip_copy:
2218                 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
2219                         tp->urg_data = 0;
2220                         tcp_fast_path_check(sk);
2221                 }
2222
2223                 if (TCP_SKB_CB(skb)->has_rxtstamp) {
2224                         tcp_update_recv_tstamps(skb, &tss);
2225                         cmsg_flags |= 2;
2226                 }
2227
2228                 if (used + offset < skb->len)
2229                         continue;
2230
2231                 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2232                         goto found_fin_ok;
2233                 if (!(flags & MSG_PEEK))
2234                         sk_eat_skb(sk, skb);
2235                 continue;
2236
2237 found_fin_ok:
2238                 /* Process the FIN. */
2239                 WRITE_ONCE(*seq, *seq + 1);
2240                 if (!(flags & MSG_PEEK))
2241                         sk_eat_skb(sk, skb);
2242                 break;
2243         } while (len > 0);
2244
2245         /* According to UNIX98, msg_name/msg_namelen are ignored
2246          * on connected socket. I was just happy when found this 8) --ANK
2247          */
2248
2249         /* Clean up data we have read: This will do ACK frames. */
2250         tcp_cleanup_rbuf(sk, copied);
2251
2252         release_sock(sk);
2253
2254         if (cmsg_flags) {
2255                 if (cmsg_flags & 2)
2256                         tcp_recv_timestamp(msg, sk, &tss);
2257                 if (cmsg_flags & 1) {
2258                         inq = tcp_inq_hint(sk);
2259                         put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq);
2260                 }
2261         }
2262
2263         return copied;
2264
2265 out:
2266         release_sock(sk);
2267         return err;
2268
2269 recv_urg:
2270         err = tcp_recv_urg(sk, msg, len, flags);
2271         goto out;
2272
2273 recv_sndq:
2274         err = tcp_peek_sndq(sk, msg, len);
2275         goto out;
2276 }
2277 EXPORT_SYMBOL(tcp_recvmsg);
2278
2279 void tcp_set_state(struct sock *sk, int state)
2280 {
2281         int oldstate = sk->sk_state;
2282
2283         /* We defined a new enum for TCP states that are exported in BPF
2284          * so as not force the internal TCP states to be frozen. The
2285          * following checks will detect if an internal state value ever
2286          * differs from the BPF value. If this ever happens, then we will
2287          * need to remap the internal value to the BPF value before calling
2288          * tcp_call_bpf_2arg.
2289          */
2290         BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
2291         BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
2292         BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
2293         BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
2294         BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
2295         BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
2296         BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
2297         BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
2298         BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
2299         BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
2300         BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
2301         BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
2302         BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
2303
2304         if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
2305                 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
2306
2307         switch (state) {
2308         case TCP_ESTABLISHED:
2309                 if (oldstate != TCP_ESTABLISHED)
2310                         TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2311                 break;
2312
2313         case TCP_CLOSE:
2314                 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
2315                         TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
2316
2317                 sk->sk_prot->unhash(sk);
2318                 if (inet_csk(sk)->icsk_bind_hash &&
2319                     !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
2320                         inet_put_port(sk);
2321                 fallthrough;
2322         default:
2323                 if (oldstate == TCP_ESTABLISHED)
2324                         TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2325         }
2326
2327         /* Change state AFTER socket is unhashed to avoid closed
2328          * socket sitting in hash tables.
2329          */
2330         inet_sk_state_store(sk, state);
2331 }
2332 EXPORT_SYMBOL_GPL(tcp_set_state);
2333
2334 /*
2335  *      State processing on a close. This implements the state shift for
2336  *      sending our FIN frame. Note that we only send a FIN for some
2337  *      states. A shutdown() may have already sent the FIN, or we may be
2338  *      closed.
2339  */
2340
2341 static const unsigned char new_state[16] = {
2342   /* current state:        new state:      action:      */
2343   [0 /* (Invalid) */]   = TCP_CLOSE,
2344   [TCP_ESTABLISHED]     = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2345   [TCP_SYN_SENT]        = TCP_CLOSE,
2346   [TCP_SYN_RECV]        = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2347   [TCP_FIN_WAIT1]       = TCP_FIN_WAIT1,
2348   [TCP_FIN_WAIT2]       = TCP_FIN_WAIT2,
2349   [TCP_TIME_WAIT]       = TCP_CLOSE,
2350   [TCP_CLOSE]           = TCP_CLOSE,
2351   [TCP_CLOSE_WAIT]      = TCP_LAST_ACK  | TCP_ACTION_FIN,
2352   [TCP_LAST_ACK]        = TCP_LAST_ACK,
2353   [TCP_LISTEN]          = TCP_CLOSE,
2354   [TCP_CLOSING]         = TCP_CLOSING,
2355   [TCP_NEW_SYN_RECV]    = TCP_CLOSE,    /* should not happen ! */
2356 };
2357
2358 static int tcp_close_state(struct sock *sk)
2359 {
2360         int next = (int)new_state[sk->sk_state];
2361         int ns = next & TCP_STATE_MASK;
2362
2363         tcp_set_state(sk, ns);
2364
2365         return next & TCP_ACTION_FIN;
2366 }
2367
2368 /*
2369  *      Shutdown the sending side of a connection. Much like close except
2370  *      that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
2371  */
2372
2373 void tcp_shutdown(struct sock *sk, int how)
2374 {
2375         /*      We need to grab some memory, and put together a FIN,
2376          *      and then put it into the queue to be sent.
2377          *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
2378          */
2379         if (!(how & SEND_SHUTDOWN))
2380                 return;
2381
2382         /* If we've already sent a FIN, or it's a closed state, skip this. */
2383         if ((1 << sk->sk_state) &
2384             (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2385              TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2386                 /* Clear out any half completed packets.  FIN if needed. */
2387                 if (tcp_close_state(sk))
2388                         tcp_send_fin(sk);
2389         }
2390 }
2391 EXPORT_SYMBOL(tcp_shutdown);
2392
2393 bool tcp_check_oom(struct sock *sk, int shift)
2394 {
2395         bool too_many_orphans, out_of_socket_memory;
2396
2397         too_many_orphans = tcp_too_many_orphans(sk, shift);
2398         out_of_socket_memory = tcp_out_of_memory(sk);
2399
2400         if (too_many_orphans)
2401                 net_info_ratelimited("too many orphaned sockets\n");
2402         if (out_of_socket_memory)
2403                 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2404         return too_many_orphans || out_of_socket_memory;
2405 }
2406
2407 void tcp_close(struct sock *sk, long timeout)
2408 {
2409         struct sk_buff *skb;
2410         int data_was_unread = 0;
2411         int state;
2412
2413         lock_sock(sk);
2414         sk->sk_shutdown = SHUTDOWN_MASK;
2415
2416         if (sk->sk_state == TCP_LISTEN) {
2417                 tcp_set_state(sk, TCP_CLOSE);
2418
2419                 /* Special case. */
2420                 inet_csk_listen_stop(sk);
2421
2422                 goto adjudge_to_death;
2423         }
2424
2425         /*  We need to flush the recv. buffs.  We do this only on the
2426          *  descriptor close, not protocol-sourced closes, because the
2427          *  reader process may not have drained the data yet!
2428          */
2429         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2430                 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2431
2432                 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2433                         len--;
2434                 data_was_unread += len;
2435                 __kfree_skb(skb);
2436         }
2437
2438         sk_mem_reclaim(sk);
2439
2440         /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2441         if (sk->sk_state == TCP_CLOSE)
2442                 goto adjudge_to_death;
2443
2444         /* As outlined in RFC 2525, section 2.17, we send a RST here because
2445          * data was lost. To witness the awful effects of the old behavior of
2446          * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2447          * GET in an FTP client, suspend the process, wait for the client to
2448          * advertise a zero window, then kill -9 the FTP client, wheee...
2449          * Note: timeout is always zero in such a case.
2450          */
2451         if (unlikely(tcp_sk(sk)->repair)) {
2452                 sk->sk_prot->disconnect(sk, 0);
2453         } else if (data_was_unread) {
2454                 /* Unread data was tossed, zap the connection. */
2455                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2456                 tcp_set_state(sk, TCP_CLOSE);
2457                 tcp_send_active_reset(sk, sk->sk_allocation);
2458         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2459                 /* Check zero linger _after_ checking for unread data. */
2460                 sk->sk_prot->disconnect(sk, 0);
2461                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2462         } else if (tcp_close_state(sk)) {
2463                 /* We FIN if the application ate all the data before
2464                  * zapping the connection.
2465                  */
2466
2467                 /* RED-PEN. Formally speaking, we have broken TCP state
2468                  * machine. State transitions:
2469                  *
2470                  * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2471                  * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
2472                  * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2473                  *
2474                  * are legal only when FIN has been sent (i.e. in window),
2475                  * rather than queued out of window. Purists blame.
2476                  *
2477                  * F.e. "RFC state" is ESTABLISHED,
2478                  * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2479                  *
2480                  * The visible declinations are that sometimes
2481                  * we enter time-wait state, when it is not required really
2482                  * (harmless), do not send active resets, when they are
2483                  * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2484                  * they look as CLOSING or LAST_ACK for Linux)
2485                  * Probably, I missed some more holelets.
2486                  *                                              --ANK
2487                  * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2488                  * in a single packet! (May consider it later but will
2489                  * probably need API support or TCP_CORK SYN-ACK until
2490                  * data is written and socket is closed.)
2491                  */
2492                 tcp_send_fin(sk);
2493         }
2494
2495         sk_stream_wait_close(sk, timeout);
2496
2497 adjudge_to_death:
2498         state = sk->sk_state;
2499         sock_hold(sk);
2500         sock_orphan(sk);
2501
2502         local_bh_disable();
2503         bh_lock_sock(sk);
2504         /* remove backlog if any, without releasing ownership. */
2505         __release_sock(sk);
2506
2507         percpu_counter_inc(sk->sk_prot->orphan_count);
2508
2509         /* Have we already been destroyed by a softirq or backlog? */
2510         if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2511                 goto out;
2512
2513         /*      This is a (useful) BSD violating of the RFC. There is a
2514          *      problem with TCP as specified in that the other end could
2515          *      keep a socket open forever with no application left this end.
2516          *      We use a 1 minute timeout (about the same as BSD) then kill
2517          *      our end. If they send after that then tough - BUT: long enough
2518          *      that we won't make the old 4*rto = almost no time - whoops
2519          *      reset mistake.
2520          *
2521          *      Nope, it was not mistake. It is really desired behaviour
2522          *      f.e. on http servers, when such sockets are useless, but
2523          *      consume significant resources. Let's do it with special
2524          *      linger2 option.                                 --ANK
2525          */
2526
2527         if (sk->sk_state == TCP_FIN_WAIT2) {
2528                 struct tcp_sock *tp = tcp_sk(sk);
2529                 if (tp->linger2 < 0) {
2530                         tcp_set_state(sk, TCP_CLOSE);
2531                         tcp_send_active_reset(sk, GFP_ATOMIC);
2532                         __NET_INC_STATS(sock_net(sk),
2533                                         LINUX_MIB_TCPABORTONLINGER);
2534                 } else {
2535                         const int tmo = tcp_fin_time(sk);
2536
2537                         if (tmo > TCP_TIMEWAIT_LEN) {
2538                                 inet_csk_reset_keepalive_timer(sk,
2539                                                 tmo - TCP_TIMEWAIT_LEN);
2540                         } else {
2541                                 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2542                                 goto out;
2543                         }
2544                 }
2545         }
2546         if (sk->sk_state != TCP_CLOSE) {
2547                 sk_mem_reclaim(sk);
2548                 if (tcp_check_oom(sk, 0)) {
2549                         tcp_set_state(sk, TCP_CLOSE);
2550                         tcp_send_active_reset(sk, GFP_ATOMIC);
2551                         __NET_INC_STATS(sock_net(sk),
2552                                         LINUX_MIB_TCPABORTONMEMORY);
2553                 } else if (!check_net(sock_net(sk))) {
2554                         /* Not possible to send reset; just close */
2555                         tcp_set_state(sk, TCP_CLOSE);
2556                 }
2557         }
2558
2559         if (sk->sk_state == TCP_CLOSE) {
2560                 struct request_sock *req;
2561
2562                 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
2563                                                 lockdep_sock_is_held(sk));
2564                 /* We could get here with a non-NULL req if the socket is
2565                  * aborted (e.g., closed with unread data) before 3WHS
2566                  * finishes.
2567                  */
2568                 if (req)
2569                         reqsk_fastopen_remove(sk, req, false);
2570                 inet_csk_destroy_sock(sk);
2571         }
2572         /* Otherwise, socket is reprieved until protocol close. */
2573
2574 out:
2575         bh_unlock_sock(sk);
2576         local_bh_enable();
2577         release_sock(sk);
2578         sock_put(sk);
2579 }
2580 EXPORT_SYMBOL(tcp_close);
2581
2582 /* These states need RST on ABORT according to RFC793 */
2583
2584 static inline bool tcp_need_reset(int state)
2585 {
2586         return (1 << state) &
2587                (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2588                 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2589 }
2590
2591 static void tcp_rtx_queue_purge(struct sock *sk)
2592 {
2593         struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
2594
2595         tcp_sk(sk)->highest_sack = NULL;
2596         while (p) {
2597                 struct sk_buff *skb = rb_to_skb(p);
2598
2599                 p = rb_next(p);
2600                 /* Since we are deleting whole queue, no need to
2601                  * list_del(&skb->tcp_tsorted_anchor)
2602                  */
2603                 tcp_rtx_queue_unlink(skb, sk);
2604                 sk_wmem_free_skb(sk, skb);
2605         }
2606 }
2607
2608 void tcp_write_queue_purge(struct sock *sk)
2609 {
2610         struct sk_buff *skb;
2611
2612         tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
2613         while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
2614                 tcp_skb_tsorted_anchor_cleanup(skb);
2615                 sk_wmem_free_skb(sk, skb);
2616         }
2617         tcp_rtx_queue_purge(sk);
2618         skb = sk->sk_tx_skb_cache;
2619         if (skb) {
2620                 __kfree_skb(skb);
2621                 sk->sk_tx_skb_cache = NULL;
2622         }
2623         INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
2624         sk_mem_reclaim(sk);
2625         tcp_clear_all_retrans_hints(tcp_sk(sk));
2626         tcp_sk(sk)->packets_out = 0;
2627         inet_csk(sk)->icsk_backoff = 0;
2628 }
2629
2630 int tcp_disconnect(struct sock *sk, int flags)
2631 {
2632         struct inet_sock *inet = inet_sk(sk);
2633         struct inet_connection_sock *icsk = inet_csk(sk);
2634         struct tcp_sock *tp = tcp_sk(sk);
2635         int old_state = sk->sk_state;
2636         u32 seq;
2637
2638         if (old_state != TCP_CLOSE)
2639                 tcp_set_state(sk, TCP_CLOSE);
2640
2641         /* ABORT function of RFC793 */
2642         if (old_state == TCP_LISTEN) {
2643                 inet_csk_listen_stop(sk);
2644         } else if (unlikely(tp->repair)) {
2645                 sk->sk_err = ECONNABORTED;
2646         } else if (tcp_need_reset(old_state) ||
2647                    (tp->snd_nxt != tp->write_seq &&
2648                     (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2649                 /* The last check adjusts for discrepancy of Linux wrt. RFC
2650                  * states
2651                  */
2652                 tcp_send_active_reset(sk, gfp_any());
2653                 sk->sk_err = ECONNRESET;
2654         } else if (old_state == TCP_SYN_SENT)
2655                 sk->sk_err = ECONNRESET;
2656
2657         tcp_clear_xmit_timers(sk);
2658         __skb_queue_purge(&sk->sk_receive_queue);
2659         if (sk->sk_rx_skb_cache) {
2660                 __kfree_skb(sk->sk_rx_skb_cache);
2661                 sk->sk_rx_skb_cache = NULL;
2662         }
2663         WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
2664         tp->urg_data = 0;
2665         tcp_write_queue_purge(sk);
2666         tcp_fastopen_active_disable_ofo_check(sk);
2667         skb_rbtree_purge(&tp->out_of_order_queue);
2668
2669         inet->inet_dport = 0;
2670
2671         if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2672                 inet_reset_saddr(sk);
2673
2674         sk->sk_shutdown = 0;
2675         sock_reset_flag(sk, SOCK_DONE);
2676         tp->srtt_us = 0;
2677         tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
2678         tp->rcv_rtt_last_tsecr = 0;
2679
2680         seq = tp->write_seq + tp->max_window + 2;
2681         if (!seq)
2682                 seq = 1;
2683         WRITE_ONCE(tp->write_seq, seq);
2684
2685         icsk->icsk_backoff = 0;
2686         icsk->icsk_probes_out = 0;
2687         icsk->icsk_rto = TCP_TIMEOUT_INIT;
2688         icsk->icsk_rto_min = TCP_RTO_MIN;
2689         icsk->icsk_delack_max = TCP_DELACK_MAX;
2690         tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2691         tp->snd_cwnd = TCP_INIT_CWND;
2692         tp->snd_cwnd_cnt = 0;
2693         tp->window_clamp = 0;
2694         tp->delivered = 0;
2695         tp->delivered_ce = 0;
2696         if (icsk->icsk_ca_ops->release)
2697                 icsk->icsk_ca_ops->release(sk);
2698         memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
2699         tcp_set_ca_state(sk, TCP_CA_Open);
2700         tp->is_sack_reneg = 0;
2701         tcp_clear_retrans(tp);
2702         tp->total_retrans = 0;
2703         inet_csk_delack_init(sk);
2704         /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
2705          * issue in __tcp_select_window()
2706          */
2707         icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
2708         memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2709         __sk_dst_reset(sk);
2710         dst_release(sk->sk_rx_dst);
2711         sk->sk_rx_dst = NULL;
2712         tcp_saved_syn_free(tp);
2713         tp->compressed_ack = 0;
2714         tp->segs_in = 0;
2715         tp->segs_out = 0;
2716         tp->bytes_sent = 0;
2717         tp->bytes_acked = 0;
2718         tp->bytes_received = 0;
2719         tp->bytes_retrans = 0;
2720         tp->data_segs_in = 0;
2721         tp->data_segs_out = 0;
2722         tp->duplicate_sack[0].start_seq = 0;
2723         tp->duplicate_sack[0].end_seq = 0;
2724         tp->dsack_dups = 0;
2725         tp->reord_seen = 0;
2726         tp->retrans_out = 0;
2727         tp->sacked_out = 0;
2728         tp->tlp_high_seq = 0;
2729         tp->last_oow_ack_time = 0;
2730         /* There's a bubble in the pipe until at least the first ACK. */
2731         tp->app_limited = ~0U;
2732         tp->rack.mstamp = 0;
2733         tp->rack.advanced = 0;
2734         tp->rack.reo_wnd_steps = 1;
2735         tp->rack.last_delivered = 0;
2736         tp->rack.reo_wnd_persist = 0;
2737         tp->rack.dsack_seen = 0;
2738         tp->syn_data_acked = 0;
2739         tp->rx_opt.saw_tstamp = 0;
2740         tp->rx_opt.dsack = 0;
2741         tp->rx_opt.num_sacks = 0;
2742         tp->rcv_ooopack = 0;
2743
2744
2745         /* Clean up fastopen related fields */
2746         tcp_free_fastopen_req(tp);
2747         inet->defer_connect = 0;
2748         tp->fastopen_client_fail = 0;
2749
2750         WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2751
2752         if (sk->sk_frag.page) {
2753                 put_page(sk->sk_frag.page);
2754                 sk->sk_frag.page = NULL;
2755                 sk->sk_frag.offset = 0;
2756         }
2757
2758         sk->sk_error_report(sk);
2759         return 0;
2760 }
2761 EXPORT_SYMBOL(tcp_disconnect);
2762
2763 static inline bool tcp_can_repair_sock(const struct sock *sk)
2764 {
2765         return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2766                 (sk->sk_state != TCP_LISTEN);
2767 }
2768
2769 static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
2770 {
2771         struct tcp_repair_window opt;
2772
2773         if (!tp->repair)
2774                 return -EPERM;
2775
2776         if (len != sizeof(opt))
2777                 return -EINVAL;
2778
2779         if (copy_from_sockptr(&opt, optbuf, sizeof(opt)))
2780                 return -EFAULT;
2781
2782         if (opt.max_window < opt.snd_wnd)
2783                 return -EINVAL;
2784
2785         if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
2786                 return -EINVAL;
2787
2788         if (after(opt.rcv_wup, tp->rcv_nxt))
2789                 return -EINVAL;
2790
2791         tp->snd_wl1     = opt.snd_wl1;
2792         tp->snd_wnd     = opt.snd_wnd;
2793         tp->max_window  = opt.max_window;
2794
2795         tp->rcv_wnd     = opt.rcv_wnd;
2796         tp->rcv_wup     = opt.rcv_wup;
2797
2798         return 0;
2799 }
2800
2801 static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
2802                 unsigned int len)
2803 {
2804         struct tcp_sock *tp = tcp_sk(sk);
2805         struct tcp_repair_opt opt;
2806         size_t offset = 0;
2807
2808         while (len >= sizeof(opt)) {
2809                 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt)))
2810                         return -EFAULT;
2811
2812                 offset += sizeof(opt);
2813                 len -= sizeof(opt);
2814
2815                 switch (opt.opt_code) {
2816                 case TCPOPT_MSS:
2817                         tp->rx_opt.mss_clamp = opt.opt_val;
2818                         tcp_mtup_init(sk);
2819                         break;
2820                 case TCPOPT_WINDOW:
2821                         {
2822                                 u16 snd_wscale = opt.opt_val & 0xFFFF;
2823                                 u16 rcv_wscale = opt.opt_val >> 16;
2824
2825                                 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
2826                                         return -EFBIG;
2827
2828                                 tp->rx_opt.snd_wscale = snd_wscale;
2829                                 tp->rx_opt.rcv_wscale = rcv_wscale;
2830                                 tp->rx_opt.wscale_ok = 1;
2831                         }
2832                         break;
2833                 case TCPOPT_SACK_PERM:
2834                         if (opt.opt_val != 0)
2835                                 return -EINVAL;
2836
2837                         tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2838                         break;
2839                 case TCPOPT_TIMESTAMP:
2840                         if (opt.opt_val != 0)
2841                                 return -EINVAL;
2842
2843                         tp->rx_opt.tstamp_ok = 1;
2844                         break;
2845                 }
2846         }
2847
2848         return 0;
2849 }
2850
2851 DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2852 EXPORT_SYMBOL(tcp_tx_delay_enabled);
2853
2854 static void tcp_enable_tx_delay(void)
2855 {
2856         if (!static_branch_unlikely(&tcp_tx_delay_enabled)) {
2857                 static int __tcp_tx_delay_enabled = 0;
2858
2859                 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
2860                         static_branch_enable(&tcp_tx_delay_enabled);
2861                         pr_info("TCP_TX_DELAY enabled\n");
2862                 }
2863         }
2864 }
2865
2866 /* When set indicates to always queue non-full frames.  Later the user clears
2867  * this option and we transmit any pending partial frames in the queue.  This is
2868  * meant to be used alongside sendfile() to get properly filled frames when the
2869  * user (for example) must write out headers with a write() call first and then
2870  * use sendfile to send out the data parts.
2871  *
2872  * TCP_CORK can be set together with TCP_NODELAY and it is stronger than
2873  * TCP_NODELAY.
2874  */
2875 static void __tcp_sock_set_cork(struct sock *sk, bool on)
2876 {
2877         struct tcp_sock *tp = tcp_sk(sk);
2878
2879         if (on) {
2880                 tp->nonagle |= TCP_NAGLE_CORK;
2881         } else {
2882                 tp->nonagle &= ~TCP_NAGLE_CORK;
2883                 if (tp->nonagle & TCP_NAGLE_OFF)
2884                         tp->nonagle |= TCP_NAGLE_PUSH;
2885                 tcp_push_pending_frames(sk);
2886         }
2887 }
2888
2889 void tcp_sock_set_cork(struct sock *sk, bool on)
2890 {
2891         lock_sock(sk);
2892         __tcp_sock_set_cork(sk, on);
2893         release_sock(sk);
2894 }
2895 EXPORT_SYMBOL(tcp_sock_set_cork);
2896
2897 /* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is
2898  * remembered, but it is not activated until cork is cleared.
2899  *
2900  * However, when TCP_NODELAY is set we make an explicit push, which overrides
2901  * even TCP_CORK for currently queued segments.
2902  */
2903 static void __tcp_sock_set_nodelay(struct sock *sk, bool on)
2904 {
2905         if (on) {
2906                 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2907                 tcp_push_pending_frames(sk);
2908         } else {
2909                 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
2910         }
2911 }
2912
2913 void tcp_sock_set_nodelay(struct sock *sk)
2914 {
2915         lock_sock(sk);
2916         __tcp_sock_set_nodelay(sk, true);
2917         release_sock(sk);
2918 }
2919 EXPORT_SYMBOL(tcp_sock_set_nodelay);
2920
2921 static void __tcp_sock_set_quickack(struct sock *sk, int val)
2922 {
2923         if (!val) {
2924                 inet_csk_enter_pingpong_mode(sk);
2925                 return;
2926         }
2927
2928         inet_csk_exit_pingpong_mode(sk);
2929         if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2930             inet_csk_ack_scheduled(sk)) {
2931                 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED;
2932                 tcp_cleanup_rbuf(sk, 1);
2933                 if (!(val & 1))
2934                         inet_csk_enter_pingpong_mode(sk);
2935         }
2936 }
2937
2938 void tcp_sock_set_quickack(struct sock *sk, int val)
2939 {
2940         lock_sock(sk);
2941         __tcp_sock_set_quickack(sk, val);
2942         release_sock(sk);
2943 }
2944 EXPORT_SYMBOL(tcp_sock_set_quickack);
2945
2946 int tcp_sock_set_syncnt(struct sock *sk, int val)
2947 {
2948         if (val < 1 || val > MAX_TCP_SYNCNT)
2949                 return -EINVAL;
2950
2951         lock_sock(sk);
2952         inet_csk(sk)->icsk_syn_retries = val;
2953         release_sock(sk);
2954         return 0;
2955 }
2956 EXPORT_SYMBOL(tcp_sock_set_syncnt);
2957
2958 void tcp_sock_set_user_timeout(struct sock *sk, u32 val)
2959 {
2960         lock_sock(sk);
2961         inet_csk(sk)->icsk_user_timeout = val;
2962         release_sock(sk);
2963 }
2964 EXPORT_SYMBOL(tcp_sock_set_user_timeout);
2965
2966 int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
2967 {
2968         struct tcp_sock *tp = tcp_sk(sk);
2969
2970         if (val < 1 || val > MAX_TCP_KEEPIDLE)
2971                 return -EINVAL;
2972
2973         tp->keepalive_time = val * HZ;
2974         if (sock_flag(sk, SOCK_KEEPOPEN) &&
2975             !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
2976                 u32 elapsed = keepalive_time_elapsed(tp);
2977
2978                 if (tp->keepalive_time > elapsed)
2979                         elapsed = tp->keepalive_time - elapsed;
2980                 else
2981                         elapsed = 0;
2982                 inet_csk_reset_keepalive_timer(sk, elapsed);
2983         }
2984
2985         return 0;
2986 }
2987
2988 int tcp_sock_set_keepidle(struct sock *sk, int val)
2989 {
2990         int err;
2991
2992         lock_sock(sk);
2993         err = tcp_sock_set_keepidle_locked(sk, val);
2994         release_sock(sk);
2995         return err;
2996 }
2997 EXPORT_SYMBOL(tcp_sock_set_keepidle);
2998
2999 int tcp_sock_set_keepintvl(struct sock *sk, int val)
3000 {
3001         if (val < 1 || val > MAX_TCP_KEEPINTVL)
3002                 return -EINVAL;
3003
3004         lock_sock(sk);
3005         tcp_sk(sk)->keepalive_intvl = val * HZ;
3006         release_sock(sk);
3007         return 0;
3008 }
3009 EXPORT_SYMBOL(tcp_sock_set_keepintvl);
3010
3011 int tcp_sock_set_keepcnt(struct sock *sk, int val)
3012 {
3013         if (val < 1 || val > MAX_TCP_KEEPCNT)
3014                 return -EINVAL;
3015
3016         lock_sock(sk);
3017         tcp_sk(sk)->keepalive_probes = val;
3018         release_sock(sk);
3019         return 0;
3020 }
3021 EXPORT_SYMBOL(tcp_sock_set_keepcnt);
3022
3023 /*
3024  *      Socket option code for TCP.
3025  */
3026 static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
3027                 sockptr_t optval, unsigned int optlen)
3028 {
3029         struct tcp_sock *tp = tcp_sk(sk);
3030         struct inet_connection_sock *icsk = inet_csk(sk);
3031         struct net *net = sock_net(sk);
3032         int val;
3033         int err = 0;
3034
3035         /* These are data/string values, all the others are ints */
3036         switch (optname) {
3037         case TCP_CONGESTION: {
3038                 char name[TCP_CA_NAME_MAX];
3039
3040                 if (optlen < 1)
3041                         return -EINVAL;
3042
3043                 val = strncpy_from_sockptr(name, optval,
3044                                         min_t(long, TCP_CA_NAME_MAX-1, optlen));
3045                 if (val < 0)
3046                         return -EFAULT;
3047                 name[val] = 0;
3048
3049                 lock_sock(sk);
3050                 err = tcp_set_congestion_control(sk, name, true, true,
3051                                                  ns_capable(sock_net(sk)->user_ns,
3052                                                             CAP_NET_ADMIN));
3053                 release_sock(sk);
3054                 return err;
3055         }
3056         case TCP_ULP: {
3057                 char name[TCP_ULP_NAME_MAX];
3058
3059                 if (optlen < 1)
3060                         return -EINVAL;
3061
3062                 val = strncpy_from_sockptr(name, optval,
3063                                         min_t(long, TCP_ULP_NAME_MAX - 1,
3064                                               optlen));
3065                 if (val < 0)
3066                         return -EFAULT;
3067                 name[val] = 0;
3068
3069                 lock_sock(sk);
3070                 err = tcp_set_ulp(sk, name);
3071                 release_sock(sk);
3072                 return err;
3073         }
3074         case TCP_FASTOPEN_KEY: {
3075                 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
3076                 __u8 *backup_key = NULL;
3077
3078                 /* Allow a backup key as well to facilitate key rotation
3079                  * First key is the active one.
3080                  */
3081                 if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
3082                     optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
3083                         return -EINVAL;
3084
3085                 if (copy_from_sockptr(key, optval, optlen))
3086                         return -EFAULT;
3087
3088                 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
3089                         backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
3090
3091                 return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
3092         }
3093         default:
3094                 /* fallthru */
3095                 break;
3096         }
3097
3098         if (optlen < sizeof(int))
3099                 return -EINVAL;
3100
3101         if (copy_from_sockptr(&val, optval, sizeof(val)))
3102                 return -EFAULT;
3103
3104         lock_sock(sk);
3105
3106         switch (optname) {
3107         case TCP_MAXSEG:
3108                 /* Values greater than interface MTU won't take effect. However
3109                  * at the point when this call is done we typically don't yet
3110                  * know which interface is going to be used
3111                  */
3112                 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) {
3113                         err = -EINVAL;
3114                         break;
3115                 }
3116                 tp->rx_opt.user_mss = val;
3117                 break;
3118
3119         case TCP_NODELAY:
3120                 __tcp_sock_set_nodelay(sk, val);
3121                 break;
3122
3123         case TCP_THIN_LINEAR_TIMEOUTS:
3124                 if (val < 0 || val > 1)
3125                         err = -EINVAL;
3126                 else
3127                         tp->thin_lto = val;
3128                 break;
3129
3130         case TCP_THIN_DUPACK:
3131                 if (val < 0 || val > 1)
3132                         err = -EINVAL;
3133                 break;
3134
3135         case TCP_REPAIR:
3136                 if (!tcp_can_repair_sock(sk))
3137                         err = -EPERM;
3138                 else if (val == TCP_REPAIR_ON) {
3139                         tp->repair = 1;
3140                         sk->sk_reuse = SK_FORCE_REUSE;
3141                         tp->repair_queue = TCP_NO_QUEUE;
3142                 } else if (val == TCP_REPAIR_OFF) {
3143                         tp->repair = 0;
3144                         sk->sk_reuse = SK_NO_REUSE;
3145                         tcp_send_window_probe(sk);
3146                 } else if (val == TCP_REPAIR_OFF_NO_WP) {
3147                         tp->repair = 0;
3148                         sk->sk_reuse = SK_NO_REUSE;
3149                 } else
3150                         err = -EINVAL;
3151
3152                 break;
3153
3154         case TCP_REPAIR_QUEUE:
3155                 if (!tp->repair)
3156                         err = -EPERM;
3157                 else if ((unsigned int)val < TCP_QUEUES_NR)
3158                         tp->repair_queue = val;
3159                 else
3160                         err = -EINVAL;
3161                 break;
3162
3163         case TCP_QUEUE_SEQ:
3164                 if (sk->sk_state != TCP_CLOSE)
3165                         err = -EPERM;
3166                 else if (tp->repair_queue == TCP_SEND_QUEUE)
3167                         WRITE_ONCE(tp->write_seq, val);
3168                 else if (tp->repair_queue == TCP_RECV_QUEUE) {
3169                         WRITE_ONCE(tp->rcv_nxt, val);
3170                         WRITE_ONCE(tp->copied_seq, val);
3171                 }
3172                 else
3173                         err = -EINVAL;
3174                 break;
3175
3176         case TCP_REPAIR_OPTIONS:
3177                 if (!tp->repair)
3178                         err = -EINVAL;
3179                 else if (sk->sk_state == TCP_ESTABLISHED)
3180                         err = tcp_repair_options_est(sk, optval, optlen);
3181                 else
3182                         err = -EPERM;
3183                 break;
3184
3185         case TCP_CORK:
3186                 __tcp_sock_set_cork(sk, val);
3187                 break;
3188
3189         case TCP_KEEPIDLE:
3190                 err = tcp_sock_set_keepidle_locked(sk, val);
3191                 break;
3192         case TCP_KEEPINTVL:
3193                 if (val < 1 || val > MAX_TCP_KEEPINTVL)
3194                         err = -EINVAL;
3195                 else
3196                         tp->keepalive_intvl = val * HZ;
3197                 break;
3198         case TCP_KEEPCNT:
3199                 if (val < 1 || val > MAX_TCP_KEEPCNT)
3200                         err = -EINVAL;
3201                 else
3202                         tp->keepalive_probes = val;
3203                 break;
3204         case TCP_SYNCNT:
3205                 if (val < 1 || val > MAX_TCP_SYNCNT)
3206                         err = -EINVAL;
3207                 else
3208                         icsk->icsk_syn_retries = val;
3209                 break;
3210
3211         case TCP_SAVE_SYN:
3212                 /* 0: disable, 1: enable, 2: start from ether_header */
3213                 if (val < 0 || val > 2)
3214                         err = -EINVAL;
3215                 else
3216                         tp->save_syn = val;
3217                 break;
3218
3219         case TCP_LINGER2:
3220                 if (val < 0)
3221                         tp->linger2 = -1;
3222                 else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
3223                         tp->linger2 = TCP_FIN_TIMEOUT_MAX;
3224                 else
3225                         tp->linger2 = val * HZ;
3226                 break;
3227
3228         case TCP_DEFER_ACCEPT:
3229                 /* Translate value in seconds to number of retransmits */
3230                 icsk->icsk_accept_queue.rskq_defer_accept =
3231                         secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
3232                                         TCP_RTO_MAX / HZ);
3233                 break;
3234
3235         case TCP_WINDOW_CLAMP:
3236                 if (!val) {
3237                         if (sk->sk_state != TCP_CLOSE) {
3238                                 err = -EINVAL;
3239                                 break;
3240                         }
3241                         tp->window_clamp = 0;
3242                 } else
3243                         tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
3244                                                 SOCK_MIN_RCVBUF / 2 : val;
3245                 break;
3246
3247         case TCP_QUICKACK:
3248                 __tcp_sock_set_quickack(sk, val);
3249                 break;
3250
3251 #ifdef CONFIG_TCP_MD5SIG
3252         case TCP_MD5SIG:
3253         case TCP_MD5SIG_EXT:
3254                 err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
3255                 break;
3256 #endif
3257         case TCP_USER_TIMEOUT:
3258                 /* Cap the max time in ms TCP will retry or probe the window
3259                  * before giving up and aborting (ETIMEDOUT) a connection.
3260                  */
3261                 if (val < 0)
3262                         err = -EINVAL;
3263                 else
3264                         icsk->icsk_user_timeout = val;
3265                 break;
3266
3267         case TCP_FASTOPEN:
3268                 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
3269                     TCPF_LISTEN))) {
3270                         tcp_fastopen_init_key_once(net);
3271
3272                         fastopen_queue_tune(sk, val);
3273                 } else {
3274                         err = -EINVAL;
3275                 }
3276                 break;
3277         case TCP_FASTOPEN_CONNECT:
3278                 if (val > 1 || val < 0) {
3279                         err = -EINVAL;
3280                 } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) {
3281                         if (sk->sk_state == TCP_CLOSE)
3282                                 tp->fastopen_connect = val;
3283                         else
3284                                 err = -EINVAL;
3285                 } else {
3286                         err = -EOPNOTSUPP;
3287                 }
3288                 break;
3289         case TCP_FASTOPEN_NO_COOKIE:
3290                 if (val > 1 || val < 0)
3291                         err = -EINVAL;
3292                 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3293                         err = -EINVAL;
3294                 else
3295                         tp->fastopen_no_cookie = val;
3296                 break;
3297         case TCP_TIMESTAMP:
3298                 if (!tp->repair)
3299                         err = -EPERM;
3300                 else
3301                         tp->tsoffset = val - tcp_time_stamp_raw();
3302                 break;
3303         case TCP_REPAIR_WINDOW:
3304                 err = tcp_repair_set_window(tp, optval, optlen);
3305                 break;
3306         case TCP_NOTSENT_LOWAT:
3307                 tp->notsent_lowat = val;
3308                 sk->sk_write_space(sk);
3309                 break;
3310         case TCP_INQ:
3311                 if (val > 1 || val < 0)
3312                         err = -EINVAL;
3313                 else
3314                         tp->recvmsg_inq = val;
3315                 break;
3316         case TCP_TX_DELAY:
3317                 if (val)
3318                         tcp_enable_tx_delay();
3319                 tp->tcp_tx_delay = val;
3320                 break;
3321         default:
3322                 err = -ENOPROTOOPT;
3323                 break;
3324         }
3325
3326         release_sock(sk);
3327         return err;
3328 }
3329
3330 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
3331                    unsigned int optlen)
3332 {
3333         const struct inet_connection_sock *icsk = inet_csk(sk);
3334
3335         if (level != SOL_TCP)
3336                 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
3337                                                      optval, optlen);
3338         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3339 }
3340 EXPORT_SYMBOL(tcp_setsockopt);
3341
3342 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
3343                                       struct tcp_info *info)
3344 {
3345         u64 stats[__TCP_CHRONO_MAX], total = 0;
3346         enum tcp_chrono i;
3347
3348         for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
3349                 stats[i] = tp->chrono_stat[i - 1];
3350                 if (i == tp->chrono_type)
3351                         stats[i] += tcp_jiffies32 - tp->chrono_start;
3352                 stats[i] *= USEC_PER_SEC / HZ;
3353                 total += stats[i];
3354         }
3355
3356         info->tcpi_busy_time = total;
3357         info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
3358         info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
3359 }
3360
3361 /* Return information about state of tcp endpoint in API format. */
3362 void tcp_get_info(struct sock *sk, struct tcp_info *info)
3363 {
3364         const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
3365         const struct inet_connection_sock *icsk = inet_csk(sk);
3366         unsigned long rate;
3367         u32 now;
3368         u64 rate64;
3369         bool slow;
3370
3371         memset(info, 0, sizeof(*info));
3372         if (sk->sk_type != SOCK_STREAM)
3373                 return;
3374
3375         info->tcpi_state = inet_sk_state_load(sk);
3376
3377         /* Report meaningful fields for all TCP states, including listeners */
3378         rate = READ_ONCE(sk->sk_pacing_rate);
3379         rate64 = (rate != ~0UL) ? rate : ~0ULL;
3380         info->tcpi_pacing_rate = rate64;
3381
3382         rate = READ_ONCE(sk->sk_max_pacing_rate);
3383         rate64 = (rate != ~0UL) ? rate : ~0ULL;
3384         info->tcpi_max_pacing_rate = rate64;
3385
3386         info->tcpi_reordering = tp->reordering;
3387         info->tcpi_snd_cwnd = tp->snd_cwnd;
3388
3389         if (info->tcpi_state == TCP_LISTEN) {
3390                 /* listeners aliased fields :
3391                  * tcpi_unacked -> Number of children ready for accept()
3392                  * tcpi_sacked  -> max backlog
3393                  */
3394                 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
3395                 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
3396                 return;
3397         }
3398
3399         slow = lock_sock_fast(sk);
3400
3401         info->tcpi_ca_state = icsk->icsk_ca_state;
3402         info->tcpi_retransmits = icsk->icsk_retransmits;
3403         info->tcpi_probes = icsk->icsk_probes_out;
3404         info->tcpi_backoff = icsk->icsk_backoff;
3405
3406         if (tp->rx_opt.tstamp_ok)
3407                 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
3408         if (tcp_is_sack(tp))
3409                 info->tcpi_options |= TCPI_OPT_SACK;
3410         if (tp->rx_opt.wscale_ok) {
3411                 info->tcpi_options |= TCPI_OPT_WSCALE;
3412                 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
3413                 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
3414         }
3415
3416         if (tp->ecn_flags & TCP_ECN_OK)
3417                 info->tcpi_options |= TCPI_OPT_ECN;
3418         if (tp->ecn_flags & TCP_ECN_SEEN)
3419                 info->tcpi_options |= TCPI_OPT_ECN_SEEN;
3420         if (tp->syn_data_acked)
3421                 info->tcpi_options |= TCPI_OPT_SYN_DATA;
3422
3423         info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
3424         info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
3425         info->tcpi_snd_mss = tp->mss_cache;
3426         info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
3427
3428         info->tcpi_unacked = tp->packets_out;
3429         info->tcpi_sacked = tp->sacked_out;
3430
3431         info->tcpi_lost = tp->lost_out;
3432         info->tcpi_retrans = tp->retrans_out;
3433
3434         now = tcp_jiffies32;
3435         info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
3436         info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
3437         info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
3438
3439         info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
3440         info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
3441         info->tcpi_rtt = tp->srtt_us >> 3;
3442         info->tcpi_rttvar = tp->mdev_us >> 2;
3443         info->tcpi_snd_ssthresh = tp->snd_ssthresh;
3444         info->tcpi_advmss = tp->advmss;
3445
3446         info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
3447         info->tcpi_rcv_space = tp->rcvq_space.space;
3448
3449         info->tcpi_total_retrans = tp->total_retrans;
3450
3451         info->tcpi_bytes_acked = tp->bytes_acked;
3452         info->tcpi_bytes_received = tp->bytes_received;
3453         info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
3454         tcp_get_info_chrono_stats(tp, info);
3455
3456         info->tcpi_segs_out = tp->segs_out;
3457         info->tcpi_segs_in = tp->segs_in;
3458
3459         info->tcpi_min_rtt = tcp_min_rtt(tp);
3460         info->tcpi_data_segs_in = tp->data_segs_in;
3461         info->tcpi_data_segs_out = tp->data_segs_out;
3462
3463         info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
3464         rate64 = tcp_compute_delivery_rate(tp);
3465         if (rate64)
3466                 info->tcpi_delivery_rate = rate64;
3467         info->tcpi_delivered = tp->delivered;
3468         info->tcpi_delivered_ce = tp->delivered_ce;
3469         info->tcpi_bytes_sent = tp->bytes_sent;
3470         info->tcpi_bytes_retrans = tp->bytes_retrans;
3471         info->tcpi_dsack_dups = tp->dsack_dups;
3472         info->tcpi_reord_seen = tp->reord_seen;
3473         info->tcpi_rcv_ooopack = tp->rcv_ooopack;
3474         info->tcpi_snd_wnd = tp->snd_wnd;
3475         info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
3476         unlock_sock_fast(sk, slow);
3477 }
3478 EXPORT_SYMBOL_GPL(tcp_get_info);
3479
3480 static size_t tcp_opt_stats_get_size(void)
3481 {
3482         return
3483                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */
3484                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */
3485                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */
3486                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */
3487                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */
3488                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */
3489                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */
3490                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */
3491                 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */
3492                 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */
3493                 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */
3494                 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */
3495                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */
3496                 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */
3497                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */
3498                 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */
3499                 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */
3500                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */
3501                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */
3502                 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */
3503                 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */
3504                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */
3505                 nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */
3506                 nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */
3507                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */
3508                 0;
3509 }
3510
3511 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
3512                                                const struct sk_buff *orig_skb)
3513 {
3514         const struct tcp_sock *tp = tcp_sk(sk);
3515         struct sk_buff *stats;
3516         struct tcp_info info;
3517         unsigned long rate;
3518         u64 rate64;
3519
3520         stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
3521         if (!stats)
3522                 return NULL;
3523
3524         tcp_get_info_chrono_stats(tp, &info);
3525         nla_put_u64_64bit(stats, TCP_NLA_BUSY,
3526                           info.tcpi_busy_time, TCP_NLA_PAD);
3527         nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
3528                           info.tcpi_rwnd_limited, TCP_NLA_PAD);
3529         nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
3530                           info.tcpi_sndbuf_limited, TCP_NLA_PAD);
3531         nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
3532                           tp->data_segs_out, TCP_NLA_PAD);
3533         nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
3534                           tp->total_retrans, TCP_NLA_PAD);
3535
3536         rate = READ_ONCE(sk->sk_pacing_rate);
3537         rate64 = (rate != ~0UL) ? rate : ~0ULL;
3538         nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
3539
3540         rate64 = tcp_compute_delivery_rate(tp);
3541         nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
3542
3543         nla_put_u32(stats, TCP_NLA_SND_CWND, tp->snd_cwnd);
3544         nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering);
3545         nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp));
3546
3547         nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits);
3548         nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited);
3549         nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh);
3550         nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered);
3551         nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce);
3552
3553         nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una);
3554         nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
3555
3556         nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent,
3557                           TCP_NLA_PAD);
3558         nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans,
3559                           TCP_NLA_PAD);
3560         nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
3561         nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
3562         nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
3563         nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash);
3564         nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
3565                     max_t(int, 0, tp->write_seq - tp->snd_nxt));
3566         nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
3567                           TCP_NLA_PAD);
3568
3569         return stats;
3570 }
3571
3572 static int do_tcp_getsockopt(struct sock *sk, int level,
3573                 int optname, char __user *optval, int __user *optlen)
3574 {
3575         struct inet_connection_sock *icsk = inet_csk(sk);
3576         struct tcp_sock *tp = tcp_sk(sk);
3577         struct net *net = sock_net(sk);
3578         int val, len;
3579
3580         if (get_user(len, optlen))
3581                 return -EFAULT;
3582
3583         len = min_t(unsigned int, len, sizeof(int));
3584
3585         if (len < 0)
3586                 return -EINVAL;
3587
3588         switch (optname) {
3589         case TCP_MAXSEG:
3590                 val = tp->mss_cache;
3591                 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3592                         val = tp->rx_opt.user_mss;
3593                 if (tp->repair)
3594                         val = tp->rx_opt.mss_clamp;
3595                 break;
3596         case TCP_NODELAY:
3597                 val = !!(tp->nonagle&TCP_NAGLE_OFF);
3598                 break;
3599         case TCP_CORK:
3600                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
3601                 break;
3602         case TCP_KEEPIDLE:
3603                 val = keepalive_time_when(tp) / HZ;
3604                 break;
3605         case TCP_KEEPINTVL:
3606                 val = keepalive_intvl_when(tp) / HZ;
3607                 break;
3608         case TCP_KEEPCNT:
3609                 val = keepalive_probes(tp);
3610                 break;
3611         case TCP_SYNCNT:
3612                 val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries;
3613                 break;
3614         case TCP_LINGER2:
3615                 val = tp->linger2;
3616                 if (val >= 0)
3617                         val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ;
3618                 break;
3619         case TCP_DEFER_ACCEPT:
3620                 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
3621                                       TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
3622                 break;
3623         case TCP_WINDOW_CLAMP:
3624                 val = tp->window_clamp;
3625                 break;
3626         case TCP_INFO: {
3627                 struct tcp_info info;
3628
3629                 if (get_user(len, optlen))
3630                         return -EFAULT;
3631
3632                 tcp_get_info(sk, &info);
3633
3634                 len = min_t(unsigned int, len, sizeof(info));
3635                 if (put_user(len, optlen))
3636                         return -EFAULT;
3637                 if (copy_to_user(optval, &info, len))
3638                         return -EFAULT;
3639                 return 0;
3640         }
3641         case TCP_CC_INFO: {
3642                 const struct tcp_congestion_ops *ca_ops;
3643                 union tcp_cc_info info;
3644                 size_t sz = 0;
3645                 int attr;
3646
3647                 if (get_user(len, optlen))
3648                         return -EFAULT;
3649
3650                 ca_ops = icsk->icsk_ca_ops;
3651                 if (ca_ops && ca_ops->get_info)
3652                         sz = ca_ops->get_info(sk, ~0U, &attr, &info);
3653
3654                 len = min_t(unsigned int, len, sz);
3655                 if (put_user(len, optlen))
3656                         return -EFAULT;
3657                 if (copy_to_user(optval, &info, len))
3658                         return -EFAULT;
3659                 return 0;
3660         }
3661         case TCP_QUICKACK:
3662                 val = !inet_csk_in_pingpong_mode(sk);
3663                 break;
3664
3665         case TCP_CONGESTION:
3666                 if (get_user(len, optlen))
3667                         return -EFAULT;
3668                 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
3669                 if (put_user(len, optlen))
3670                         return -EFAULT;
3671                 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
3672                         return -EFAULT;
3673                 return 0;
3674
3675         case TCP_ULP:
3676                 if (get_user(len, optlen))
3677                         return -EFAULT;
3678                 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
3679                 if (!icsk->icsk_ulp_ops) {
3680                         if (put_user(0, optlen))
3681                                 return -EFAULT;
3682                         return 0;
3683                 }
3684                 if (put_user(len, optlen))
3685                         return -EFAULT;
3686                 if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len))
3687                         return -EFAULT;
3688                 return 0;
3689
3690         case TCP_FASTOPEN_KEY: {
3691                 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
3692                 unsigned int key_len;
3693
3694                 if (get_user(len, optlen))
3695                         return -EFAULT;
3696
3697                 key_len = tcp_fastopen_get_cipher(net, icsk, key) *
3698                                 TCP_FASTOPEN_KEY_LENGTH;
3699                 len = min_t(unsigned int, len, key_len);
3700                 if (put_user(len, optlen))
3701                         return -EFAULT;
3702                 if (copy_to_user(optval, key, len))
3703                         return -EFAULT;
3704                 return 0;
3705         }
3706         case TCP_THIN_LINEAR_TIMEOUTS:
3707                 val = tp->thin_lto;
3708                 break;
3709
3710         case TCP_THIN_DUPACK:
3711                 val = 0;
3712                 break;
3713
3714         case TCP_REPAIR:
3715                 val = tp->repair;
3716                 break;
3717
3718         case TCP_REPAIR_QUEUE:
3719                 if (tp->repair)
3720                         val = tp->repair_queue;
3721                 else
3722                         return -EINVAL;
3723                 break;
3724
3725         case TCP_REPAIR_WINDOW: {
3726                 struct tcp_repair_window opt;
3727
3728                 if (get_user(len, optlen))
3729                         return -EFAULT;
3730
3731                 if (len != sizeof(opt))
3732                         return -EINVAL;
3733
3734                 if (!tp->repair)
3735                         return -EPERM;
3736
3737                 opt.snd_wl1     = tp->snd_wl1;
3738                 opt.snd_wnd     = tp->snd_wnd;
3739                 opt.max_window  = tp->max_window;
3740                 opt.rcv_wnd     = tp->rcv_wnd;
3741                 opt.rcv_wup     = tp->rcv_wup;
3742
3743                 if (copy_to_user(optval, &opt, len))
3744                         return -EFAULT;
3745                 return 0;
3746         }
3747         case TCP_QUEUE_SEQ:
3748                 if (tp->repair_queue == TCP_SEND_QUEUE)
3749                         val = tp->write_seq;
3750                 else if (tp->repair_queue == TCP_RECV_QUEUE)
3751                         val = tp->rcv_nxt;
3752                 else
3753                         return -EINVAL;
3754                 break;
3755
3756         case TCP_USER_TIMEOUT:
3757                 val = icsk->icsk_user_timeout;
3758                 break;
3759
3760         case TCP_FASTOPEN:
3761                 val = icsk->icsk_accept_queue.fastopenq.max_qlen;
3762                 break;
3763
3764         case TCP_FASTOPEN_CONNECT:
3765                 val = tp->fastopen_connect;
3766                 break;
3767
3768         case TCP_FASTOPEN_NO_COOKIE:
3769                 val = tp->fastopen_no_cookie;
3770                 break;
3771
3772         case TCP_TX_DELAY:
3773                 val = tp->tcp_tx_delay;
3774                 break;
3775
3776         case TCP_TIMESTAMP:
3777                 val = tcp_time_stamp_raw() + tp->tsoffset;
3778                 break;
3779         case TCP_NOTSENT_LOWAT:
3780                 val = tp->notsent_lowat;
3781                 break;
3782         case TCP_INQ:
3783                 val = tp->recvmsg_inq;
3784                 break;
3785         case TCP_SAVE_SYN:
3786                 val = tp->save_syn;
3787                 break;
3788         case TCP_SAVED_SYN: {
3789                 if (get_user(len, optlen))
3790                         return -EFAULT;
3791
3792                 lock_sock(sk);
3793                 if (tp->saved_syn) {
3794                         if (len < tcp_saved_syn_len(tp->saved_syn)) {
3795                                 if (put_user(tcp_saved_syn_len(tp->saved_syn),
3796                                              optlen)) {
3797                                         release_sock(sk);
3798                                         return -EFAULT;
3799                                 }
3800                                 release_sock(sk);
3801                                 return -EINVAL;
3802                         }
3803                         len = tcp_saved_syn_len(tp->saved_syn);
3804                         if (put_user(len, optlen)) {
3805                                 release_sock(sk);
3806                                 return -EFAULT;
3807                         }
3808                         if (copy_to_user(optval, tp->saved_syn->data, len)) {
3809                                 release_sock(sk);
3810                                 return -EFAULT;
3811                         }
3812                         tcp_saved_syn_free(tp);
3813                         release_sock(sk);
3814                 } else {
3815                         release_sock(sk);
3816                         len = 0;
3817                         if (put_user(len, optlen))
3818                                 return -EFAULT;
3819                 }
3820                 return 0;
3821         }
3822 #ifdef CONFIG_MMU
3823         case TCP_ZEROCOPY_RECEIVE: {
3824                 struct tcp_zerocopy_receive zc;
3825                 int err;
3826
3827                 if (get_user(len, optlen))
3828                         return -EFAULT;
3829                 if (len < offsetofend(struct tcp_zerocopy_receive, length))
3830                         return -EINVAL;
3831                 if (len > sizeof(zc)) {
3832                         len = sizeof(zc);
3833                         if (put_user(len, optlen))
3834                                 return -EFAULT;
3835                 }
3836                 if (copy_from_user(&zc, optval, len))
3837                         return -EFAULT;
3838                 lock_sock(sk);
3839                 err = tcp_zerocopy_receive(sk, &zc);
3840                 release_sock(sk);
3841                 if (len == sizeof(zc))
3842                         goto zerocopy_rcv_sk_err;
3843                 switch (len) {
3844                 case offsetofend(struct tcp_zerocopy_receive, err):
3845                         goto zerocopy_rcv_sk_err;
3846                 case offsetofend(struct tcp_zerocopy_receive, inq):
3847                         goto zerocopy_rcv_inq;
3848                 case offsetofend(struct tcp_zerocopy_receive, length):
3849                 default:
3850                         goto zerocopy_rcv_out;
3851                 }
3852 zerocopy_rcv_sk_err:
3853                 if (!err)
3854                         zc.err = sock_error(sk);
3855 zerocopy_rcv_inq:
3856                 zc.inq = tcp_inq_hint(sk);
3857 zerocopy_rcv_out:
3858                 if (!err && copy_to_user(optval, &zc, len))
3859                         err = -EFAULT;
3860                 return err;
3861         }
3862 #endif
3863         default:
3864                 return -ENOPROTOOPT;
3865         }
3866
3867         if (put_user(len, optlen))
3868                 return -EFAULT;
3869         if (copy_to_user(optval, &val, len))
3870                 return -EFAULT;
3871         return 0;
3872 }
3873
3874 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
3875                    int __user *optlen)
3876 {
3877         struct inet_connection_sock *icsk = inet_csk(sk);
3878
3879         if (level != SOL_TCP)
3880                 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
3881                                                      optval, optlen);
3882         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3883 }
3884 EXPORT_SYMBOL(tcp_getsockopt);
3885
3886 #ifdef CONFIG_TCP_MD5SIG
3887 static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
3888 static DEFINE_MUTEX(tcp_md5sig_mutex);
3889 static bool tcp_md5sig_pool_populated = false;
3890
3891 static void __tcp_alloc_md5sig_pool(void)
3892 {
3893         struct crypto_ahash *hash;
3894         int cpu;
3895
3896         hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC);
3897         if (IS_ERR(hash))
3898                 return;
3899
3900         for_each_possible_cpu(cpu) {
3901                 void *scratch = per_cpu(tcp_md5sig_pool, cpu).scratch;
3902                 struct ahash_request *req;
3903
3904                 if (!scratch) {
3905                         scratch = kmalloc_node(sizeof(union tcp_md5sum_block) +
3906                                                sizeof(struct tcphdr),
3907                                                GFP_KERNEL,
3908                                                cpu_to_node(cpu));
3909                         if (!scratch)
3910                                 return;
3911                         per_cpu(tcp_md5sig_pool, cpu).scratch = scratch;
3912                 }
3913                 if (per_cpu(tcp_md5sig_pool, cpu).md5_req)
3914                         continue;
3915
3916                 req = ahash_request_alloc(hash, GFP_KERNEL);
3917                 if (!req)
3918                         return;
3919
3920                 ahash_request_set_callback(req, 0, NULL, NULL);
3921
3922                 per_cpu(tcp_md5sig_pool, cpu).md5_req = req;
3923         }
3924         /* before setting tcp_md5sig_pool_populated, we must commit all writes
3925          * to memory. See smp_rmb() in tcp_get_md5sig_pool()
3926          */
3927         smp_wmb();
3928         tcp_md5sig_pool_populated = true;
3929 }
3930
3931 bool tcp_alloc_md5sig_pool(void)
3932 {
3933         if (unlikely(!tcp_md5sig_pool_populated)) {
3934                 mutex_lock(&tcp_md5sig_mutex);
3935
3936                 if (!tcp_md5sig_pool_populated) {
3937                         __tcp_alloc_md5sig_pool();
3938                         if (tcp_md5sig_pool_populated)
3939                                 static_branch_inc(&tcp_md5_needed);
3940                 }
3941
3942                 mutex_unlock(&tcp_md5sig_mutex);
3943         }
3944         return tcp_md5sig_pool_populated;
3945 }
3946 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3947
3948
3949 /**
3950  *      tcp_get_md5sig_pool - get md5sig_pool for this user
3951  *
3952  *      We use percpu structure, so if we succeed, we exit with preemption
3953  *      and BH disabled, to make sure another thread or softirq handling
3954  *      wont try to get same context.
3955  */
3956 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
3957 {
3958         local_bh_disable();
3959
3960         if (tcp_md5sig_pool_populated) {
3961                 /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
3962                 smp_rmb();
3963                 return this_cpu_ptr(&tcp_md5sig_pool);
3964         }
3965         local_bh_enable();
3966         return NULL;
3967 }
3968 EXPORT_SYMBOL(tcp_get_md5sig_pool);
3969
3970 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
3971                           const struct sk_buff *skb, unsigned int header_len)
3972 {
3973         struct scatterlist sg;
3974         const struct tcphdr *tp = tcp_hdr(skb);
3975         struct ahash_request *req = hp->md5_req;
3976         unsigned int i;
3977         const unsigned int head_data_len = skb_headlen(skb) > header_len ?
3978                                            skb_headlen(skb) - header_len : 0;
3979         const struct skb_shared_info *shi = skb_shinfo(skb);
3980         struct sk_buff *frag_iter;
3981
3982         sg_init_table(&sg, 1);
3983
3984         sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
3985         ahash_request_set_crypt(req, &sg, NULL, head_data_len);
3986         if (crypto_ahash_update(req))
3987                 return 1;
3988
3989         for (i = 0; i < shi->nr_frags; ++i) {
3990                 const skb_frag_t *f = &shi->frags[i];
3991                 unsigned int offset = skb_frag_off(f);
3992                 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3993
3994                 sg_set_page(&sg, page, skb_frag_size(f),
3995                             offset_in_page(offset));
3996                 ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
3997                 if (crypto_ahash_update(req))
3998                         return 1;
3999         }
4000
4001         skb_walk_frags(skb, frag_iter)
4002                 if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
4003                         return 1;
4004
4005         return 0;
4006 }
4007 EXPORT_SYMBOL(tcp_md5_hash_skb_data);
4008
4009 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
4010 {
4011         u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */
4012         struct scatterlist sg;
4013
4014         sg_init_one(&sg, key->key, keylen);
4015         ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen);
4016
4017         /* We use data_race() because tcp_md5_do_add() might change key->key under us */
4018         return data_race(crypto_ahash_update(hp->md5_req));
4019 }
4020 EXPORT_SYMBOL(tcp_md5_hash_key);
4021
4022 #endif
4023
4024 void tcp_done(struct sock *sk)
4025 {
4026         struct request_sock *req;
4027
4028         /* We might be called with a new socket, after
4029          * inet_csk_prepare_forced_close() has been called
4030          * so we can not use lockdep_sock_is_held(sk)
4031          */
4032         req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
4033
4034         if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
4035                 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
4036
4037         tcp_set_state(sk, TCP_CLOSE);
4038         tcp_clear_xmit_timers(sk);
4039         if (req)
4040                 reqsk_fastopen_remove(sk, req, false);
4041
4042         sk->sk_shutdown = SHUTDOWN_MASK;
4043
4044         if (!sock_flag(sk, SOCK_DEAD))
4045                 sk->sk_state_change(sk);
4046         else
4047                 inet_csk_destroy_sock(sk);
4048 }
4049 EXPORT_SYMBOL_GPL(tcp_done);
4050
4051 int tcp_abort(struct sock *sk, int err)
4052 {
4053         if (!sk_fullsock(sk)) {
4054                 if (sk->sk_state == TCP_NEW_SYN_RECV) {
4055                         struct request_sock *req = inet_reqsk(sk);
4056
4057                         local_bh_disable();
4058                         inet_csk_reqsk_queue_drop(req->rsk_listener, req);
4059                         local_bh_enable();
4060                         return 0;
4061                 }
4062                 return -EOPNOTSUPP;
4063         }
4064
4065         /* Don't race with userspace socket closes such as tcp_close. */
4066         lock_sock(sk);
4067
4068         if (sk->sk_state == TCP_LISTEN) {
4069                 tcp_set_state(sk, TCP_CLOSE);
4070                 inet_csk_listen_stop(sk);
4071         }
4072
4073         /* Don't race with BH socket closes such as inet_csk_listen_stop. */
4074         local_bh_disable();
4075         bh_lock_sock(sk);
4076
4077         if (!sock_flag(sk, SOCK_DEAD)) {
4078                 sk->sk_err = err;
4079                 /* This barrier is coupled with smp_rmb() in tcp_poll() */
4080                 smp_wmb();
4081                 sk->sk_error_report(sk);
4082                 if (tcp_need_reset(sk->sk_state))
4083                         tcp_send_active_reset(sk, GFP_ATOMIC);
4084                 tcp_done(sk);
4085         }
4086
4087         bh_unlock_sock(sk);
4088         local_bh_enable();
4089         tcp_write_queue_purge(sk);
4090         release_sock(sk);
4091         return 0;
4092 }
4093 EXPORT_SYMBOL_GPL(tcp_abort);
4094
4095 extern struct tcp_congestion_ops tcp_reno;
4096
4097 static __initdata unsigned long thash_entries;
4098 static int __init set_thash_entries(char *str)
4099 {
4100         ssize_t ret;
4101
4102         if (!str)
4103                 return 0;
4104
4105         ret = kstrtoul(str, 0, &thash_entries);
4106         if (ret)
4107                 return 0;
4108
4109         return 1;
4110 }
4111 __setup("thash_entries=", set_thash_entries);
4112
4113 static void __init tcp_init_mem(void)
4114 {
4115         unsigned long limit = nr_free_buffer_pages() / 16;
4116
4117         limit = max(limit, 128UL);
4118         sysctl_tcp_mem[0] = limit / 4 * 3;              /* 4.68 % */
4119         sysctl_tcp_mem[1] = limit;                      /* 6.25 % */
4120         sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;      /* 9.37 % */
4121 }
4122
4123 void __init tcp_init(void)
4124 {
4125         int max_rshare, max_wshare, cnt;
4126         unsigned long limit;
4127         unsigned int i;
4128
4129         BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
4130         BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
4131                      sizeof_field(struct sk_buff, cb));
4132
4133         percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
4134         percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
4135         inet_hashinfo_init(&tcp_hashinfo);
4136         inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
4137                             thash_entries, 21,  /* one slot per 2 MB*/
4138                             0, 64 * 1024);
4139         tcp_hashinfo.bind_bucket_cachep =
4140                 kmem_cache_create("tcp_bind_bucket",
4141                                   sizeof(struct inet_bind_bucket), 0,
4142                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
4143
4144         /* Size and allocate the main established and bind bucket
4145          * hash tables.
4146          *
4147          * The methodology is similar to that of the buffer cache.
4148          */
4149         tcp_hashinfo.ehash =
4150                 alloc_large_system_hash("TCP established",
4151                                         sizeof(struct inet_ehash_bucket),
4152                                         thash_entries,
4153                                         17, /* one slot per 128 KB of memory */
4154                                         0,
4155                                         NULL,
4156                                         &tcp_hashinfo.ehash_mask,
4157                                         0,
4158                                         thash_entries ? 0 : 512 * 1024);
4159         for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
4160                 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
4161
4162         if (inet_ehash_locks_alloc(&tcp_hashinfo))
4163                 panic("TCP: failed to alloc ehash_locks");
4164         tcp_hashinfo.bhash =
4165                 alloc_large_system_hash("TCP bind",
4166                                         sizeof(struct inet_bind_hashbucket),
4167                                         tcp_hashinfo.ehash_mask + 1,
4168                                         17, /* one slot per 128 KB of memory */
4169                                         0,
4170                                         &tcp_hashinfo.bhash_size,
4171                                         NULL,
4172                                         0,
4173                                         64 * 1024);
4174         tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
4175         for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
4176                 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
4177                 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
4178         }
4179
4180
4181         cnt = tcp_hashinfo.ehash_mask + 1;
4182         sysctl_tcp_max_orphans = cnt / 2;
4183
4184         tcp_init_mem();
4185         /* Set per-socket limits to no more than 1/128 the pressure threshold */
4186         limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
4187         max_wshare = min(4UL*1024*1024, limit);
4188         max_rshare = min(6UL*1024*1024, limit);
4189
4190         init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
4191         init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
4192         init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
4193
4194         init_net.ipv4.sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
4195         init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
4196         init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
4197
4198         pr_info("Hash tables configured (established %u bind %u)\n",
4199                 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
4200
4201         tcp_v4_init();
4202         tcp_metrics_init();
4203         BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
4204         tcp_tasklet_init();
4205         mptcp_init();
4206 }