drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include <net/pkt_cls.h>
  49 #include "stmmac_ptp.h"
  50 #include "stmmac.h"
  51 #include <linux/reset.h>
  52 #include <linux/of_mdio.h>
  53 #include "dwmac1000.h"
  54 #include "dwxgmac2.h"
  55 #include "hwif.h"
  56
  57 #define STMMAC_ALIGN(x)         __ALIGN_KERNEL(x, SMP_CACHE_BYTES)
  58 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  59
  60 /* Module parameters */
  61 #define TX_TIMEO        5000
  62 static int watchdog = TX_TIMEO;
  63 module_param(watchdog, int, 0644);
  64 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  65
  66 static int debug = -1;
  67 module_param(debug, int, 0644);
  68 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  69
  70 static int phyaddr = -1;
  71 module_param(phyaddr, int, 0444);
  72 MODULE_PARM_DESC(phyaddr, "Physical device address");
  73
  74 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  75 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  76
  77 static int flow_ctrl = FLOW_OFF;
  78 module_param(flow_ctrl, int, 0644);
  79 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  80
  81 static int pause = PAUSE_TIME;
  82 module_param(pause, int, 0644);
  83 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  84
  85 #define TC_DEFAULT 64
  86 static int tc = TC_DEFAULT;
  87 module_param(tc, int, 0644);
  88 MODULE_PARM_DESC(tc, "DMA threshold control value");
  89
  90 #define DEFAULT_BUFSIZE 1536
  91 static int buf_sz = DEFAULT_BUFSIZE;
  92 module_param(buf_sz, int, 0644);
  93 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  94
  95 #define STMMAC_RX_COPYBREAK     256
  96
  97 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  98                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  99                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
 100
 101 #define STMMAC_DEFAULT_LPI_TIMER        1000
 102 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 103 module_param(eee_timer, int, 0644);
 104 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 105 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 106
 107 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 108  * but allow user to force to use the chain instead of the ring
 109  */
 110 static unsigned int chain_mode;
 111 module_param(chain_mode, int, 0444);
 112 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 113
 114 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 115
 116 #ifdef CONFIG_DEBUG_FS
 117 static int stmmac_init_fs(struct net_device *dev);
 118 static void stmmac_exit_fs(struct net_device *dev);
 119 #endif
 120
 121 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 122
 123 /**
 124  * stmmac_verify_args - verify the driver parameters.
 125  * Description: it checks the driver parameters and set a default in case of
 126  * errors.
 127  */
 128 static void stmmac_verify_args(void)
 129 {
 130         if (unlikely(watchdog < 0))
 131                 watchdog = TX_TIMEO;
 132         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 133                 buf_sz = DEFAULT_BUFSIZE;
 134         if (unlikely(flow_ctrl > 1))
 135                 flow_ctrl = FLOW_AUTO;
 136         else if (likely(flow_ctrl < 0))
 137                 flow_ctrl = FLOW_OFF;
 138         if (unlikely((pause < 0) || (pause > 0xffff)))
 139                 pause = PAUSE_TIME;
 140         if (eee_timer < 0)
 141                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 142 }
 143
 144 /**
 145  * stmmac_disable_all_queues - Disable all queues
 146  * @priv: driver private structure
 147  */
 148 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
 149 {
 150         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 151         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 152         u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
 153         u32 queue;
 154
 155         for (queue = 0; queue < maxq; queue++) {
 156                 struct stmmac_channel *ch = &priv->channel[queue];
 157
 158                 napi_disable(&ch->napi);
 159         }
 160 }
 161
 162 /**
 163  * stmmac_enable_all_queues - Enable all queues
 164  * @priv: driver private structure
 165  */
 166 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
 167 {
 168         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 169         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 170         u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
 171         u32 queue;
 172
 173         for (queue = 0; queue < maxq; queue++) {
 174                 struct stmmac_channel *ch = &priv->channel[queue];
 175
 176                 napi_enable(&ch->napi);
 177         }
 178 }
 179
 180 /**
 181  * stmmac_stop_all_queues - Stop all queues
 182  * @priv: driver private structure
 183  */
 184 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
 185 {
 186         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 187         u32 queue;
 188
 189         for (queue = 0; queue < tx_queues_cnt; queue++)
 190                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
 191 }
 192
 193 /**
 194  * stmmac_start_all_queues - Start all queues
 195  * @priv: driver private structure
 196  */
 197 static void stmmac_start_all_queues(struct stmmac_priv *priv)
 198 {
 199         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 200         u32 queue;
 201
 202         for (queue = 0; queue < tx_queues_cnt; queue++)
 203                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
 204 }
 205
 206 static void stmmac_service_event_schedule(struct stmmac_priv *priv)
 207 {
 208         if (!test_bit(STMMAC_DOWN, &priv->state) &&
 209             !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
 210                 queue_work(priv->wq, &priv->service_task);
 211 }
 212
 213 static void stmmac_global_err(struct stmmac_priv *priv)
 214 {
 215         netif_carrier_off(priv->dev);
 216         set_bit(STMMAC_RESET_REQUESTED, &priv->state);
 217         stmmac_service_event_schedule(priv);
 218 }
 219
 220 /**
 221  * stmmac_clk_csr_set - dynamically set the MDC clock
 222  * @priv: driver private structure
 223  * Description: this is to dynamically set the MDC clock according to the csr
 224  * clock input.
 225  * Note:
 226  *      If a specific clk_csr value is passed from the platform
 227  *      this means that the CSR Clock Range selection cannot be
 228  *      changed at run-time and it is fixed (as reported in the driver
 229  *      documentation). Viceversa the driver will try to set the MDC
 230  *      clock dynamically according to the actual clock input.
 231  */
 232 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 233 {
 234         u32 clk_rate;
 235
 236         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 237
 238         /* Platform provided default clk_csr would be assumed valid
 239          * for all other cases except for the below mentioned ones.
 240          * For values higher than the IEEE 802.3 specified frequency
 241          * we can not estimate the proper divider as it is not known
 242          * the frequency of clk_csr_i. So we do not change the default
 243          * divider.
 244          */
 245         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 246                 if (clk_rate < CSR_F_35M)
 247                         priv->clk_csr = STMMAC_CSR_20_35M;
 248                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 249                         priv->clk_csr = STMMAC_CSR_35_60M;
 250                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 251                         priv->clk_csr = STMMAC_CSR_60_100M;
 252                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 253                         priv->clk_csr = STMMAC_CSR_100_150M;
 254                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 255                         priv->clk_csr = STMMAC_CSR_150_250M;
 256                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 257                         priv->clk_csr = STMMAC_CSR_250_300M;
 258         }
 259
 260         if (priv->plat->has_sun8i) {
 261                 if (clk_rate > 160000000)
 262                         priv->clk_csr = 0x03;
 263                 else if (clk_rate > 80000000)
 264                         priv->clk_csr = 0x02;
 265                 else if (clk_rate > 40000000)
 266                         priv->clk_csr = 0x01;
 267                 else
 268                         priv->clk_csr = 0;
 269         }
 270
 271         if (priv->plat->has_xgmac) {
 272                 if (clk_rate > 400000000)
 273                         priv->clk_csr = 0x5;
 274                 else if (clk_rate > 350000000)
 275                         priv->clk_csr = 0x4;
 276                 else if (clk_rate > 300000000)
 277                         priv->clk_csr = 0x3;
 278                 else if (clk_rate > 250000000)
 279                         priv->clk_csr = 0x2;
 280                 else if (clk_rate > 150000000)
 281                         priv->clk_csr = 0x1;
 282                 else
 283                         priv->clk_csr = 0x0;
 284         }
 285 }
 286
 287 static void print_pkt(unsigned char *buf, int len)
 288 {
 289         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 290         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 291 }
 292
 293 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 294 {
 295         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 296         u32 avail;
 297
 298         if (tx_q->dirty_tx > tx_q->cur_tx)
 299                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 300         else
 301                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 302
 303         return avail;
 304 }
 305
 306 /**
 307  * stmmac_rx_dirty - Get RX queue dirty
 308  * @priv: driver private structure
 309  * @queue: RX queue index
 310  */
 311 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 312 {
 313         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 314         u32 dirty;
 315
 316         if (rx_q->dirty_rx <= rx_q->cur_rx)
 317                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 318         else
 319                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 320
 321         return dirty;
 322 }
 323
 324 /**
 325  * stmmac_hw_fix_mac_speed - callback for speed selection
 326  * @priv: driver private structure
 327  * Description: on some platforms (e.g. ST), some HW system configuration
 328  * registers have to be set according to the link speed negotiated.
 329  */
 330 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 331 {
 332         struct net_device *ndev = priv->dev;
 333         struct phy_device *phydev = ndev->phydev;
 334
 335         if (likely(priv->plat->fix_mac_speed))
 336                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 337 }
 338
 339 /**
 340  * stmmac_enable_eee_mode - check and enter in LPI mode
 341  * @priv: driver private structure
 342  * Description: this function is to verify and enter in LPI mode in case of
 343  * EEE.
 344  */
 345 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 346 {
 347         u32 tx_cnt = priv->plat->tx_queues_to_use;
 348         u32 queue;
 349
 350         /* check if all TX queues have the work finished */
 351         for (queue = 0; queue < tx_cnt; queue++) {
 352                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 353
 354                 if (tx_q->dirty_tx != tx_q->cur_tx)
 355                         return; /* still unfinished work */
 356         }
 357
 358         /* Check and enter in LPI mode */
 359         if (!priv->tx_path_in_lpi_mode)
 360                 stmmac_set_eee_mode(priv, priv->hw,
 361                                 priv->plat->en_tx_lpi_clockgating);
 362 }
 363
 364 /**
 365  * stmmac_disable_eee_mode - disable and exit from LPI mode
 366  * @priv: driver private structure
 367  * Description: this function is to exit and disable EEE in case of
 368  * LPI state is true. This is called by the xmit.
 369  */
 370 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 371 {
 372         stmmac_reset_eee_mode(priv, priv->hw);
 373         del_timer_sync(&priv->eee_ctrl_timer);
 374         priv->tx_path_in_lpi_mode = false;
 375 }
 376
 377 /**
 378  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 379  * @arg : data hook
 380  * Description:
 381  *  if there is no data transfer and if we are not in LPI state,
 382  *  then MAC Transmitter can be moved to LPI state.
 383  */
 384 static void stmmac_eee_ctrl_timer(struct timer_list *t)
 385 {
 386         struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
 387
 388         stmmac_enable_eee_mode(priv);
 389         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 390 }
 391
 392 /**
 393  * stmmac_eee_init - init EEE
 394  * @priv: driver private structure
 395  * Description:
 396  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 397  *  can also manage EEE, this function enable the LPI state and start related
 398  *  timer.
 399  */
 400 bool stmmac_eee_init(struct stmmac_priv *priv)
 401 {
 402         struct net_device *ndev = priv->dev;
 403         int interface = priv->plat->interface;
 404         bool ret = false;
 405
 406         if ((interface != PHY_INTERFACE_MODE_MII) &&
 407             (interface != PHY_INTERFACE_MODE_GMII) &&
 408             !phy_interface_mode_is_rgmii(interface))
 409                 goto out;
 410
 411         /* Using PCS we cannot dial with the phy registers at this stage
 412          * so we do not support extra feature like EEE.
 413          */
 414         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 415             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 416             (priv->hw->pcs == STMMAC_PCS_RTBI))
 417                 goto out;
 418
 419         /* MAC core supports the EEE feature. */
 420         if (priv->dma_cap.eee) {
 421                 int tx_lpi_timer = priv->tx_lpi_timer;
 422
 423                 /* Check if the PHY supports EEE */
 424                 if (phy_init_eee(ndev->phydev, 1)) {
 425                         /* To manage at run-time if the EEE cannot be supported
 426                          * anymore (for example because the lp caps have been
 427                          * changed).
 428                          * In that case the driver disable own timers.
 429                          */
 430                         mutex_lock(&priv->lock);
 431                         if (priv->eee_active) {
 432                                 netdev_dbg(priv->dev, "disable EEE\n");
 433                                 del_timer_sync(&priv->eee_ctrl_timer);
 434                                 stmmac_set_eee_timer(priv, priv->hw, 0,
 435                                                 tx_lpi_timer);
 436                         }
 437                         priv->eee_active = 0;
 438                         mutex_unlock(&priv->lock);
 439                         goto out;
 440                 }
 441                 /* Activate the EEE and start timers */
 442                 mutex_lock(&priv->lock);
 443                 if (!priv->eee_active) {
 444                         priv->eee_active = 1;
 445                         timer_setup(&priv->eee_ctrl_timer,
 446                                     stmmac_eee_ctrl_timer, 0);
 447                         mod_timer(&priv->eee_ctrl_timer,
 448                                   STMMAC_LPI_T(eee_timer));
 449
 450                         stmmac_set_eee_timer(priv, priv->hw,
 451                                         STMMAC_DEFAULT_LIT_LS, tx_lpi_timer);
 452                 }
 453                 /* Set HW EEE according to the speed */
 454                 stmmac_set_eee_pls(priv, priv->hw, ndev->phydev->link);
 455
 456                 ret = true;
 457                 mutex_unlock(&priv->lock);
 458
 459                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 460         }
 461 out:
 462         return ret;
 463 }
 464
 465 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 466  * @priv: driver private structure
 467  * @p : descriptor pointer
 468  * @skb : the socket buffer
 469  * Description :
 470  * This function will read timestamp from the descriptor & pass it to stack.
 471  * and also perform some sanity checks.
 472  */
 473 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 474                                    struct dma_desc *p, struct sk_buff *skb)
 475 {
 476         struct skb_shared_hwtstamps shhwtstamp;
 477         u64 ns;
 478
 479         if (!priv->hwts_tx_en)
 480                 return;
 481
 482         /* exit if skb doesn't support hw tstamp */
 483         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 484                 return;
 485
 486         /* check tx tstamp status */
 487         if (stmmac_get_tx_timestamp_status(priv, p)) {
 488                 /* get the valid tstamp */
 489                 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
 490
 491                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 492                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 493
 494                 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 495                 /* pass tstamp to stack */
 496                 skb_tstamp_tx(skb, &shhwtstamp);
 497         }
 498
 499         return;
 500 }
 501
 502 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 503  * @priv: driver private structure
 504  * @p : descriptor pointer
 505  * @np : next descriptor pointer
 506  * @skb : the socket buffer
 507  * Description :
 508  * This function will read received packet's timestamp from the descriptor
 509  * and pass it to stack. It also perform some sanity checks.
 510  */
 511 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 512                                    struct dma_desc *np, struct sk_buff *skb)
 513 {
 514         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 515         struct dma_desc *desc = p;
 516         u64 ns;
 517
 518         if (!priv->hwts_rx_en)
 519                 return;
 520         /* For GMAC4, the valid timestamp is from CTX next desc. */
 521         if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
 522                 desc = np;
 523
 524         /* Check if timestamp is available */
 525         if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
 526                 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
 527                 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 528                 shhwtstamp = skb_hwtstamps(skb);
 529                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 530                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 531         } else  {
 532                 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
 533         }
 534 }
 535
 536 /**
 537  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 538  *  @dev: device pointer.
 539  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 540  *  a proprietary structure used to pass information to the driver.
 541  *  Description:
 542  *  This function configures the MAC to enable/disable both outgoing(TX)
 543  *  and incoming(RX) packets time stamping based on user input.
 544  *  Return Value:
 545  *  0 on success and an appropriate -ve integer on failure.
 546  */
 547 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 548 {
 549         struct stmmac_priv *priv = netdev_priv(dev);
 550         struct hwtstamp_config config;
 551         struct timespec64 now;
 552         u64 temp = 0;
 553         u32 ptp_v2 = 0;
 554         u32 tstamp_all = 0;
 555         u32 ptp_over_ipv4_udp = 0;
 556         u32 ptp_over_ipv6_udp = 0;
 557         u32 ptp_over_ethernet = 0;
 558         u32 snap_type_sel = 0;
 559         u32 ts_master_en = 0;
 560         u32 ts_event_en = 0;
 561         u32 value = 0;
 562         u32 sec_inc;
 563         bool xmac;
 564
 565         xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
 566
 567         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 568                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 569                 priv->hwts_tx_en = 0;
 570                 priv->hwts_rx_en = 0;
 571
 572                 return -EOPNOTSUPP;
 573         }
 574
 575         if (copy_from_user(&config, ifr->ifr_data,
 576                            sizeof(struct hwtstamp_config)))
 577                 return -EFAULT;
 578
 579         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 580                    __func__, config.flags, config.tx_type, config.rx_filter);
 581
 582         /* reserved for future extensions */
 583         if (config.flags)
 584                 return -EINVAL;
 585
 586         if (config.tx_type != HWTSTAMP_TX_OFF &&
 587             config.tx_type != HWTSTAMP_TX_ON)
 588                 return -ERANGE;
 589
 590         if (priv->adv_ts) {
 591                 switch (config.rx_filter) {
 592                 case HWTSTAMP_FILTER_NONE:
 593                         /* time stamp no incoming packet at all */
 594                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 595                         break;
 596
 597                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 598                         /* PTP v1, UDP, any kind of event packet */
 599                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 600                         /* take time stamp for all event messages */
 601                         if (xmac)
 602                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 603                         else
 604                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 605
 606                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 607                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 608                         break;
 609
 610                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 611                         /* PTP v1, UDP, Sync packet */
 612                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 613                         /* take time stamp for SYNC messages only */
 614                         ts_event_en = PTP_TCR_TSEVNTENA;
 615
 616                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 617                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 618                         break;
 619
 620                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 621                         /* PTP v1, UDP, Delay_req packet */
 622                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 623                         /* take time stamp for Delay_Req messages only */
 624                         ts_master_en = PTP_TCR_TSMSTRENA;
 625                         ts_event_en = PTP_TCR_TSEVNTENA;
 626
 627                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 628                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 629                         break;
 630
 631                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 632                         /* PTP v2, UDP, any kind of event packet */
 633                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 634                         ptp_v2 = PTP_TCR_TSVER2ENA;
 635                         /* take time stamp for all event messages */
 636                         if (xmac)
 637                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 638                         else
 639                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 640
 641                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 642                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 643                         break;
 644
 645                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 646                         /* PTP v2, UDP, Sync packet */
 647                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 648                         ptp_v2 = PTP_TCR_TSVER2ENA;
 649                         /* take time stamp for SYNC messages only */
 650                         ts_event_en = PTP_TCR_TSEVNTENA;
 651
 652                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 653                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 654                         break;
 655
 656                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 657                         /* PTP v2, UDP, Delay_req packet */
 658                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 659                         ptp_v2 = PTP_TCR_TSVER2ENA;
 660                         /* take time stamp for Delay_Req messages only */
 661                         ts_master_en = PTP_TCR_TSMSTRENA;
 662                         ts_event_en = PTP_TCR_TSEVNTENA;
 663
 664                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 665                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 666                         break;
 667
 668                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 669                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 670                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 671                         ptp_v2 = PTP_TCR_TSVER2ENA;
 672                         /* take time stamp for all event messages */
 673                         if (xmac)
 674                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 675                         else
 676                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 677
 678                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 679                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 680                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 681                         break;
 682
 683                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 684                         /* PTP v2/802.AS1, any layer, Sync packet */
 685                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 686                         ptp_v2 = PTP_TCR_TSVER2ENA;
 687                         /* take time stamp for SYNC messages only */
 688                         ts_event_en = PTP_TCR_TSEVNTENA;
 689
 690                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 691                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 692                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 693                         break;
 694
 695                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 696                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 697                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 698                         ptp_v2 = PTP_TCR_TSVER2ENA;
 699                         /* take time stamp for Delay_Req messages only */
 700                         ts_master_en = PTP_TCR_TSMSTRENA;
 701                         ts_event_en = PTP_TCR_TSEVNTENA;
 702
 703                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 704                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 705                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 706                         break;
 707
 708                 case HWTSTAMP_FILTER_NTP_ALL:
 709                 case HWTSTAMP_FILTER_ALL:
 710                         /* time stamp any incoming packet */
 711                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 712                         tstamp_all = PTP_TCR_TSENALL;
 713                         break;
 714
 715                 default:
 716                         return -ERANGE;
 717                 }
 718         } else {
 719                 switch (config.rx_filter) {
 720                 case HWTSTAMP_FILTER_NONE:
 721                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 722                         break;
 723                 default:
 724                         /* PTP v1, UDP, any kind of event packet */
 725                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 726                         break;
 727                 }
 728         }
 729         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 730         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 731
 732         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 733                 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
 734         else {
 735                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 736                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 737                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 738                          ts_master_en | snap_type_sel);
 739                 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
 740
 741                 /* program Sub Second Increment reg */
 742                 stmmac_config_sub_second_increment(priv,
 743                                 priv->ptpaddr, priv->plat->clk_ptp_rate,
 744                                 xmac, &sec_inc);
 745                 temp = div_u64(1000000000ULL, sec_inc);
 746
 747                 /* Store sub second increment and flags for later use */
 748                 priv->sub_second_inc = sec_inc;
 749                 priv->systime_flags = value;
 750
 751                 /* calculate default added value:
 752                  * formula is :
 753                  * addend = (2^32)/freq_div_ratio;
 754                  * where, freq_div_ratio = 1e9ns/sec_inc
 755                  */
 756                 temp = (u64)(temp << 32);
 757                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 758                 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
 759
 760                 /* initialize system time */
 761                 ktime_get_real_ts64(&now);
 762
 763                 /* lower 32 bits of tv_sec are safe until y2106 */
 764                 stmmac_init_systime(priv, priv->ptpaddr,
 765                                 (u32)now.tv_sec, now.tv_nsec);
 766         }
 767
 768         return copy_to_user(ifr->ifr_data, &config,
 769                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 770 }
 771
 772 /**
 773  * stmmac_init_ptp - init PTP
 774  * @priv: driver private structure
 775  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 776  * This is done by looking at the HW cap. register.
 777  * This function also registers the ptp driver.
 778  */
 779 static int stmmac_init_ptp(struct stmmac_priv *priv)
 780 {
 781         bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
 782
 783         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 784                 return -EOPNOTSUPP;
 785
 786         priv->adv_ts = 0;
 787         /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
 788         if (xmac && priv->dma_cap.atime_stamp)
 789                 priv->adv_ts = 1;
 790         /* Dwmac 3.x core with extend_desc can support adv_ts */
 791         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 792                 priv->adv_ts = 1;
 793
 794         if (priv->dma_cap.time_stamp)
 795                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 796
 797         if (priv->adv_ts)
 798                 netdev_info(priv->dev,
 799                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 800
 801         priv->hwts_tx_en = 0;
 802         priv->hwts_rx_en = 0;
 803
 804         stmmac_ptp_register(priv);
 805
 806         return 0;
 807 }
 808
 809 static void stmmac_release_ptp(struct stmmac_priv *priv)
 810 {
 811         if (priv->plat->clk_ptp_ref)
 812                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 813         stmmac_ptp_unregister(priv);
 814 }
 815
 816 /**
 817  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 818  *  @priv: driver private structure
 819  *  Description: It is used for configuring the flow control in all queues
 820  */
 821 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 822 {
 823         u32 tx_cnt = priv->plat->tx_queues_to_use;
 824
 825         stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
 826                         priv->pause, tx_cnt);
 827 }
 828
 829 /**
 830  * stmmac_adjust_link - adjusts the link parameters
 831  * @dev: net device structure
 832  * Description: this is the helper called by the physical abstraction layer
 833  * drivers to communicate the phy link status. According the speed and duplex
 834  * this driver can invoke registered glue-logic as well.
 835  * It also invoke the eee initialization because it could happen when switch
 836  * on different networks (that are eee capable).
 837  */
 838 static void stmmac_adjust_link(struct net_device *dev)
 839 {
 840         struct stmmac_priv *priv = netdev_priv(dev);
 841         struct phy_device *phydev = dev->phydev;
 842         bool new_state = false;
 843
 844         if (!phydev)
 845                 return;
 846
 847         mutex_lock(&priv->lock);
 848
 849         if (phydev->link) {
 850                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 851
 852                 /* Now we make sure that we can be in full duplex mode.
 853                  * If not, we operate in half-duplex mode. */
 854                 if (phydev->duplex != priv->oldduplex) {
 855                         new_state = true;
 856                         if (!phydev->duplex)
 857                                 ctrl &= ~priv->hw->link.duplex;
 858                         else
 859                                 ctrl |= priv->hw->link.duplex;
 860                         priv->oldduplex = phydev->duplex;
 861                 }
 862                 /* Flow Control operation */
 863                 if (phydev->pause)
 864                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 865
 866                 if (phydev->speed != priv->speed) {
 867                         new_state = true;
 868                         ctrl &= ~priv->hw->link.speed_mask;
 869                         switch (phydev->speed) {
 870                         case SPEED_1000:
 871                                 ctrl |= priv->hw->link.speed1000;
 872                                 break;
 873                         case SPEED_100:
 874                                 ctrl |= priv->hw->link.speed100;
 875                                 break;
 876                         case SPEED_10:
 877                                 ctrl |= priv->hw->link.speed10;
 878                                 break;
 879                         default:
 880                                 netif_warn(priv, link, priv->dev,
 881                                            "broken speed: %d\n", phydev->speed);
 882                                 phydev->speed = SPEED_UNKNOWN;
 883                                 break;
 884                         }
 885                         if (phydev->speed != SPEED_UNKNOWN)
 886                                 stmmac_hw_fix_mac_speed(priv);
 887                         priv->speed = phydev->speed;
 888                 }
 889
 890                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 891
 892                 if (!priv->oldlink) {
 893                         new_state = true;
 894                         priv->oldlink = true;
 895                 }
 896         } else if (priv->oldlink) {
 897                 new_state = true;
 898                 priv->oldlink = false;
 899                 priv->speed = SPEED_UNKNOWN;
 900                 priv->oldduplex = DUPLEX_UNKNOWN;
 901         }
 902
 903         if (new_state && netif_msg_link(priv))
 904                 phy_print_status(phydev);
 905
 906         mutex_unlock(&priv->lock);
 907
 908         if (phydev->is_pseudo_fixed_link)
 909                 /* Stop PHY layer to call the hook to adjust the link in case
 910                  * of a switch is attached to the stmmac driver.
 911                  */
 912                 phydev->irq = PHY_IGNORE_INTERRUPT;
 913         else
 914                 /* At this stage, init the EEE if supported.
 915                  * Never called in case of fixed_link.
 916                  */
 917                 priv->eee_enabled = stmmac_eee_init(priv);
 918 }
 919
 920 /**
 921  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 922  * @priv: driver private structure
 923  * Description: this is to verify if the HW supports the PCS.
 924  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 925  * configured for the TBI, RTBI, or SGMII PHY interface.
 926  */
 927 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 928 {
 929         int interface = priv->plat->interface;
 930
 931         if (priv->dma_cap.pcs) {
 932                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 933                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 934                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 935                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 936                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 937                         priv->hw->pcs = STMMAC_PCS_RGMII;
 938                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 939                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 940                         priv->hw->pcs = STMMAC_PCS_SGMII;
 941                 }
 942         }
 943 }
 944
 945 /**
 946  * stmmac_init_phy - PHY initialization
 947  * @dev: net device structure
 948  * Description: it initializes the driver's PHY state, and attaches the PHY
 949  * to the mac driver.
 950  *  Return value:
 951  *  0 on success
 952  */
 953 static int stmmac_init_phy(struct net_device *dev)
 954 {
 955         struct stmmac_priv *priv = netdev_priv(dev);
 956         u32 tx_cnt = priv->plat->tx_queues_to_use;
 957         struct phy_device *phydev;
 958         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 959         char bus_id[MII_BUS_ID_SIZE];
 960         int interface = priv->plat->interface;
 961         int max_speed = priv->plat->max_speed;
 962         priv->oldlink = false;
 963         priv->speed = SPEED_UNKNOWN;
 964         priv->oldduplex = DUPLEX_UNKNOWN;
 965
 966         if (priv->plat->phy_node) {
 967                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 968                                         &stmmac_adjust_link, 0, interface);
 969         } else {
 970                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 971                          priv->plat->bus_id);
 972
 973                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 974                          priv->plat->phy_addr);
 975                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 976                            phy_id_fmt);
 977
 978                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 979                                      interface);
 980         }
 981
 982         if (IS_ERR_OR_NULL(phydev)) {
 983                 netdev_err(priv->dev, "Could not attach to PHY\n");
 984                 if (!phydev)
 985                         return -ENODEV;
 986
 987                 return PTR_ERR(phydev);
 988         }
 989
 990         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 991         if ((interface == PHY_INTERFACE_MODE_MII) ||
 992             (interface == PHY_INTERFACE_MODE_RMII) ||
 993                 (max_speed < 1000 && max_speed > 0))
 994                 phy_set_max_speed(phydev, SPEED_100);
 995
 996         /*
 997          * Half-duplex mode not supported with multiqueue
 998          * half-duplex can only works with single queue
 999          */
1000         if (tx_cnt > 1) {
1001                 phy_remove_link_mode(phydev,
1002                                      ETHTOOL_LINK_MODE_10baseT_Half_BIT);
1003                 phy_remove_link_mode(phydev,
1004                                      ETHTOOL_LINK_MODE_100baseT_Half_BIT);
1005                 phy_remove_link_mode(phydev,
1006                                      ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
1007         }
1008
1009         /*
1010          * Broken HW is sometimes missing the pull-up resistor on the
1011          * MDIO line, which results in reads to non-existent devices returning
1012          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
1013          * device as well.
1014          * Note: phydev->phy_id is the result of reading the UID PHY registers.
1015          */
1016         if (!priv->plat->phy_node && phydev->phy_id == 0) {
1017                 phy_disconnect(phydev);
1018                 return -ENODEV;
1019         }
1020
1021         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
1022          * subsequent PHY polling, make sure we force a link transition if
1023          * we have a UP/DOWN/UP transition
1024          */
1025         if (phydev->is_pseudo_fixed_link)
1026                 phydev->irq = PHY_POLL;
1027
1028         phy_attached_info(phydev);
1029         return 0;
1030 }
1031
1032 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
1033 {
1034         u32 rx_cnt = priv->plat->rx_queues_to_use;
1035         void *head_rx;
1036         u32 queue;
1037
1038         /* Display RX rings */
1039         for (queue = 0; queue < rx_cnt; queue++) {
1040                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1041
1042                 pr_info("\tRX Queue %u rings\n", queue);
1043
1044                 if (priv->extend_desc)
1045                         head_rx = (void *)rx_q->dma_erx;
1046                 else
1047                         head_rx = (void *)rx_q->dma_rx;
1048
1049                 /* Display RX ring */
1050                 stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true);
1051         }
1052 }
1053
1054 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1055 {
1056         u32 tx_cnt = priv->plat->tx_queues_to_use;
1057         void *head_tx;
1058         u32 queue;
1059
1060         /* Display TX rings */
1061         for (queue = 0; queue < tx_cnt; queue++) {
1062                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1063
1064                 pr_info("\tTX Queue %d rings\n", queue);
1065
1066                 if (priv->extend_desc)
1067                         head_tx = (void *)tx_q->dma_etx;
1068                 else
1069                         head_tx = (void *)tx_q->dma_tx;
1070
1071                 stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false);
1072         }
1073 }
1074
1075 static void stmmac_display_rings(struct stmmac_priv *priv)
1076 {
1077         /* Display RX ring */
1078         stmmac_display_rx_rings(priv);
1079
1080         /* Display TX ring */
1081         stmmac_display_tx_rings(priv);
1082 }
1083
1084 static int stmmac_set_bfsize(int mtu, int bufsize)
1085 {
1086         int ret = bufsize;
1087
1088         if (mtu >= BUF_SIZE_4KiB)
1089                 ret = BUF_SIZE_8KiB;
1090         else if (mtu >= BUF_SIZE_2KiB)
1091                 ret = BUF_SIZE_4KiB;
1092         else if (mtu > DEFAULT_BUFSIZE)
1093                 ret = BUF_SIZE_2KiB;
1094         else
1095                 ret = DEFAULT_BUFSIZE;
1096
1097         return ret;
1098 }
1099
1100 /**
1101  * stmmac_clear_rx_descriptors - clear RX descriptors
1102  * @priv: driver private structure
1103  * @queue: RX queue index
1104  * Description: this function is called to clear the RX descriptors
1105  * in case of both basic and extended descriptors are used.
1106  */
1107 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1108 {
1109         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1110         int i;
1111
1112         /* Clear the RX descriptors */
1113         for (i = 0; i < DMA_RX_SIZE; i++)
1114                 if (priv->extend_desc)
1115                         stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1116                                         priv->use_riwt, priv->mode,
1117                                         (i == DMA_RX_SIZE - 1));
1118                 else
1119                         stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1120                                         priv->use_riwt, priv->mode,
1121                                         (i == DMA_RX_SIZE - 1));
1122 }
1123
1124 /**
1125  * stmmac_clear_tx_descriptors - clear tx descriptors
1126  * @priv: driver private structure
1127  * @queue: TX queue index.
1128  * Description: this function is called to clear the TX descriptors
1129  * in case of both basic and extended descriptors are used.
1130  */
1131 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1132 {
1133         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1134         int i;
1135
1136         /* Clear the TX descriptors */
1137         for (i = 0; i < DMA_TX_SIZE; i++)
1138                 if (priv->extend_desc)
1139                         stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
1140                                         priv->mode, (i == DMA_TX_SIZE - 1));
1141                 else
1142                         stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
1143                                         priv->mode, (i == DMA_TX_SIZE - 1));
1144 }
1145
1146 /**
1147  * stmmac_clear_descriptors - clear descriptors
1148  * @priv: driver private structure
1149  * Description: this function is called to clear the TX and RX descriptors
1150  * in case of both basic and extended descriptors are used.
1151  */
1152 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1153 {
1154         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1155         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1156         u32 queue;
1157
1158         /* Clear the RX descriptors */
1159         for (queue = 0; queue < rx_queue_cnt; queue++)
1160                 stmmac_clear_rx_descriptors(priv, queue);
1161
1162         /* Clear the TX descriptors */
1163         for (queue = 0; queue < tx_queue_cnt; queue++)
1164                 stmmac_clear_tx_descriptors(priv, queue);
1165 }
1166
1167 /**
1168  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1169  * @priv: driver private structure
1170  * @p: descriptor pointer
1171  * @i: descriptor index
1172  * @flags: gfp flag
1173  * @queue: RX queue index
1174  * Description: this function is called to allocate a receive buffer, perform
1175  * the DMA mapping and init the descriptor.
1176  */
1177 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1178                                   int i, gfp_t flags, u32 queue)
1179 {
1180         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1181         struct sk_buff *skb;
1182
1183         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1184         if (!skb) {
1185                 netdev_err(priv->dev,
1186                            "%s: Rx init fails; skb is NULL\n", __func__);
1187                 return -ENOMEM;
1188         }
1189         rx_q->rx_skbuff[i] = skb;
1190         rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1191                                                 priv->dma_buf_sz,
1192                                                 DMA_FROM_DEVICE);
1193         if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1194                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1195                 dev_kfree_skb_any(skb);
1196                 return -EINVAL;
1197         }
1198
1199         stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[i]);
1200
1201         if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1202                 stmmac_init_desc3(priv, p);
1203
1204         return 0;
1205 }
1206
1207 /**
1208  * stmmac_free_rx_buffer - free RX dma buffers
1209  * @priv: private structure
1210  * @queue: RX queue index
1211  * @i: buffer index.
1212  */
1213 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1214 {
1215         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1216
1217         if (rx_q->rx_skbuff[i]) {
1218                 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1219                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1220                 dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1221         }
1222         rx_q->rx_skbuff[i] = NULL;
1223 }
1224
1225 /**
1226  * stmmac_free_tx_buffer - free RX dma buffers
1227  * @priv: private structure
1228  * @queue: RX queue index
1229  * @i: buffer index.
1230  */
1231 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1232 {
1233         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1234
1235         if (tx_q->tx_skbuff_dma[i].buf) {
1236                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1237                         dma_unmap_page(priv->device,
1238                                        tx_q->tx_skbuff_dma[i].buf,
1239                                        tx_q->tx_skbuff_dma[i].len,
1240                                        DMA_TO_DEVICE);
1241                 else
1242                         dma_unmap_single(priv->device,
1243                                          tx_q->tx_skbuff_dma[i].buf,
1244                                          tx_q->tx_skbuff_dma[i].len,
1245                                          DMA_TO_DEVICE);
1246         }
1247
1248         if (tx_q->tx_skbuff[i]) {
1249                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1250                 tx_q->tx_skbuff[i] = NULL;
1251                 tx_q->tx_skbuff_dma[i].buf = 0;
1252                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1253         }
1254 }
1255
1256 /**
1257  * init_dma_rx_desc_rings - init the RX descriptor rings
1258  * @dev: net device structure
1259  * @flags: gfp flag.
1260  * Description: this function initializes the DMA RX descriptors
1261  * and allocates the socket buffers. It supports the chained and ring
1262  * modes.
1263  */
1264 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1265 {
1266         struct stmmac_priv *priv = netdev_priv(dev);
1267         u32 rx_count = priv->plat->rx_queues_to_use;
1268         int ret = -ENOMEM;
1269         int bfsize = 0;
1270         int queue;
1271         int i;
1272
1273         bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
1274         if (bfsize < 0)
1275                 bfsize = 0;
1276
1277         if (bfsize < BUF_SIZE_16KiB)
1278                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1279
1280         priv->dma_buf_sz = bfsize;
1281
1282         /* RX INITIALIZATION */
1283         netif_dbg(priv, probe, priv->dev,
1284                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1285
1286         for (queue = 0; queue < rx_count; queue++) {
1287                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1288
1289                 netif_dbg(priv, probe, priv->dev,
1290                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1291                           (u32)rx_q->dma_rx_phy);
1292
1293                 for (i = 0; i < DMA_RX_SIZE; i++) {
1294                         struct dma_desc *p;
1295
1296                         if (priv->extend_desc)
1297                                 p = &((rx_q->dma_erx + i)->basic);
1298                         else
1299                                 p = rx_q->dma_rx + i;
1300
1301                         ret = stmmac_init_rx_buffers(priv, p, i, flags,
1302                                                      queue);
1303                         if (ret)
1304                                 goto err_init_rx_buffers;
1305
1306                         netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1307                                   rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data,
1308                                   (unsigned int)rx_q->rx_skbuff_dma[i]);
1309                 }
1310
1311                 rx_q->cur_rx = 0;
1312                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1313
1314                 stmmac_clear_rx_descriptors(priv, queue);
1315
1316                 /* Setup the chained descriptor addresses */
1317                 if (priv->mode == STMMAC_CHAIN_MODE) {
1318                         if (priv->extend_desc)
1319                                 stmmac_mode_init(priv, rx_q->dma_erx,
1320                                                 rx_q->dma_rx_phy, DMA_RX_SIZE, 1);
1321                         else
1322                                 stmmac_mode_init(priv, rx_q->dma_rx,
1323                                                 rx_q->dma_rx_phy, DMA_RX_SIZE, 0);
1324                 }
1325         }
1326
1327         buf_sz = bfsize;
1328
1329         return 0;
1330
1331 err_init_rx_buffers:
1332         while (queue >= 0) {
1333                 while (--i >= 0)
1334                         stmmac_free_rx_buffer(priv, queue, i);
1335
1336                 if (queue == 0)
1337                         break;
1338
1339                 i = DMA_RX_SIZE;
1340                 queue--;
1341         }
1342
1343         return ret;
1344 }
1345
1346 /**
1347  * init_dma_tx_desc_rings - init the TX descriptor rings
1348  * @dev: net device structure.
1349  * Description: this function initializes the DMA TX descriptors
1350  * and allocates the socket buffers. It supports the chained and ring
1351  * modes.
1352  */
1353 static int init_dma_tx_desc_rings(struct net_device *dev)
1354 {
1355         struct stmmac_priv *priv = netdev_priv(dev);
1356         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1357         u32 queue;
1358         int i;
1359
1360         for (queue = 0; queue < tx_queue_cnt; queue++) {
1361                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1362
1363                 netif_dbg(priv, probe, priv->dev,
1364                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1365                          (u32)tx_q->dma_tx_phy);
1366
1367                 /* Setup the chained descriptor addresses */
1368                 if (priv->mode == STMMAC_CHAIN_MODE) {
1369                         if (priv->extend_desc)
1370                                 stmmac_mode_init(priv, tx_q->dma_etx,
1371                                                 tx_q->dma_tx_phy, DMA_TX_SIZE, 1);
1372                         else
1373                                 stmmac_mode_init(priv, tx_q->dma_tx,
1374                                                 tx_q->dma_tx_phy, DMA_TX_SIZE, 0);
1375                 }
1376
1377                 for (i = 0; i < DMA_TX_SIZE; i++) {
1378                         struct dma_desc *p;
1379                         if (priv->extend_desc)
1380                                 p = &((tx_q->dma_etx + i)->basic);
1381                         else
1382                                 p = tx_q->dma_tx + i;
1383
1384                         stmmac_clear_desc(priv, p);
1385
1386                         tx_q->tx_skbuff_dma[i].buf = 0;
1387                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1388                         tx_q->tx_skbuff_dma[i].len = 0;
1389                         tx_q->tx_skbuff_dma[i].last_segment = false;
1390                         tx_q->tx_skbuff[i] = NULL;
1391                 }
1392
1393                 tx_q->dirty_tx = 0;
1394                 tx_q->cur_tx = 0;
1395                 tx_q->mss = 0;
1396
1397                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1398         }
1399
1400         return 0;
1401 }
1402
1403 /**
1404  * init_dma_desc_rings - init the RX/TX descriptor rings
1405  * @dev: net device structure
1406  * @flags: gfp flag.
1407  * Description: this function initializes the DMA RX/TX descriptors
1408  * and allocates the socket buffers. It supports the chained and ring
1409  * modes.
1410  */
1411 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1412 {
1413         struct stmmac_priv *priv = netdev_priv(dev);
1414         int ret;
1415
1416         ret = init_dma_rx_desc_rings(dev, flags);
1417         if (ret)
1418                 return ret;
1419
1420         ret = init_dma_tx_desc_rings(dev);
1421
1422         stmmac_clear_descriptors(priv);
1423
1424         if (netif_msg_hw(priv))
1425                 stmmac_display_rings(priv);
1426
1427         return ret;
1428 }
1429
1430 /**
1431  * dma_free_rx_skbufs - free RX dma buffers
1432  * @priv: private structure
1433  * @queue: RX queue index
1434  */
1435 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1436 {
1437         int i;
1438
1439         for (i = 0; i < DMA_RX_SIZE; i++)
1440                 stmmac_free_rx_buffer(priv, queue, i);
1441 }
1442
1443 /**
1444  * dma_free_tx_skbufs - free TX dma buffers
1445  * @priv: private structure
1446  * @queue: TX queue index
1447  */
1448 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1449 {
1450         int i;
1451
1452         for (i = 0; i < DMA_TX_SIZE; i++)
1453                 stmmac_free_tx_buffer(priv, queue, i);
1454 }
1455
1456 /**
1457  * free_dma_rx_desc_resources - free RX dma desc resources
1458  * @priv: private structure
1459  */
1460 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1461 {
1462         u32 rx_count = priv->plat->rx_queues_to_use;
1463         u32 queue;
1464
1465         /* Free RX queue resources */
1466         for (queue = 0; queue < rx_count; queue++) {
1467                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1468
1469                 /* Release the DMA RX socket buffers */
1470                 dma_free_rx_skbufs(priv, queue);
1471
1472                 /* Free DMA regions of consistent memory previously allocated */
1473                 if (!priv->extend_desc)
1474                         dma_free_coherent(priv->device,
1475                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1476                                           rx_q->dma_rx, rx_q->dma_rx_phy);
1477                 else
1478                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1479                                           sizeof(struct dma_extended_desc),
1480                                           rx_q->dma_erx, rx_q->dma_rx_phy);
1481
1482                 kfree(rx_q->rx_skbuff_dma);
1483                 kfree(rx_q->rx_skbuff);
1484         }
1485 }
1486
1487 /**
1488  * free_dma_tx_desc_resources - free TX dma desc resources
1489  * @priv: private structure
1490  */
1491 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1492 {
1493         u32 tx_count = priv->plat->tx_queues_to_use;
1494         u32 queue;
1495
1496         /* Free TX queue resources */
1497         for (queue = 0; queue < tx_count; queue++) {
1498                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1499
1500                 /* Release the DMA TX socket buffers */
1501                 dma_free_tx_skbufs(priv, queue);
1502
1503                 /* Free DMA regions of consistent memory previously allocated */
1504                 if (!priv->extend_desc)
1505                         dma_free_coherent(priv->device,
1506                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1507                                           tx_q->dma_tx, tx_q->dma_tx_phy);
1508                 else
1509                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1510                                           sizeof(struct dma_extended_desc),
1511                                           tx_q->dma_etx, tx_q->dma_tx_phy);
1512
1513                 kfree(tx_q->tx_skbuff_dma);
1514                 kfree(tx_q->tx_skbuff);
1515         }
1516 }
1517
1518 /**
1519  * alloc_dma_rx_desc_resources - alloc RX resources.
1520  * @priv: private structure
1521  * Description: according to which descriptor can be used (extend or basic)
1522  * this function allocates the resources for TX and RX paths. In case of
1523  * reception, for example, it pre-allocated the RX socket buffer in order to
1524  * allow zero-copy mechanism.
1525  */
1526 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1527 {
1528         u32 rx_count = priv->plat->rx_queues_to_use;
1529         int ret = -ENOMEM;
1530         u32 queue;
1531
1532         /* RX queues buffers and DMA */
1533         for (queue = 0; queue < rx_count; queue++) {
1534                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1535
1536                 rx_q->queue_index = queue;
1537                 rx_q->priv_data = priv;
1538
1539                 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
1540                                                     sizeof(dma_addr_t),
1541                                                     GFP_KERNEL);
1542                 if (!rx_q->rx_skbuff_dma)
1543                         goto err_dma;
1544
1545                 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
1546                                                 sizeof(struct sk_buff *),
1547                                                 GFP_KERNEL);
1548                 if (!rx_q->rx_skbuff)
1549                         goto err_dma;
1550
1551                 if (priv->extend_desc) {
1552                         rx_q->dma_erx = dma_alloc_coherent(priv->device,
1553                                                            DMA_RX_SIZE * sizeof(struct dma_extended_desc),
1554                                                            &rx_q->dma_rx_phy,
1555                                                            GFP_KERNEL);
1556                         if (!rx_q->dma_erx)
1557                                 goto err_dma;
1558
1559                 } else {
1560                         rx_q->dma_rx = dma_alloc_coherent(priv->device,
1561                                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1562                                                           &rx_q->dma_rx_phy,
1563                                                           GFP_KERNEL);
1564                         if (!rx_q->dma_rx)
1565                                 goto err_dma;
1566                 }
1567         }
1568
1569         return 0;
1570
1571 err_dma:
1572         free_dma_rx_desc_resources(priv);
1573
1574         return ret;
1575 }
1576
1577 /**
1578  * alloc_dma_tx_desc_resources - alloc TX resources.
1579  * @priv: private structure
1580  * Description: according to which descriptor can be used (extend or basic)
1581  * this function allocates the resources for TX and RX paths. In case of
1582  * reception, for example, it pre-allocated the RX socket buffer in order to
1583  * allow zero-copy mechanism.
1584  */
1585 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1586 {
1587         u32 tx_count = priv->plat->tx_queues_to_use;
1588         int ret = -ENOMEM;
1589         u32 queue;
1590
1591         /* TX queues buffers and DMA */
1592         for (queue = 0; queue < tx_count; queue++) {
1593                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1594
1595                 tx_q->queue_index = queue;
1596                 tx_q->priv_data = priv;
1597
1598                 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1599                                                     sizeof(*tx_q->tx_skbuff_dma),
1600                                                     GFP_KERNEL);
1601                 if (!tx_q->tx_skbuff_dma)
1602                         goto err_dma;
1603
1604                 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
1605                                                 sizeof(struct sk_buff *),
1606                                                 GFP_KERNEL);
1607                 if (!tx_q->tx_skbuff)
1608                         goto err_dma;
1609
1610                 if (priv->extend_desc) {
1611                         tx_q->dma_etx = dma_alloc_coherent(priv->device,
1612                                                            DMA_TX_SIZE * sizeof(struct dma_extended_desc),
1613                                                            &tx_q->dma_tx_phy,
1614                                                            GFP_KERNEL);
1615                         if (!tx_q->dma_etx)
1616                                 goto err_dma;
1617                 } else {
1618                         tx_q->dma_tx = dma_alloc_coherent(priv->device,
1619                                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1620                                                           &tx_q->dma_tx_phy,
1621                                                           GFP_KERNEL);
1622                         if (!tx_q->dma_tx)
1623                                 goto err_dma;
1624                 }
1625         }
1626
1627         return 0;
1628
1629 err_dma:
1630         free_dma_tx_desc_resources(priv);
1631
1632         return ret;
1633 }
1634
1635 /**
1636  * alloc_dma_desc_resources - alloc TX/RX resources.
1637  * @priv: private structure
1638  * Description: according to which descriptor can be used (extend or basic)
1639  * this function allocates the resources for TX and RX paths. In case of
1640  * reception, for example, it pre-allocated the RX socket buffer in order to
1641  * allow zero-copy mechanism.
1642  */
1643 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1644 {
1645         /* RX Allocation */
1646         int ret = alloc_dma_rx_desc_resources(priv);
1647
1648         if (ret)
1649                 return ret;
1650
1651         ret = alloc_dma_tx_desc_resources(priv);
1652
1653         return ret;
1654 }
1655
1656 /**
1657  * free_dma_desc_resources - free dma desc resources
1658  * @priv: private structure
1659  */
1660 static void free_dma_desc_resources(struct stmmac_priv *priv)
1661 {
1662         /* Release the DMA RX socket buffers */
1663         free_dma_rx_desc_resources(priv);
1664
1665         /* Release the DMA TX socket buffers */
1666         free_dma_tx_desc_resources(priv);
1667 }
1668
1669 /**
1670  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1671  *  @priv: driver private structure
1672  *  Description: It is used for enabling the rx queues in the MAC
1673  */
1674 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1675 {
1676         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1677         int queue;
1678         u8 mode;
1679
1680         for (queue = 0; queue < rx_queues_count; queue++) {
1681                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1682                 stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
1683         }
1684 }
1685
1686 /**
1687  * stmmac_start_rx_dma - start RX DMA channel
1688  * @priv: driver private structure
1689  * @chan: RX channel index
1690  * Description:
1691  * This starts a RX DMA channel
1692  */
1693 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1694 {
1695         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1696         stmmac_start_rx(priv, priv->ioaddr, chan);
1697 }
1698
1699 /**
1700  * stmmac_start_tx_dma - start TX DMA channel
1701  * @priv: driver private structure
1702  * @chan: TX channel index
1703  * Description:
1704  * This starts a TX DMA channel
1705  */
1706 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1707 {
1708         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1709         stmmac_start_tx(priv, priv->ioaddr, chan);
1710 }
1711
1712 /**
1713  * stmmac_stop_rx_dma - stop RX DMA channel
1714  * @priv: driver private structure
1715  * @chan: RX channel index
1716  * Description:
1717  * This stops a RX DMA channel
1718  */
1719 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1720 {
1721         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1722         stmmac_stop_rx(priv, priv->ioaddr, chan);
1723 }
1724
1725 /**
1726  * stmmac_stop_tx_dma - stop TX DMA channel
1727  * @priv: driver private structure
1728  * @chan: TX channel index
1729  * Description:
1730  * This stops a TX DMA channel
1731  */
1732 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1733 {
1734         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1735         stmmac_stop_tx(priv, priv->ioaddr, chan);
1736 }
1737
1738 /**
1739  * stmmac_start_all_dma - start all RX and TX DMA channels
1740  * @priv: driver private structure
1741  * Description:
1742  * This starts all the RX and TX DMA channels
1743  */
1744 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1745 {
1746         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1747         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1748         u32 chan = 0;
1749
1750         for (chan = 0; chan < rx_channels_count; chan++)
1751                 stmmac_start_rx_dma(priv, chan);
1752
1753         for (chan = 0; chan < tx_channels_count; chan++)
1754                 stmmac_start_tx_dma(priv, chan);
1755 }
1756
1757 /**
1758  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1759  * @priv: driver private structure
1760  * Description:
1761  * This stops the RX and TX DMA channels
1762  */
1763 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1764 {
1765         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1766         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1767         u32 chan = 0;
1768
1769         for (chan = 0; chan < rx_channels_count; chan++)
1770                 stmmac_stop_rx_dma(priv, chan);
1771
1772         for (chan = 0; chan < tx_channels_count; chan++)
1773                 stmmac_stop_tx_dma(priv, chan);
1774 }
1775
1776 /**
1777  *  stmmac_dma_operation_mode - HW DMA operation mode
1778  *  @priv: driver private structure
1779  *  Description: it is used for configuring the DMA operation mode register in
1780  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1781  */
1782 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1783 {
1784         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1785         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1786         int rxfifosz = priv->plat->rx_fifo_size;
1787         int txfifosz = priv->plat->tx_fifo_size;
1788         u32 txmode = 0;
1789         u32 rxmode = 0;
1790         u32 chan = 0;
1791         u8 qmode = 0;
1792
1793         if (rxfifosz == 0)
1794                 rxfifosz = priv->dma_cap.rx_fifo_size;
1795         if (txfifosz == 0)
1796                 txfifosz = priv->dma_cap.tx_fifo_size;
1797
1798         /* Adjust for real per queue fifo size */
1799         rxfifosz /= rx_channels_count;
1800         txfifosz /= tx_channels_count;
1801
1802         if (priv->plat->force_thresh_dma_mode) {
1803                 txmode = tc;
1804                 rxmode = tc;
1805         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1806                 /*
1807                  * In case of GMAC, SF mode can be enabled
1808                  * to perform the TX COE in HW. This depends on:
1809                  * 1) TX COE if actually supported
1810                  * 2) There is no bugged Jumbo frame support
1811                  *    that needs to not insert csum in the TDES.
1812                  */
1813                 txmode = SF_DMA_MODE;
1814                 rxmode = SF_DMA_MODE;
1815                 priv->xstats.threshold = SF_DMA_MODE;
1816         } else {
1817                 txmode = tc;
1818                 rxmode = SF_DMA_MODE;
1819         }
1820
1821         /* configure all channels */
1822         for (chan = 0; chan < rx_channels_count; chan++) {
1823                 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1824
1825                 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
1826                                 rxfifosz, qmode);
1827                 stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz,
1828                                 chan);
1829         }
1830
1831         for (chan = 0; chan < tx_channels_count; chan++) {
1832                 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1833
1834                 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
1835                                 txfifosz, qmode);
1836         }
1837 }
1838
1839 /**
1840  * stmmac_tx_clean - to manage the transmission completion
1841  * @priv: driver private structure
1842  * @queue: TX queue index
1843  * Description: it reclaims the transmit resources after transmission completes.
1844  */
1845 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
1846 {
1847         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1848         unsigned int bytes_compl = 0, pkts_compl = 0;
1849         unsigned int entry, count = 0;
1850
1851         __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
1852
1853         priv->xstats.tx_clean++;
1854
1855         entry = tx_q->dirty_tx;
1856         while ((entry != tx_q->cur_tx) && (count < budget)) {
1857                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1858                 struct dma_desc *p;
1859                 int status;
1860
1861                 if (priv->extend_desc)
1862                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1863                 else
1864                         p = tx_q->dma_tx + entry;
1865
1866                 status = stmmac_tx_status(priv, &priv->dev->stats,
1867                                 &priv->xstats, p, priv->ioaddr);
1868                 /* Check if the descriptor is owned by the DMA */
1869                 if (unlikely(status & tx_dma_own))
1870                         break;
1871
1872                 count++;
1873
1874                 /* Make sure descriptor fields are read after reading
1875                  * the own bit.
1876                  */
1877                 dma_rmb();
1878
1879                 /* Just consider the last segment and ...*/
1880                 if (likely(!(status & tx_not_ls))) {
1881                         /* ... verify the status error condition */
1882                         if (unlikely(status & tx_err)) {
1883                                 priv->dev->stats.tx_errors++;
1884                         } else {
1885                                 priv->dev->stats.tx_packets++;
1886                                 priv->xstats.tx_pkt_n++;
1887                         }
1888                         stmmac_get_tx_hwtstamp(priv, p, skb);
1889                 }
1890
1891                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1892                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1893                                 dma_unmap_page(priv->device,
1894                                                tx_q->tx_skbuff_dma[entry].buf,
1895                                                tx_q->tx_skbuff_dma[entry].len,
1896                                                DMA_TO_DEVICE);
1897                         else
1898                                 dma_unmap_single(priv->device,
1899                                                  tx_q->tx_skbuff_dma[entry].buf,
1900                                                  tx_q->tx_skbuff_dma[entry].len,
1901                                                  DMA_TO_DEVICE);
1902                         tx_q->tx_skbuff_dma[entry].buf = 0;
1903                         tx_q->tx_skbuff_dma[entry].len = 0;
1904                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1905                 }
1906
1907                 stmmac_clean_desc3(priv, tx_q, p);
1908
1909                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1910                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1911
1912                 if (likely(skb != NULL)) {
1913                         pkts_compl++;
1914                         bytes_compl += skb->len;
1915                         dev_consume_skb_any(skb);
1916                         tx_q->tx_skbuff[entry] = NULL;
1917                 }
1918
1919                 stmmac_release_tx_desc(priv, p, priv->mode);
1920
1921                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1922         }
1923         tx_q->dirty_tx = entry;
1924
1925         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1926                                   pkts_compl, bytes_compl);
1927
1928         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1929                                                                 queue))) &&
1930             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1931
1932                 netif_dbg(priv, tx_done, priv->dev,
1933                           "%s: restart transmit\n", __func__);
1934                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1935         }
1936
1937         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1938                 stmmac_enable_eee_mode(priv);
1939                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1940         }
1941
1942         __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
1943
1944         return count;
1945 }
1946
1947 /**
1948  * stmmac_tx_err - to manage the tx error
1949  * @priv: driver private structure
1950  * @chan: channel index
1951  * Description: it cleans the descriptors and restarts the transmission
1952  * in case of transmission errors.
1953  */
1954 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1955 {
1956         struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1957         int i;
1958
1959         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
1960
1961         stmmac_stop_tx_dma(priv, chan);
1962         dma_free_tx_skbufs(priv, chan);
1963         for (i = 0; i < DMA_TX_SIZE; i++)
1964                 if (priv->extend_desc)
1965                         stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
1966                                         priv->mode, (i == DMA_TX_SIZE - 1));
1967                 else
1968                         stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
1969                                         priv->mode, (i == DMA_TX_SIZE - 1));
1970         tx_q->dirty_tx = 0;
1971         tx_q->cur_tx = 0;
1972         tx_q->mss = 0;
1973         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
1974         stmmac_start_tx_dma(priv, chan);
1975
1976         priv->dev->stats.tx_errors++;
1977         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
1978 }
1979
1980 /**
1981  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1982  *  @priv: driver private structure
1983  *  @txmode: TX operating mode
1984  *  @rxmode: RX operating mode
1985  *  @chan: channel index
1986  *  Description: it is used for configuring of the DMA operation mode in
1987  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1988  *  mode.
1989  */
1990 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1991                                           u32 rxmode, u32 chan)
1992 {
1993         u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1994         u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1995         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1996         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1997         int rxfifosz = priv->plat->rx_fifo_size;
1998         int txfifosz = priv->plat->tx_fifo_size;
1999
2000         if (rxfifosz == 0)
2001                 rxfifosz = priv->dma_cap.rx_fifo_size;
2002         if (txfifosz == 0)
2003                 txfifosz = priv->dma_cap.tx_fifo_size;
2004
2005         /* Adjust for real per queue fifo size */
2006         rxfifosz /= rx_channels_count;
2007         txfifosz /= tx_channels_count;
2008
2009         stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2010         stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2011 }
2012
2013 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2014 {
2015         int ret;
2016
2017         ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2018                         priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2019         if (ret && (ret != -EINVAL)) {
2020                 stmmac_global_err(priv);
2021                 return true;
2022         }
2023
2024         return false;
2025 }
2026
2027 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
2028 {
2029         int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2030                                                  &priv->xstats, chan);
2031         struct stmmac_channel *ch = &priv->channel[chan];
2032         bool needs_work = false;
2033
2034         if ((status & handle_rx) && ch->has_rx) {
2035                 needs_work = true;
2036         } else {
2037                 status &= ~handle_rx;
2038         }
2039
2040         if ((status & handle_tx) && ch->has_tx) {
2041                 needs_work = true;
2042         } else {
2043                 status &= ~handle_tx;
2044         }
2045
2046         if (needs_work && napi_schedule_prep(&ch->napi)) {
2047                 stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
2048                 __napi_schedule(&ch->napi);
2049         }
2050
2051         return status;
2052 }
2053
2054 /**
2055  * stmmac_dma_interrupt - DMA ISR
2056  * @priv: driver private structure
2057  * Description: this is the DMA ISR. It is called by the main ISR.
2058  * It calls the dwmac dma routine and schedule poll method in case of some
2059  * work can be done.
2060  */
2061 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2062 {
2063         u32 tx_channel_count = priv->plat->tx_queues_to_use;
2064         u32 rx_channel_count = priv->plat->rx_queues_to_use;
2065         u32 channels_to_check = tx_channel_count > rx_channel_count ?
2066                                 tx_channel_count : rx_channel_count;
2067         u32 chan;
2068         int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2069
2070         /* Make sure we never check beyond our status buffer. */
2071         if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2072                 channels_to_check = ARRAY_SIZE(status);
2073
2074         for (chan = 0; chan < channels_to_check; chan++)
2075                 status[chan] = stmmac_napi_check(priv, chan);
2076
2077         for (chan = 0; chan < tx_channel_count; chan++) {
2078                 if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2079                         /* Try to bump up the dma threshold on this failure */
2080                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2081                             (tc <= 256)) {
2082                                 tc += 64;
2083                                 if (priv->plat->force_thresh_dma_mode)
2084                                         stmmac_set_dma_operation_mode(priv,
2085                                                                       tc,
2086                                                                       tc,
2087                                                                       chan);
2088                                 else
2089                                         stmmac_set_dma_operation_mode(priv,
2090                                                                     tc,
2091                                                                     SF_DMA_MODE,
2092                                                                     chan);
2093                                 priv->xstats.threshold = tc;
2094                         }
2095                 } else if (unlikely(status[chan] == tx_hard_error)) {
2096                         stmmac_tx_err(priv, chan);
2097                 }
2098         }
2099 }
2100
2101 /**
2102  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2103  * @priv: driver private structure
2104  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2105  */
2106 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2107 {
2108         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2109                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2110
2111         dwmac_mmc_intr_all_mask(priv->mmcaddr);
2112
2113         if (priv->dma_cap.rmon) {
2114                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
2115                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2116         } else
2117                 netdev_info(priv->dev, "No MAC Management Counters available\n");
2118 }
2119
2120 /**
2121  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2122  * @priv: driver private structure
2123  * Description:
2124  *  new GMAC chip generations have a new register to indicate the
2125  *  presence of the optional feature/functions.
2126  *  This can be also used to override the value passed through the
2127  *  platform and necessary for old MAC10/100 and GMAC chips.
2128  */
2129 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2130 {
2131         return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2132 }
2133
2134 /**
2135  * stmmac_check_ether_addr - check if the MAC addr is valid
2136  * @priv: driver private structure
2137  * Description:
2138  * it is to verify if the MAC address is valid, in case of failures it
2139  * generates a random MAC address
2140  */
2141 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2142 {
2143         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2144                 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
2145                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2146                         eth_hw_addr_random(priv->dev);
2147                 netdev_info(priv->dev, "device MAC address %pM\n",
2148                             priv->dev->dev_addr);
2149         }
2150 }
2151
2152 /**
2153  * stmmac_init_dma_engine - DMA init.
2154  * @priv: driver private structure
2155  * Description:
2156  * It inits the DMA invoking the specific MAC/GMAC callback.
2157  * Some DMA parameters can be passed from the platform;
2158  * in case of these are not passed a default is kept for the MAC or GMAC.
2159  */
2160 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2161 {
2162         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2163         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2164         u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2165         struct stmmac_rx_queue *rx_q;
2166         struct stmmac_tx_queue *tx_q;
2167         u32 chan = 0;
2168         int atds = 0;
2169         int ret = 0;
2170
2171         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2172                 dev_err(priv->device, "Invalid DMA configuration\n");
2173                 return -EINVAL;
2174         }
2175
2176         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2177                 atds = 1;
2178
2179         ret = stmmac_reset(priv, priv->ioaddr);
2180         if (ret) {
2181                 dev_err(priv->device, "Failed to reset the dma\n");
2182                 return ret;
2183         }
2184
2185         /* DMA Configuration */
2186         stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2187
2188         if (priv->plat->axi)
2189                 stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2190
2191         /* DMA RX Channel Configuration */
2192         for (chan = 0; chan < rx_channels_count; chan++) {
2193                 rx_q = &priv->rx_queue[chan];
2194
2195                 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2196                                     rx_q->dma_rx_phy, chan);
2197
2198                 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2199                             (DMA_RX_SIZE * sizeof(struct dma_desc));
2200                 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2201                                        rx_q->rx_tail_addr, chan);
2202         }
2203
2204         /* DMA TX Channel Configuration */
2205         for (chan = 0; chan < tx_channels_count; chan++) {
2206                 tx_q = &priv->tx_queue[chan];
2207
2208                 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2209                                     tx_q->dma_tx_phy, chan);
2210
2211                 tx_q->tx_tail_addr = tx_q->dma_tx_phy;
2212                 stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2213                                        tx_q->tx_tail_addr, chan);
2214         }
2215
2216         /* DMA CSR Channel configuration */
2217         for (chan = 0; chan < dma_csr_ch; chan++)
2218                 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2219
2220         return ret;
2221 }
2222
2223 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2224 {
2225         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2226
2227         mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2228 }
2229
2230 /**
2231  * stmmac_tx_timer - mitigation sw timer for tx.
2232  * @data: data pointer
2233  * Description:
2234  * This is the timer handler to directly invoke the stmmac_tx_clean.
2235  */
2236 static void stmmac_tx_timer(struct timer_list *t)
2237 {
2238         struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
2239         struct stmmac_priv *priv = tx_q->priv_data;
2240         struct stmmac_channel *ch;
2241
2242         ch = &priv->channel[tx_q->queue_index];
2243
2244         if (likely(napi_schedule_prep(&ch->napi)))
2245                 __napi_schedule(&ch->napi);
2246 }
2247
2248 /**
2249  * stmmac_init_tx_coalesce - init tx mitigation options.
2250  * @priv: driver private structure
2251  * Description:
2252  * This inits the transmit coalesce parameters: i.e. timer rate,
2253  * timer handler and default threshold used for enabling the
2254  * interrupt on completion bit.
2255  */
2256 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
2257 {
2258         u32 tx_channel_count = priv->plat->tx_queues_to_use;
2259         u32 chan;
2260
2261         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2262         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2263
2264         for (chan = 0; chan < tx_channel_count; chan++) {
2265                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2266
2267                 timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
2268         }
2269 }
2270
2271 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2272 {
2273         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2274         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2275         u32 chan;
2276
2277         /* set TX ring length */
2278         for (chan = 0; chan < tx_channels_count; chan++)
2279                 stmmac_set_tx_ring_len(priv, priv->ioaddr,
2280                                 (DMA_TX_SIZE - 1), chan);
2281
2282         /* set RX ring length */
2283         for (chan = 0; chan < rx_channels_count; chan++)
2284                 stmmac_set_rx_ring_len(priv, priv->ioaddr,
2285                                 (DMA_RX_SIZE - 1), chan);
2286 }
2287
2288 /**
2289  *  stmmac_set_tx_queue_weight - Set TX queue weight
2290  *  @priv: driver private structure
2291  *  Description: It is used for setting TX queues weight
2292  */
2293 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2294 {
2295         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2296         u32 weight;
2297         u32 queue;
2298
2299         for (queue = 0; queue < tx_queues_count; queue++) {
2300                 weight = priv->plat->tx_queues_cfg[queue].weight;
2301                 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2302         }
2303 }
2304
2305 /**
2306  *  stmmac_configure_cbs - Configure CBS in TX queue
2307  *  @priv: driver private structure
2308  *  Description: It is used for configuring CBS in AVB TX queues
2309  */
2310 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2311 {
2312         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2313         u32 mode_to_use;
2314         u32 queue;
2315
2316         /* queue 0 is reserved for legacy traffic */
2317         for (queue = 1; queue < tx_queues_count; queue++) {
2318                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2319                 if (mode_to_use == MTL_QUEUE_DCB)
2320                         continue;
2321
2322                 stmmac_config_cbs(priv, priv->hw,
2323                                 priv->plat->tx_queues_cfg[queue].send_slope,
2324                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2325                                 priv->plat->tx_queues_cfg[queue].high_credit,
2326                                 priv->plat->tx_queues_cfg[queue].low_credit,
2327                                 queue);
2328         }
2329 }
2330
2331 /**
2332  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2333  *  @priv: driver private structure
2334  *  Description: It is used for mapping RX queues to RX dma channels
2335  */
2336 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2337 {
2338         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2339         u32 queue;
2340         u32 chan;
2341
2342         for (queue = 0; queue < rx_queues_count; queue++) {
2343                 chan = priv->plat->rx_queues_cfg[queue].chan;
2344                 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
2345         }
2346 }
2347
2348 /**
2349  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2350  *  @priv: driver private structure
2351  *  Description: It is used for configuring the RX Queue Priority
2352  */
2353 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2354 {
2355         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2356         u32 queue;
2357         u32 prio;
2358
2359         for (queue = 0; queue < rx_queues_count; queue++) {
2360                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2361                         continue;
2362
2363                 prio = priv->plat->rx_queues_cfg[queue].prio;
2364                 stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
2365         }
2366 }
2367
2368 /**
2369  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2370  *  @priv: driver private structure
2371  *  Description: It is used for configuring the TX Queue Priority
2372  */
2373 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2374 {
2375         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2376         u32 queue;
2377         u32 prio;
2378
2379         for (queue = 0; queue < tx_queues_count; queue++) {
2380                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2381                         continue;
2382
2383                 prio = priv->plat->tx_queues_cfg[queue].prio;
2384                 stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
2385         }
2386 }
2387
2388 /**
2389  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2390  *  @priv: driver private structure
2391  *  Description: It is used for configuring the RX queue routing
2392  */
2393 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2394 {
2395         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2396         u32 queue;
2397         u8 packet;
2398
2399         for (queue = 0; queue < rx_queues_count; queue++) {
2400                 /* no specific packet type routing specified for the queue */
2401                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2402                         continue;
2403
2404                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2405                 stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
2406         }
2407 }
2408
2409 /**
2410  *  stmmac_mtl_configuration - Configure MTL
2411  *  @priv: driver private structure
2412  *  Description: It is used for configurring MTL
2413  */
2414 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2415 {
2416         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2417         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2418
2419         if (tx_queues_count > 1)
2420                 stmmac_set_tx_queue_weight(priv);
2421
2422         /* Configure MTL RX algorithms */
2423         if (rx_queues_count > 1)
2424                 stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
2425                                 priv->plat->rx_sched_algorithm);
2426
2427         /* Configure MTL TX algorithms */
2428         if (tx_queues_count > 1)
2429                 stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
2430                                 priv->plat->tx_sched_algorithm);
2431
2432         /* Configure CBS in AVB TX queues */
2433         if (tx_queues_count > 1)
2434                 stmmac_configure_cbs(priv);
2435
2436         /* Map RX MTL to DMA channels */
2437         stmmac_rx_queue_dma_chan_map(priv);
2438
2439         /* Enable MAC RX Queues */
2440         stmmac_mac_enable_rx_queues(priv);
2441
2442         /* Set RX priorities */
2443         if (rx_queues_count > 1)
2444                 stmmac_mac_config_rx_queues_prio(priv);
2445
2446         /* Set TX priorities */
2447         if (tx_queues_count > 1)
2448                 stmmac_mac_config_tx_queues_prio(priv);
2449
2450         /* Set RX routing */
2451         if (rx_queues_count > 1)
2452                 stmmac_mac_config_rx_queues_routing(priv);
2453 }
2454
2455 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
2456 {
2457         if (priv->dma_cap.asp) {
2458                 netdev_info(priv->dev, "Enabling Safety Features\n");
2459                 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
2460         } else {
2461                 netdev_info(priv->dev, "No Safety Features support found\n");
2462         }
2463 }
2464
2465 /**
2466  * stmmac_hw_setup - setup mac in a usable state.
2467  *  @dev : pointer to the device structure.
2468  *  Description:
2469  *  this is the main function to setup the HW in a usable state because the
2470  *  dma engine is reset, the core registers are configured (e.g. AXI,
2471  *  Checksum features, timers). The DMA is ready to start receiving and
2472  *  transmitting.
2473  *  Return value:
2474  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2475  *  file on failure.
2476  */
2477 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2478 {
2479         struct stmmac_priv *priv = netdev_priv(dev);
2480         u32 rx_cnt = priv->plat->rx_queues_to_use;
2481         u32 tx_cnt = priv->plat->tx_queues_to_use;
2482         u32 chan;
2483         int ret;
2484
2485         /* DMA initialization and SW reset */
2486         ret = stmmac_init_dma_engine(priv);
2487         if (ret < 0) {
2488                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2489                            __func__);
2490                 return ret;
2491         }
2492
2493         /* Copy the MAC addr into the HW  */
2494         stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
2495
2496         /* PS and related bits will be programmed according to the speed */
2497         if (priv->hw->pcs) {
2498                 int speed = priv->plat->mac_port_sel_speed;
2499
2500                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2501                     (speed == SPEED_1000)) {
2502                         priv->hw->ps = speed;
2503                 } else {
2504                         dev_warn(priv->device, "invalid port speed\n");
2505                         priv->hw->ps = 0;
2506                 }
2507         }
2508
2509         /* Initialize the MAC Core */
2510         stmmac_core_init(priv, priv->hw, dev);
2511
2512         /* Initialize MTL*/
2513         stmmac_mtl_configuration(priv);
2514
2515         /* Initialize Safety Features */
2516         stmmac_safety_feat_configuration(priv);
2517
2518         ret = stmmac_rx_ipc(priv, priv->hw);
2519         if (!ret) {
2520                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2521                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2522                 priv->hw->rx_csum = 0;
2523         }
2524
2525         /* Enable the MAC Rx/Tx */
2526         stmmac_mac_set(priv, priv->ioaddr, true);
2527
2528         /* Set the HW DMA mode and the COE */
2529         stmmac_dma_operation_mode(priv);
2530
2531         stmmac_mmc_setup(priv);
2532
2533         if (init_ptp) {
2534                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2535                 if (ret < 0)
2536                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2537
2538                 ret = stmmac_init_ptp(priv);
2539                 if (ret == -EOPNOTSUPP)
2540                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2541                 else if (ret)
2542                         netdev_warn(priv->dev, "PTP init failed\n");
2543         }
2544
2545         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2546
2547         if (priv->use_riwt) {
2548                 ret = stmmac_rx_watchdog(priv, priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2549                 if (!ret)
2550                         priv->rx_riwt = MAX_DMA_RIWT;
2551         }
2552
2553         if (priv->hw->pcs)
2554                 stmmac_pcs_ctrl_ane(priv, priv->hw, 1, priv->hw->ps, 0);
2555
2556         /* set TX and RX rings length */
2557         stmmac_set_rings_length(priv);
2558
2559         /* Enable TSO */
2560         if (priv->tso) {
2561                 for (chan = 0; chan < tx_cnt; chan++)
2562                         stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
2563         }
2564
2565         /* Start the ball rolling... */
2566         stmmac_start_all_dma(priv);
2567
2568         return 0;
2569 }
2570
2571 static void stmmac_hw_teardown(struct net_device *dev)
2572 {
2573         struct stmmac_priv *priv = netdev_priv(dev);
2574
2575         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2576 }
2577
2578 /**
2579  *  stmmac_open - open entry point of the driver
2580  *  @dev : pointer to the device structure.
2581  *  Description:
2582  *  This function is the open entry point of the driver.
2583  *  Return value:
2584  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2585  *  file on failure.
2586  */
2587 static int stmmac_open(struct net_device *dev)
2588 {
2589         struct stmmac_priv *priv = netdev_priv(dev);
2590         u32 chan;
2591         int ret;
2592
2593         stmmac_check_ether_addr(priv);
2594
2595         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2596             priv->hw->pcs != STMMAC_PCS_TBI &&
2597             priv->hw->pcs != STMMAC_PCS_RTBI) {
2598                 ret = stmmac_init_phy(dev);
2599                 if (ret) {
2600                         netdev_err(priv->dev,
2601                                    "%s: Cannot attach to PHY (error: %d)\n",
2602                                    __func__, ret);
2603                         return ret;
2604                 }
2605         }
2606
2607         /* Extra statistics */
2608         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2609         priv->xstats.threshold = tc;
2610
2611         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2612         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2613
2614         ret = alloc_dma_desc_resources(priv);
2615         if (ret < 0) {
2616                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2617                            __func__);
2618                 goto dma_desc_error;
2619         }
2620
2621         ret = init_dma_desc_rings(dev, GFP_KERNEL);
2622         if (ret < 0) {
2623                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2624                            __func__);
2625                 goto init_error;
2626         }
2627
2628         ret = stmmac_hw_setup(dev, true);
2629         if (ret < 0) {
2630                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2631                 goto init_error;
2632         }
2633
2634         stmmac_init_tx_coalesce(priv);
2635
2636         if (dev->phydev)
2637                 phy_start(dev->phydev);
2638
2639         /* Request the IRQ lines */
2640         ret = request_irq(dev->irq, stmmac_interrupt,
2641                           IRQF_SHARED, dev->name, dev);
2642         if (unlikely(ret < 0)) {
2643                 netdev_err(priv->dev,
2644                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2645                            __func__, dev->irq, ret);
2646                 goto irq_error;
2647         }
2648
2649         /* Request the Wake IRQ in case of another line is used for WoL */
2650         if (priv->wol_irq != dev->irq) {
2651                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2652                                   IRQF_SHARED, dev->name, dev);
2653                 if (unlikely(ret < 0)) {
2654                         netdev_err(priv->dev,
2655                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2656                                    __func__, priv->wol_irq, ret);
2657                         goto wolirq_error;
2658                 }
2659         }
2660
2661         /* Request the IRQ lines */
2662         if (priv->lpi_irq > 0) {
2663                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2664                                   dev->name, dev);
2665                 if (unlikely(ret < 0)) {
2666                         netdev_err(priv->dev,
2667                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2668                                    __func__, priv->lpi_irq, ret);
2669                         goto lpiirq_error;
2670                 }
2671         }
2672
2673         stmmac_enable_all_queues(priv);
2674         stmmac_start_all_queues(priv);
2675
2676         return 0;
2677
2678 lpiirq_error:
2679         if (priv->wol_irq != dev->irq)
2680                 free_irq(priv->wol_irq, dev);
2681 wolirq_error:
2682         free_irq(dev->irq, dev);
2683 irq_error:
2684         if (dev->phydev)
2685                 phy_stop(dev->phydev);
2686
2687         for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2688                 del_timer_sync(&priv->tx_queue[chan].txtimer);
2689
2690         stmmac_hw_teardown(dev);
2691 init_error:
2692         free_dma_desc_resources(priv);
2693 dma_desc_error:
2694         if (dev->phydev)
2695                 phy_disconnect(dev->phydev);
2696
2697         return ret;
2698 }
2699
2700 /**
2701  *  stmmac_release - close entry point of the driver
2702  *  @dev : device pointer.
2703  *  Description:
2704  *  This is the stop entry point of the driver.
2705  */
2706 static int stmmac_release(struct net_device *dev)
2707 {
2708         struct stmmac_priv *priv = netdev_priv(dev);
2709         u32 chan;
2710
2711         if (priv->eee_enabled)
2712                 del_timer_sync(&priv->eee_ctrl_timer);
2713
2714         /* Stop and disconnect the PHY */
2715         if (dev->phydev) {
2716                 phy_stop(dev->phydev);
2717                 phy_disconnect(dev->phydev);
2718         }
2719
2720         stmmac_stop_all_queues(priv);
2721
2722         stmmac_disable_all_queues(priv);
2723
2724         for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2725                 del_timer_sync(&priv->tx_queue[chan].txtimer);
2726
2727         /* Free the IRQ lines */
2728         free_irq(dev->irq, dev);
2729         if (priv->wol_irq != dev->irq)
2730                 free_irq(priv->wol_irq, dev);
2731         if (priv->lpi_irq > 0)
2732                 free_irq(priv->lpi_irq, dev);
2733
2734         /* Stop TX/RX DMA and clear the descriptors */
2735         stmmac_stop_all_dma(priv);
2736
2737         /* Release and free the Rx/Tx resources */
2738         free_dma_desc_resources(priv);
2739
2740         /* Disable the MAC Rx/Tx */
2741         stmmac_mac_set(priv, priv->ioaddr, false);
2742
2743         netif_carrier_off(dev);
2744
2745         stmmac_release_ptp(priv);
2746
2747         return 0;
2748 }
2749
2750 /**
2751  *  stmmac_tso_allocator - close entry point of the driver
2752  *  @priv: driver private structure
2753  *  @des: buffer start address
2754  *  @total_len: total length to fill in descriptors
2755  *  @last_segmant: condition for the last descriptor
2756  *  @queue: TX queue index
2757  *  Description:
2758  *  This function fills descriptor and request new descriptors according to
2759  *  buffer length to fill
2760  */
2761 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2762                                  int total_len, bool last_segment, u32 queue)
2763 {
2764         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2765         struct dma_desc *desc;
2766         u32 buff_size;
2767         int tmp_len;
2768
2769         tmp_len = total_len;
2770
2771         while (tmp_len > 0) {
2772                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2773                 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2774                 desc = tx_q->dma_tx + tx_q->cur_tx;
2775
2776                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2777                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2778                             TSO_MAX_BUFF_SIZE : tmp_len;
2779
2780                 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
2781                                 0, 1,
2782                                 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2783                                 0, 0);
2784
2785                 tmp_len -= TSO_MAX_BUFF_SIZE;
2786         }
2787 }
2788
2789 /**
2790  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2791  *  @skb : the socket buffer
2792  *  @dev : device pointer
2793  *  Description: this is the transmit function that is called on TSO frames
2794  *  (support available on GMAC4 and newer chips).
2795  *  Diagram below show the ring programming in case of TSO frames:
2796  *
2797  *  First Descriptor
2798  *   --------
2799  *   | DES0 |---> buffer1 = L2/L3/L4 header
2800  *   | DES1 |---> TCP Payload (can continue on next descr...)
2801  *   | DES2 |---> buffer 1 and 2 len
2802  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2803  *   --------
2804  *      |
2805  *     ...
2806  *      |
2807  *   --------
2808  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2809  *   | DES1 | --|
2810  *   | DES2 | --> buffer 1 and 2 len
2811  *   | DES3 |
2812  *   --------
2813  *
2814  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2815  */
2816 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2817 {
2818         struct dma_desc *desc, *first, *mss_desc = NULL;
2819         struct stmmac_priv *priv = netdev_priv(dev);
2820         int nfrags = skb_shinfo(skb)->nr_frags;
2821         u32 queue = skb_get_queue_mapping(skb);
2822         unsigned int first_entry, des;
2823         struct stmmac_tx_queue *tx_q;
2824         int tmp_pay_len = 0;
2825         u32 pay_len, mss;
2826         u8 proto_hdr_len;
2827         int i;
2828
2829         tx_q = &priv->tx_queue[queue];
2830
2831         /* Compute header lengths */
2832         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2833
2834         /* Desc availability based on threshold should be enough safe */
2835         if (unlikely(stmmac_tx_avail(priv, queue) <
2836                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2837                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2838                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2839                                                                 queue));
2840                         /* This is a hard error, log it. */
2841                         netdev_err(priv->dev,
2842                                    "%s: Tx Ring full when queue awake\n",
2843                                    __func__);
2844                 }
2845                 return NETDEV_TX_BUSY;
2846         }
2847
2848         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2849
2850         mss = skb_shinfo(skb)->gso_size;
2851
2852         /* set new MSS value if needed */
2853         if (mss != tx_q->mss) {
2854                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2855                 stmmac_set_mss(priv, mss_desc, mss);
2856                 tx_q->mss = mss;
2857                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2858                 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2859         }
2860
2861         if (netif_msg_tx_queued(priv)) {
2862                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2863                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2864                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2865                         skb->data_len);
2866         }
2867
2868         first_entry = tx_q->cur_tx;
2869         WARN_ON(tx_q->tx_skbuff[first_entry]);
2870
2871         desc = tx_q->dma_tx + first_entry;
2872         first = desc;
2873
2874         /* first descriptor: fill Headers on Buf1 */
2875         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2876                              DMA_TO_DEVICE);
2877         if (dma_mapping_error(priv->device, des))
2878                 goto dma_map_err;
2879
2880         tx_q->tx_skbuff_dma[first_entry].buf = des;
2881         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2882
2883         first->des0 = cpu_to_le32(des);
2884
2885         /* Fill start of payload in buff2 of first descriptor */
2886         if (pay_len)
2887                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2888
2889         /* If needed take extra descriptors to fill the remaining payload */
2890         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2891
2892         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
2893
2894         /* Prepare fragments */
2895         for (i = 0; i < nfrags; i++) {
2896                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2897
2898                 des = skb_frag_dma_map(priv->device, frag, 0,
2899                                        skb_frag_size(frag),
2900                                        DMA_TO_DEVICE);
2901                 if (dma_mapping_error(priv->device, des))
2902                         goto dma_map_err;
2903
2904                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2905                                      (i == nfrags - 1), queue);
2906
2907                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
2908                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
2909                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
2910         }
2911
2912         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
2913
2914         /* Only the last descriptor gets to point to the skb. */
2915         tx_q->tx_skbuff[tx_q->cur_tx] = skb;
2916
2917         /* We've used all descriptors we need for this skb, however,
2918          * advance cur_tx so that it references a fresh descriptor.
2919          * ndo_start_xmit will fill this descriptor the next time it's
2920          * called and stmmac_tx_clean may clean up to this descriptor.
2921          */
2922         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2923
2924         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2925                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2926                           __func__);
2927                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
2928         }
2929
2930         dev->stats.tx_bytes += skb->len;
2931         priv->xstats.tx_tso_frames++;
2932         priv->xstats.tx_tso_nfrags += nfrags;
2933
2934         /* Manage tx mitigation */
2935         tx_q->tx_count_frames += nfrags + 1;
2936         if (priv->tx_coal_frames <= tx_q->tx_count_frames) {
2937                 stmmac_set_tx_ic(priv, desc);
2938                 priv->xstats.tx_set_ic_bit++;
2939                 tx_q->tx_count_frames = 0;
2940         } else {
2941                 stmmac_tx_timer_arm(priv, queue);
2942         }
2943
2944         skb_tx_timestamp(skb);
2945
2946         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2947                      priv->hwts_tx_en)) {
2948                 /* declare that device is doing timestamping */
2949                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2950                 stmmac_enable_tx_timestamp(priv, first);
2951         }
2952
2953         /* Complete the first descriptor before granting the DMA */
2954         stmmac_prepare_tso_tx_desc(priv, first, 1,
2955                         proto_hdr_len,
2956                         pay_len,
2957                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
2958                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2959
2960         /* If context desc is used to change MSS */
2961         if (mss_desc) {
2962                 /* Make sure that first descriptor has been completely
2963                  * written, including its own bit. This is because MSS is
2964                  * actually before first descriptor, so we need to make
2965                  * sure that MSS's own bit is the last thing written.
2966                  */
2967                 dma_wmb();
2968                 stmmac_set_tx_owner(priv, mss_desc);
2969         }
2970
2971         /* The own bit must be the latest setting done when prepare the
2972          * descriptor and then barrier is needed to make sure that
2973          * all is coherent before granting the DMA engine.
2974          */
2975         wmb();
2976
2977         if (netif_msg_pktdata(priv)) {
2978                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2979                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2980                         tx_q->cur_tx, first, nfrags);
2981
2982                 stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0);
2983
2984                 pr_info(">>> frame to be transmitted: ");
2985                 print_pkt(skb->data, skb_headlen(skb));
2986         }
2987
2988         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
2989
2990         tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc));
2991         stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
2992
2993         return NETDEV_TX_OK;
2994
2995 dma_map_err:
2996         dev_err(priv->device, "Tx dma map failed\n");
2997         dev_kfree_skb(skb);
2998         priv->dev->stats.tx_dropped++;
2999         return NETDEV_TX_OK;
3000 }
3001
3002 /**
3003  *  stmmac_xmit - Tx entry point of the driver
3004  *  @skb : the socket buffer
3005  *  @dev : device pointer
3006  *  Description : this is the tx entry point of the driver.
3007  *  It programs the chain or the ring and supports oversized frames
3008  *  and SG feature.
3009  */
3010 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
3011 {
3012         struct stmmac_priv *priv = netdev_priv(dev);
3013         unsigned int nopaged_len = skb_headlen(skb);
3014         int i, csum_insertion = 0, is_jumbo = 0;
3015         u32 queue = skb_get_queue_mapping(skb);
3016         int nfrags = skb_shinfo(skb)->nr_frags;
3017         int entry;
3018         unsigned int first_entry;
3019         struct dma_desc *desc, *first;
3020         struct stmmac_tx_queue *tx_q;
3021         unsigned int enh_desc;
3022         unsigned int des;
3023
3024         tx_q = &priv->tx_queue[queue];
3025
3026         /* Manage oversized TCP frames for GMAC4 device */
3027         if (skb_is_gso(skb) && priv->tso) {
3028                 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3029                         return stmmac_tso_xmit(skb, dev);
3030         }
3031
3032         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3033                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3034                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3035                                                                 queue));
3036                         /* This is a hard error, log it. */
3037                         netdev_err(priv->dev,
3038                                    "%s: Tx Ring full when queue awake\n",
3039                                    __func__);
3040                 }
3041                 return NETDEV_TX_BUSY;
3042         }
3043
3044         if (priv->tx_path_in_lpi_mode)
3045                 stmmac_disable_eee_mode(priv);
3046
3047         entry = tx_q->cur_tx;
3048         first_entry = entry;
3049         WARN_ON(tx_q->tx_skbuff[first_entry]);
3050
3051         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3052
3053         if (likely(priv->extend_desc))
3054                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3055         else
3056                 desc = tx_q->dma_tx + entry;
3057
3058         first = desc;
3059
3060         enh_desc = priv->plat->enh_desc;
3061         /* To program the descriptors according to the size of the frame */
3062         if (enh_desc)
3063                 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
3064
3065         if (unlikely(is_jumbo)) {
3066                 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
3067                 if (unlikely(entry < 0) && (entry != -EINVAL))
3068                         goto dma_map_err;
3069         }
3070
3071         for (i = 0; i < nfrags; i++) {
3072                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3073                 int len = skb_frag_size(frag);
3074                 bool last_segment = (i == (nfrags - 1));
3075
3076                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3077                 WARN_ON(tx_q->tx_skbuff[entry]);
3078
3079                 if (likely(priv->extend_desc))
3080                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3081                 else
3082                         desc = tx_q->dma_tx + entry;
3083
3084                 des = skb_frag_dma_map(priv->device, frag, 0, len,
3085                                        DMA_TO_DEVICE);
3086                 if (dma_mapping_error(priv->device, des))
3087                         goto dma_map_err; /* should reuse desc w/o issues */
3088
3089                 tx_q->tx_skbuff_dma[entry].buf = des;
3090
3091                 stmmac_set_desc_addr(priv, desc, des);
3092
3093                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3094                 tx_q->tx_skbuff_dma[entry].len = len;
3095                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3096
3097                 /* Prepare the descriptor and set the own bit too */
3098                 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
3099                                 priv->mode, 1, last_segment, skb->len);
3100         }
3101
3102         /* Only the last descriptor gets to point to the skb. */
3103         tx_q->tx_skbuff[entry] = skb;
3104
3105         /* We've used all descriptors we need for this skb, however,
3106          * advance cur_tx so that it references a fresh descriptor.
3107          * ndo_start_xmit will fill this descriptor the next time it's
3108          * called and stmmac_tx_clean may clean up to this descriptor.
3109          */
3110         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3111         tx_q->cur_tx = entry;
3112
3113         if (netif_msg_pktdata(priv)) {
3114                 void *tx_head;
3115
3116                 netdev_dbg(priv->dev,
3117                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3118                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3119                            entry, first, nfrags);
3120
3121                 if (priv->extend_desc)
3122                         tx_head = (void *)tx_q->dma_etx;
3123                 else
3124                         tx_head = (void *)tx_q->dma_tx;
3125
3126                 stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false);
3127
3128                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3129                 print_pkt(skb->data, skb->len);
3130         }
3131
3132         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3133                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3134                           __func__);
3135                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3136         }
3137
3138         dev->stats.tx_bytes += skb->len;
3139
3140         /* According to the coalesce parameter the IC bit for the latest
3141          * segment is reset and the timer re-started to clean the tx status.
3142          * This approach takes care about the fragments: desc is the first
3143          * element in case of no SG.
3144          */
3145         tx_q->tx_count_frames += nfrags + 1;
3146         if (priv->tx_coal_frames <= tx_q->tx_count_frames) {
3147                 stmmac_set_tx_ic(priv, desc);
3148                 priv->xstats.tx_set_ic_bit++;
3149                 tx_q->tx_count_frames = 0;
3150         } else {
3151                 stmmac_tx_timer_arm(priv, queue);
3152         }
3153
3154         skb_tx_timestamp(skb);
3155
3156         /* Ready to fill the first descriptor and set the OWN bit w/o any
3157          * problems because all the descriptors are actually ready to be
3158          * passed to the DMA engine.
3159          */
3160         if (likely(!is_jumbo)) {
3161                 bool last_segment = (nfrags == 0);
3162
3163                 des = dma_map_single(priv->device, skb->data,
3164                                      nopaged_len, DMA_TO_DEVICE);
3165                 if (dma_mapping_error(priv->device, des))
3166                         goto dma_map_err;
3167
3168                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3169
3170                 stmmac_set_desc_addr(priv, first, des);
3171
3172                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3173                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3174
3175                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3176                              priv->hwts_tx_en)) {
3177                         /* declare that device is doing timestamping */
3178                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3179                         stmmac_enable_tx_timestamp(priv, first);
3180                 }
3181
3182                 /* Prepare the first descriptor setting the OWN bit too */
3183                 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
3184                                 csum_insertion, priv->mode, 1, last_segment,
3185                                 skb->len);
3186
3187                 /* The own bit must be the latest setting done when prepare the
3188                  * descriptor and then barrier is needed to make sure that
3189                  * all is coherent before granting the DMA engine.
3190                  */
3191                 wmb();
3192         }
3193
3194         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3195
3196         stmmac_enable_dma_transmission(priv, priv->ioaddr);
3197
3198         tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc));
3199         stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3200
3201         return NETDEV_TX_OK;
3202
3203 dma_map_err:
3204         netdev_err(priv->dev, "Tx DMA map failed\n");
3205         dev_kfree_skb(skb);
3206         priv->dev->stats.tx_dropped++;
3207         return NETDEV_TX_OK;
3208 }
3209
3210 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3211 {
3212         struct vlan_ethhdr *veth;
3213         __be16 vlan_proto;
3214         u16 vlanid;
3215
3216         veth = (struct vlan_ethhdr *)skb->data;
3217         vlan_proto = veth->h_vlan_proto;
3218
3219         if ((vlan_proto == htons(ETH_P_8021Q) &&
3220              dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
3221             (vlan_proto == htons(ETH_P_8021AD) &&
3222              dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
3223                 /* pop the vlan tag */
3224                 vlanid = ntohs(veth->h_vlan_TCI);
3225                 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
3226                 skb_pull(skb, VLAN_HLEN);
3227                 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
3228         }
3229 }
3230
3231
3232 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3233 {
3234         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3235                 return 0;
3236
3237         return 1;
3238 }
3239
3240 /**
3241  * stmmac_rx_refill - refill used skb preallocated buffers
3242  * @priv: driver private structure
3243  * @queue: RX queue index
3244  * Description : this is to reallocate the skb for the reception process
3245  * that is based on zero-copy.
3246  */
3247 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3248 {
3249         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3250         int dirty = stmmac_rx_dirty(priv, queue);
3251         unsigned int entry = rx_q->dirty_rx;
3252
3253         int bfsize = priv->dma_buf_sz;
3254
3255         while (dirty-- > 0) {
3256                 struct dma_desc *p;
3257
3258                 if (priv->extend_desc)
3259                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3260                 else
3261                         p = rx_q->dma_rx + entry;
3262
3263                 if (likely(!rx_q->rx_skbuff[entry])) {
3264                         struct sk_buff *skb;
3265
3266                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3267                         if (unlikely(!skb)) {
3268                                 /* so for a while no zero-copy! */
3269                                 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3270                                 if (unlikely(net_ratelimit()))
3271                                         dev_err(priv->device,
3272                                                 "fail to alloc skb entry %d\n",
3273                                                 entry);
3274                                 break;
3275                         }
3276
3277                         rx_q->rx_skbuff[entry] = skb;
3278                         rx_q->rx_skbuff_dma[entry] =
3279                             dma_map_single(priv->device, skb->data, bfsize,
3280                                            DMA_FROM_DEVICE);
3281                         if (dma_mapping_error(priv->device,
3282                                               rx_q->rx_skbuff_dma[entry])) {
3283                                 netdev_err(priv->dev, "Rx DMA map failed\n");
3284                                 dev_kfree_skb(skb);
3285                                 break;
3286                         }
3287
3288                         stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[entry]);
3289                         stmmac_refill_desc3(priv, rx_q, p);
3290
3291                         if (rx_q->rx_zeroc_thresh > 0)
3292                                 rx_q->rx_zeroc_thresh--;
3293
3294                         netif_dbg(priv, rx_status, priv->dev,
3295                                   "refill entry #%d\n", entry);
3296                 }
3297                 dma_wmb();
3298
3299                 stmmac_set_rx_owner(priv, p, priv->use_riwt);
3300
3301                 dma_wmb();
3302
3303                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3304         }
3305         rx_q->dirty_rx = entry;
3306 }
3307
3308 /**
3309  * stmmac_rx - manage the receive process
3310  * @priv: driver private structure
3311  * @limit: napi bugget
3312  * @queue: RX queue index.
3313  * Description :  this the function called by the napi poll method.
3314  * It gets all the frames inside the ring.
3315  */
3316 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3317 {
3318         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3319         struct stmmac_channel *ch = &priv->channel[queue];
3320         unsigned int entry = rx_q->cur_rx;
3321         int coe = priv->hw->rx_csum;
3322         unsigned int next_entry;
3323         unsigned int count = 0;
3324         bool xmac;
3325
3326         xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
3327
3328         if (netif_msg_rx_status(priv)) {
3329                 void *rx_head;
3330
3331                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3332                 if (priv->extend_desc)
3333                         rx_head = (void *)rx_q->dma_erx;
3334                 else
3335                         rx_head = (void *)rx_q->dma_rx;
3336
3337                 stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true);
3338         }
3339         while (count < limit) {
3340                 int status;
3341                 struct dma_desc *p;
3342                 struct dma_desc *np;
3343
3344                 if (priv->extend_desc)
3345                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3346                 else
3347                         p = rx_q->dma_rx + entry;
3348
3349                 /* read the status of the incoming frame */
3350                 status = stmmac_rx_status(priv, &priv->dev->stats,
3351                                 &priv->xstats, p);
3352                 /* check if managed by the DMA otherwise go ahead */
3353                 if (unlikely(status & dma_own))
3354                         break;
3355
3356                 count++;
3357
3358                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3359                 next_entry = rx_q->cur_rx;
3360
3361                 if (priv->extend_desc)
3362                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3363                 else
3364                         np = rx_q->dma_rx + next_entry;
3365
3366                 prefetch(np);
3367
3368                 if (priv->extend_desc)
3369                         stmmac_rx_extended_status(priv, &priv->dev->stats,
3370                                         &priv->xstats, rx_q->dma_erx + entry);
3371                 if (unlikely(status == discard_frame)) {
3372                         priv->dev->stats.rx_errors++;
3373                         if (priv->hwts_rx_en && !priv->extend_desc) {
3374                                 /* DESC2 & DESC3 will be overwritten by device
3375                                  * with timestamp value, hence reinitialize
3376                                  * them in stmmac_rx_refill() function so that
3377                                  * device can reuse it.
3378                                  */
3379                                 dev_kfree_skb_any(rx_q->rx_skbuff[entry]);
3380                                 rx_q->rx_skbuff[entry] = NULL;
3381                                 dma_unmap_single(priv->device,
3382                                                  rx_q->rx_skbuff_dma[entry],
3383                                                  priv->dma_buf_sz,
3384                                                  DMA_FROM_DEVICE);
3385                         }
3386                 } else {
3387                         struct sk_buff *skb;
3388                         int frame_len;
3389                         unsigned int des;
3390
3391                         stmmac_get_desc_addr(priv, p, &des);
3392                         frame_len = stmmac_get_rx_frame_len(priv, p, coe);
3393
3394                         /*  If frame length is greater than skb buffer size
3395                          *  (preallocated during init) then the packet is
3396                          *  ignored
3397                          */
3398                         if (frame_len > priv->dma_buf_sz) {
3399                                 netdev_err(priv->dev,
3400                                            "len %d larger than size (%d)\n",
3401                                            frame_len, priv->dma_buf_sz);
3402                                 priv->dev->stats.rx_length_errors++;
3403                                 break;
3404                         }
3405
3406                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3407                          * Type frames (LLC/LLC-SNAP)
3408                          *
3409                          * llc_snap is never checked in GMAC >= 4, so this ACS
3410                          * feature is always disabled and packets need to be
3411                          * stripped manually.
3412                          */
3413                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
3414                             unlikely(status != llc_snap))
3415                                 frame_len -= ETH_FCS_LEN;
3416
3417                         if (netif_msg_rx_status(priv)) {
3418                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
3419                                            p, entry, des);
3420                                 netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
3421                                            frame_len, status);
3422                         }
3423
3424                         /* The zero-copy is always used for all the sizes
3425                          * in case of GMAC4 because it needs
3426                          * to refill the used descriptors, always.
3427                          */
3428                         if (unlikely(!xmac &&
3429                                      ((frame_len < priv->rx_copybreak) ||
3430                                      stmmac_rx_threshold_count(rx_q)))) {
3431                                 skb = netdev_alloc_skb_ip_align(priv->dev,
3432                                                                 frame_len);
3433                                 if (unlikely(!skb)) {
3434                                         if (net_ratelimit())
3435                                                 dev_warn(priv->device,
3436                                                          "packet dropped\n");
3437                                         priv->dev->stats.rx_dropped++;
3438                                         break;
3439                                 }
3440
3441                                 dma_sync_single_for_cpu(priv->device,
3442                                                         rx_q->rx_skbuff_dma
3443                                                         [entry], frame_len,
3444                                                         DMA_FROM_DEVICE);
3445                                 skb_copy_to_linear_data(skb,
3446                                                         rx_q->
3447                                                         rx_skbuff[entry]->data,
3448                                                         frame_len);
3449
3450                                 skb_put(skb, frame_len);
3451                                 dma_sync_single_for_device(priv->device,
3452                                                            rx_q->rx_skbuff_dma
3453                                                            [entry], frame_len,
3454                                                            DMA_FROM_DEVICE);
3455                         } else {
3456                                 skb = rx_q->rx_skbuff[entry];
3457                                 if (unlikely(!skb)) {
3458                                         netdev_err(priv->dev,
3459                                                    "%s: Inconsistent Rx chain\n",
3460                                                    priv->dev->name);
3461                                         priv->dev->stats.rx_dropped++;
3462                                         break;
3463                                 }
3464                                 prefetch(skb->data - NET_IP_ALIGN);
3465                                 rx_q->rx_skbuff[entry] = NULL;
3466                                 rx_q->rx_zeroc_thresh++;
3467
3468                                 skb_put(skb, frame_len);
3469                                 dma_unmap_single(priv->device,
3470                                                  rx_q->rx_skbuff_dma[entry],
3471                                                  priv->dma_buf_sz,
3472                                                  DMA_FROM_DEVICE);
3473                         }
3474
3475                         if (netif_msg_pktdata(priv)) {
3476                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3477                                            frame_len);
3478                                 print_pkt(skb->data, frame_len);
3479                         }
3480
3481                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3482
3483                         stmmac_rx_vlan(priv->dev, skb);
3484
3485                         skb->protocol = eth_type_trans(skb, priv->dev);
3486
3487                         if (unlikely(!coe))
3488                                 skb_checksum_none_assert(skb);
3489                         else
3490                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3491
3492                         napi_gro_receive(&ch->napi, skb);
3493
3494                         priv->dev->stats.rx_packets++;
3495                         priv->dev->stats.rx_bytes += frame_len;
3496                 }
3497                 entry = next_entry;
3498         }
3499
3500         stmmac_rx_refill(priv, queue);
3501
3502         priv->xstats.rx_pkt_n += count;
3503
3504         return count;
3505 }
3506
3507 /**
3508  *  stmmac_poll - stmmac poll method (NAPI)
3509  *  @napi : pointer to the napi structure.
3510  *  @budget : maximum number of packets that the current CPU can receive from
3511  *            all interfaces.
3512  *  Description :
3513  *  To look at the incoming frames and clear the tx resources.
3514  */
3515 static int stmmac_napi_poll(struct napi_struct *napi, int budget)
3516 {
3517         struct stmmac_channel *ch =
3518                 container_of(napi, struct stmmac_channel, napi);
3519         struct stmmac_priv *priv = ch->priv_data;
3520         int work_done, rx_done = 0, tx_done = 0;
3521         u32 chan = ch->index;
3522
3523         priv->xstats.napi_poll++;
3524
3525         if (ch->has_tx)
3526                 tx_done = stmmac_tx_clean(priv, budget, chan);
3527         if (ch->has_rx)
3528                 rx_done = stmmac_rx(priv, budget, chan);
3529
3530         work_done = max(rx_done, tx_done);
3531         work_done = min(work_done, budget);
3532
3533         if (work_done < budget && napi_complete_done(napi, work_done)) {
3534                 int stat;
3535
3536                 stmmac_enable_dma_irq(priv, priv->ioaddr, chan);
3537                 stat = stmmac_dma_interrupt_status(priv, priv->ioaddr,
3538                                                    &priv->xstats, chan);
3539                 if (stat && napi_reschedule(napi))
3540                         stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
3541         }
3542
3543         return work_done;
3544 }
3545
3546 /**
3547  *  stmmac_tx_timeout
3548  *  @dev : Pointer to net device structure
3549  *  Description: this function is called when a packet transmission fails to
3550  *   complete within a reasonable time. The driver will mark the error in the
3551  *   netdev structure and arrange for the device to be reset to a sane state
3552  *   in order to transmit a new packet.
3553  */
3554 static void stmmac_tx_timeout(struct net_device *dev)
3555 {
3556         struct stmmac_priv *priv = netdev_priv(dev);
3557
3558         stmmac_global_err(priv);
3559 }
3560
3561 /**
3562  *  stmmac_set_rx_mode - entry point for multicast addressing
3563  *  @dev : pointer to the device structure
3564  *  Description:
3565  *  This function is a driver entry point which gets called by the kernel
3566  *  whenever multicast addresses must be enabled/disabled.
3567  *  Return value:
3568  *  void.
3569  */
3570 static void stmmac_set_rx_mode(struct net_device *dev)
3571 {
3572         struct stmmac_priv *priv = netdev_priv(dev);
3573
3574         stmmac_set_filter(priv, priv->hw, dev);
3575 }
3576
3577 /**
3578  *  stmmac_change_mtu - entry point to change MTU size for the device.
3579  *  @dev : device pointer.
3580  *  @new_mtu : the new MTU size for the device.
3581  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3582  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3583  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3584  *  Return value:
3585  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3586  *  file on failure.
3587  */
3588 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3589 {
3590         struct stmmac_priv *priv = netdev_priv(dev);
3591
3592         if (netif_running(dev)) {
3593                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3594                 return -EBUSY;
3595         }
3596
3597         dev->mtu = new_mtu;
3598
3599         netdev_update_features(dev);
3600
3601         return 0;
3602 }
3603
3604 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3605                                              netdev_features_t features)
3606 {
3607         struct stmmac_priv *priv = netdev_priv(dev);
3608
3609         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3610                 features &= ~NETIF_F_RXCSUM;
3611
3612         if (!priv->plat->tx_coe)
3613                 features &= ~NETIF_F_CSUM_MASK;
3614
3615         /* Some GMAC devices have a bugged Jumbo frame support that
3616          * needs to have the Tx COE disabled for oversized frames
3617          * (due to limited buffer sizes). In this case we disable
3618          * the TX csum insertion in the TDES and not use SF.
3619          */
3620         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3621                 features &= ~NETIF_F_CSUM_MASK;
3622
3623         /* Disable tso if asked by ethtool */
3624         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3625                 if (features & NETIF_F_TSO)
3626                         priv->tso = true;
3627                 else
3628                         priv->tso = false;
3629         }
3630
3631         return features;
3632 }
3633
3634 static int stmmac_set_features(struct net_device *netdev,
3635                                netdev_features_t features)
3636 {
3637         struct stmmac_priv *priv = netdev_priv(netdev);
3638
3639         /* Keep the COE Type in case of csum is supporting */
3640         if (features & NETIF_F_RXCSUM)
3641                 priv->hw->rx_csum = priv->plat->rx_coe;
3642         else
3643                 priv->hw->rx_csum = 0;
3644         /* No check needed because rx_coe has been set before and it will be
3645          * fixed in case of issue.
3646          */
3647         stmmac_rx_ipc(priv, priv->hw);
3648
3649         return 0;
3650 }
3651
3652 /**
3653  *  stmmac_interrupt - main ISR
3654  *  @irq: interrupt number.
3655  *  @dev_id: to pass the net device pointer.
3656  *  Description: this is the main driver interrupt service routine.
3657  *  It can call:
3658  *  o DMA service routine (to manage incoming frame reception and transmission
3659  *    status)
3660  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3661  *    interrupts.
3662  */
3663 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3664 {
3665         struct net_device *dev = (struct net_device *)dev_id;
3666         struct stmmac_priv *priv = netdev_priv(dev);
3667         u32 rx_cnt = priv->plat->rx_queues_to_use;
3668         u32 tx_cnt = priv->plat->tx_queues_to_use;
3669         u32 queues_count;
3670         u32 queue;
3671         bool xmac;
3672
3673         xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
3674         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3675
3676         if (priv->irq_wake)
3677                 pm_wakeup_event(priv->device, 0);
3678
3679         if (unlikely(!dev)) {
3680                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3681                 return IRQ_NONE;
3682         }
3683
3684         /* Check if adapter is up */
3685         if (test_bit(STMMAC_DOWN, &priv->state))
3686                 return IRQ_HANDLED;
3687         /* Check if a fatal error happened */
3688         if (stmmac_safety_feat_interrupt(priv))
3689                 return IRQ_HANDLED;
3690
3691         /* To handle GMAC own interrupts */
3692         if ((priv->plat->has_gmac) || xmac) {
3693                 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
3694                 int mtl_status;
3695
3696                 if (unlikely(status)) {
3697                         /* For LPI we need to save the tx status */
3698                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3699                                 priv->tx_path_in_lpi_mode = true;
3700                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3701                                 priv->tx_path_in_lpi_mode = false;
3702                 }
3703
3704                 for (queue = 0; queue < queues_count; queue++) {
3705                         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3706
3707                         mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw,
3708                                                                 queue);
3709                         if (mtl_status != -EINVAL)
3710                                 status |= mtl_status;
3711
3712                         if (status & CORE_IRQ_MTL_RX_OVERFLOW)
3713                                 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
3714                                                        rx_q->rx_tail_addr,
3715                                                        queue);
3716                 }
3717
3718                 /* PCS link status */
3719                 if (priv->hw->pcs) {
3720                         if (priv->xstats.pcs_link)
3721                                 netif_carrier_on(dev);
3722                         else
3723                                 netif_carrier_off(dev);
3724                 }
3725         }
3726
3727         /* To handle DMA interrupts */
3728         stmmac_dma_interrupt(priv);
3729
3730         return IRQ_HANDLED;
3731 }
3732
3733 #ifdef CONFIG_NET_POLL_CONTROLLER
3734 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3735  * to allow network I/O with interrupts disabled.
3736  */
3737 static void stmmac_poll_controller(struct net_device *dev)
3738 {
3739         disable_irq(dev->irq);
3740         stmmac_interrupt(dev->irq, dev);
3741         enable_irq(dev->irq);
3742 }
3743 #endif
3744
3745 /**
3746  *  stmmac_ioctl - Entry point for the Ioctl
3747  *  @dev: Device pointer.
3748  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3749  *  a proprietary structure used to pass information to the driver.
3750  *  @cmd: IOCTL command
3751  *  Description:
3752  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3753  */
3754 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3755 {
3756         int ret = -EOPNOTSUPP;
3757
3758         if (!netif_running(dev))
3759                 return -EINVAL;
3760
3761         switch (cmd) {
3762         case SIOCGMIIPHY:
3763         case SIOCGMIIREG:
3764         case SIOCSMIIREG:
3765                 if (!dev->phydev)
3766                         return -EINVAL;
3767                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3768                 break;
3769         case SIOCSHWTSTAMP:
3770                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3771                 break;
3772         default:
3773                 break;
3774         }
3775
3776         return ret;
3777 }
3778
3779 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
3780                                     void *cb_priv)
3781 {
3782         struct stmmac_priv *priv = cb_priv;
3783         int ret = -EOPNOTSUPP;
3784
3785         stmmac_disable_all_queues(priv);
3786
3787         switch (type) {
3788         case TC_SETUP_CLSU32:
3789                 if (tc_cls_can_offload_and_chain0(priv->dev, type_data))
3790                         ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
3791                 break;
3792         default:
3793                 break;
3794         }
3795
3796         stmmac_enable_all_queues(priv);
3797         return ret;
3798 }
3799
3800 static int stmmac_setup_tc_block(struct stmmac_priv *priv,
3801                                  struct tc_block_offload *f)
3802 {
3803         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
3804                 return -EOPNOTSUPP;
3805
3806         switch (f->command) {
3807         case TC_BLOCK_BIND:
3808                 return tcf_block_cb_register(f->block, stmmac_setup_tc_block_cb,
3809                                 priv, priv, f->extack);
3810         case TC_BLOCK_UNBIND:
3811                 tcf_block_cb_unregister(f->block, stmmac_setup_tc_block_cb, priv);
3812                 return 0;
3813         default:
3814                 return -EOPNOTSUPP;
3815         }
3816 }
3817
3818 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
3819                            void *type_data)
3820 {
3821         struct stmmac_priv *priv = netdev_priv(ndev);
3822
3823         switch (type) {
3824         case TC_SETUP_BLOCK:
3825                 return stmmac_setup_tc_block(priv, type_data);
3826         case TC_SETUP_QDISC_CBS:
3827                 return stmmac_tc_setup_cbs(priv, priv, type_data);
3828         default:
3829                 return -EOPNOTSUPP;
3830         }
3831 }
3832
3833 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
3834 {
3835         struct stmmac_priv *priv = netdev_priv(ndev);
3836         int ret = 0;
3837
3838         ret = eth_mac_addr(ndev, addr);
3839         if (ret)
3840                 return ret;
3841
3842         stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
3843
3844         return ret;
3845 }
3846
3847 #ifdef CONFIG_DEBUG_FS
3848 static struct dentry *stmmac_fs_dir;
3849
3850 static void sysfs_display_ring(void *head, int size, int extend_desc,
3851                                struct seq_file *seq)
3852 {
3853         int i;
3854         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3855         struct dma_desc *p = (struct dma_desc *)head;
3856
3857         for (i = 0; i < size; i++) {
3858                 if (extend_desc) {
3859                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3860                                    i, (unsigned int)virt_to_phys(ep),
3861                                    le32_to_cpu(ep->basic.des0),
3862                                    le32_to_cpu(ep->basic.des1),
3863                                    le32_to_cpu(ep->basic.des2),
3864                                    le32_to_cpu(ep->basic.des3));
3865                         ep++;
3866                 } else {
3867                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3868                                    i, (unsigned int)virt_to_phys(p),
3869                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3870                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3871                         p++;
3872                 }
3873                 seq_printf(seq, "\n");
3874         }
3875 }
3876
3877 static int stmmac_rings_status_show(struct seq_file *seq, void *v)
3878 {
3879         struct net_device *dev = seq->private;
3880         struct stmmac_priv *priv = netdev_priv(dev);
3881         u32 rx_count = priv->plat->rx_queues_to_use;
3882         u32 tx_count = priv->plat->tx_queues_to_use;
3883         u32 queue;
3884
3885         if ((dev->flags & IFF_UP) == 0)
3886                 return 0;
3887
3888         for (queue = 0; queue < rx_count; queue++) {
3889                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3890
3891                 seq_printf(seq, "RX Queue %d:\n", queue);
3892
3893                 if (priv->extend_desc) {
3894                         seq_printf(seq, "Extended descriptor ring:\n");
3895                         sysfs_display_ring((void *)rx_q->dma_erx,
3896                                            DMA_RX_SIZE, 1, seq);
3897                 } else {
3898                         seq_printf(seq, "Descriptor ring:\n");
3899                         sysfs_display_ring((void *)rx_q->dma_rx,
3900                                            DMA_RX_SIZE, 0, seq);
3901                 }
3902         }
3903
3904         for (queue = 0; queue < tx_count; queue++) {
3905                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3906
3907                 seq_printf(seq, "TX Queue %d:\n", queue);
3908
3909                 if (priv->extend_desc) {
3910                         seq_printf(seq, "Extended descriptor ring:\n");
3911                         sysfs_display_ring((void *)tx_q->dma_etx,
3912                                            DMA_TX_SIZE, 1, seq);
3913                 } else {
3914                         seq_printf(seq, "Descriptor ring:\n");
3915                         sysfs_display_ring((void *)tx_q->dma_tx,
3916                                            DMA_TX_SIZE, 0, seq);
3917                 }
3918         }
3919
3920         return 0;
3921 }
3922 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
3923
3924 static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
3925 {
3926         struct net_device *dev = seq->private;
3927         struct stmmac_priv *priv = netdev_priv(dev);
3928
3929         if (!priv->hw_cap_support) {
3930                 seq_printf(seq, "DMA HW features not supported\n");
3931                 return 0;
3932         }
3933
3934         seq_printf(seq, "==============================\n");
3935         seq_printf(seq, "\tDMA HW features\n");
3936         seq_printf(seq, "==============================\n");
3937
3938         seq_printf(seq, "\t10/100 Mbps: %s\n",
3939                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3940         seq_printf(seq, "\t1000 Mbps: %s\n",
3941                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3942         seq_printf(seq, "\tHalf duplex: %s\n",
3943                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3944         seq_printf(seq, "\tHash Filter: %s\n",
3945                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3946         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3947                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3948         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3949                    (priv->dma_cap.pcs) ? "Y" : "N");
3950         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3951                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3952         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3953                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3954         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3955                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3956         seq_printf(seq, "\tRMON module: %s\n",
3957                    (priv->dma_cap.rmon) ? "Y" : "N");
3958         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3959                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3960         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3961                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3962         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3963                    (priv->dma_cap.eee) ? "Y" : "N");
3964         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3965         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3966                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3967         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3968                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3969                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3970         } else {
3971                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3972                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3973                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3974                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3975         }
3976         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3977                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3978         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3979                    priv->dma_cap.number_rx_channel);
3980         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3981                    priv->dma_cap.number_tx_channel);
3982         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3983                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3984
3985         return 0;
3986 }
3987 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
3988
3989 static int stmmac_init_fs(struct net_device *dev)
3990 {
3991         struct stmmac_priv *priv = netdev_priv(dev);
3992
3993         /* Create per netdev entries */
3994         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3995
3996         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3997                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3998
3999                 return -ENOMEM;
4000         }
4001
4002         /* Entry to report DMA RX/TX rings */
4003         priv->dbgfs_rings_status =
4004                 debugfs_create_file("descriptors_status", 0444,
4005                                     priv->dbgfs_dir, dev,
4006                                     &stmmac_rings_status_fops);
4007
4008         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
4009                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
4010                 debugfs_remove_recursive(priv->dbgfs_dir);
4011
4012                 return -ENOMEM;
4013         }
4014
4015         /* Entry to report the DMA HW features */
4016         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", 0444,
4017                                                   priv->dbgfs_dir,
4018                                                   dev, &stmmac_dma_cap_fops);
4019
4020         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
4021                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
4022                 debugfs_remove_recursive(priv->dbgfs_dir);
4023
4024                 return -ENOMEM;
4025         }
4026
4027         return 0;
4028 }
4029
4030 static void stmmac_exit_fs(struct net_device *dev)
4031 {
4032         struct stmmac_priv *priv = netdev_priv(dev);
4033
4034         debugfs_remove_recursive(priv->dbgfs_dir);
4035 }
4036 #endif /* CONFIG_DEBUG_FS */
4037
4038 static const struct net_device_ops stmmac_netdev_ops = {
4039         .ndo_open = stmmac_open,
4040         .ndo_start_xmit = stmmac_xmit,
4041         .ndo_stop = stmmac_release,
4042         .ndo_change_mtu = stmmac_change_mtu,
4043         .ndo_fix_features = stmmac_fix_features,
4044         .ndo_set_features = stmmac_set_features,
4045         .ndo_set_rx_mode = stmmac_set_rx_mode,
4046         .ndo_tx_timeout = stmmac_tx_timeout,
4047         .ndo_do_ioctl = stmmac_ioctl,
4048         .ndo_setup_tc = stmmac_setup_tc,
4049 #ifdef CONFIG_NET_POLL_CONTROLLER
4050         .ndo_poll_controller = stmmac_poll_controller,
4051 #endif
4052         .ndo_set_mac_address = stmmac_set_mac_address,
4053 };
4054
4055 static void stmmac_reset_subtask(struct stmmac_priv *priv)
4056 {
4057         if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
4058                 return;
4059         if (test_bit(STMMAC_DOWN, &priv->state))
4060                 return;
4061
4062         netdev_err(priv->dev, "Reset adapter.\n");
4063
4064         rtnl_lock();
4065         netif_trans_update(priv->dev);
4066         while (test_and_set_bit(STMMAC_RESETING, &priv->state))
4067                 usleep_range(1000, 2000);
4068
4069         set_bit(STMMAC_DOWN, &priv->state);
4070         dev_close(priv->dev);
4071         dev_open(priv->dev, NULL);
4072         clear_bit(STMMAC_DOWN, &priv->state);
4073         clear_bit(STMMAC_RESETING, &priv->state);
4074         rtnl_unlock();
4075 }
4076
4077 static void stmmac_service_task(struct work_struct *work)
4078 {
4079         struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
4080                         service_task);
4081
4082         stmmac_reset_subtask(priv);
4083         clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
4084 }
4085
4086 /**
4087  *  stmmac_hw_init - Init the MAC device
4088  *  @priv: driver private structure
4089  *  Description: this function is to configure the MAC device according to
4090  *  some platform parameters or the HW capability register. It prepares the
4091  *  driver to use either ring or chain modes and to setup either enhanced or
4092  *  normal descriptors.
4093  */
4094 static int stmmac_hw_init(struct stmmac_priv *priv)
4095 {
4096         int ret;
4097
4098         /* dwmac-sun8i only work in chain mode */
4099         if (priv->plat->has_sun8i)
4100                 chain_mode = 1;
4101         priv->chain_mode = chain_mode;
4102
4103         /* Initialize HW Interface */
4104         ret = stmmac_hwif_init(priv);
4105         if (ret)
4106                 return ret;
4107
4108         /* Get the HW capability (new GMAC newer than 3.50a) */
4109         priv->hw_cap_support = stmmac_get_hw_features(priv);
4110         if (priv->hw_cap_support) {
4111                 dev_info(priv->device, "DMA HW capability register supported\n");
4112
4113                 /* We can override some gmac/dma configuration fields: e.g.
4114                  * enh_desc, tx_coe (e.g. that are passed through the
4115                  * platform) with the values from the HW capability
4116                  * register (if supported).
4117                  */
4118                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
4119                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4120                 priv->hw->pmt = priv->plat->pmt;
4121
4122                 /* TXCOE doesn't work in thresh DMA mode */
4123                 if (priv->plat->force_thresh_dma_mode)
4124                         priv->plat->tx_coe = 0;
4125                 else
4126                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
4127
4128                 /* In case of GMAC4 rx_coe is from HW cap register. */
4129                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
4130
4131                 if (priv->dma_cap.rx_coe_type2)
4132                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4133                 else if (priv->dma_cap.rx_coe_type1)
4134                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4135
4136         } else {
4137                 dev_info(priv->device, "No HW DMA feature register supported\n");
4138         }
4139
4140         if (priv->plat->rx_coe) {
4141                 priv->hw->rx_csum = priv->plat->rx_coe;
4142                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4143                 if (priv->synopsys_id < DWMAC_CORE_4_00)
4144                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4145         }
4146         if (priv->plat->tx_coe)
4147                 dev_info(priv->device, "TX Checksum insertion supported\n");
4148
4149         if (priv->plat->pmt) {
4150                 dev_info(priv->device, "Wake-Up On Lan supported\n");
4151                 device_set_wakeup_capable(priv->device, 1);
4152         }
4153
4154         if (priv->dma_cap.tsoen)
4155                 dev_info(priv->device, "TSO supported\n");
4156
4157         /* Run HW quirks, if any */
4158         if (priv->hwif_quirks) {
4159                 ret = priv->hwif_quirks(priv);
4160                 if (ret)
4161                         return ret;
4162         }
4163
4164         /* Rx Watchdog is available in the COREs newer than the 3.40.
4165          * In some case, for example on bugged HW this feature
4166          * has to be disable and this can be done by passing the
4167          * riwt_off field from the platform.
4168          */
4169         if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
4170             (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
4171                 priv->use_riwt = 1;
4172                 dev_info(priv->device,
4173                          "Enable RX Mitigation via HW Watchdog Timer\n");
4174         }
4175
4176         return 0;
4177 }
4178
4179 /**
4180  * stmmac_dvr_probe
4181  * @device: device pointer
4182  * @plat_dat: platform data pointer
4183  * @res: stmmac resource pointer
4184  * Description: this is the main probe function used to
4185  * call the alloc_etherdev, allocate the priv structure.
4186  * Return:
4187  * returns 0 on success, otherwise errno.
4188  */
4189 int stmmac_dvr_probe(struct device *device,
4190                      struct plat_stmmacenet_data *plat_dat,
4191                      struct stmmac_resources *res)
4192 {
4193         struct net_device *ndev = NULL;
4194         struct stmmac_priv *priv;
4195         u32 queue, maxq;
4196         int ret = 0;
4197
4198         ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
4199                                   MTL_MAX_TX_QUEUES,
4200                                   MTL_MAX_RX_QUEUES);
4201         if (!ndev)
4202                 return -ENOMEM;
4203
4204         SET_NETDEV_DEV(ndev, device);
4205
4206         priv = netdev_priv(ndev);
4207         priv->device = device;
4208         priv->dev = ndev;
4209
4210         stmmac_set_ethtool_ops(ndev);
4211         priv->pause = pause;
4212         priv->plat = plat_dat;
4213         priv->ioaddr = res->addr;
4214         priv->dev->base_addr = (unsigned long)res->addr;
4215
4216         priv->dev->irq = res->irq;
4217         priv->wol_irq = res->wol_irq;
4218         priv->lpi_irq = res->lpi_irq;
4219
4220         if (res->mac)
4221                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4222
4223         dev_set_drvdata(device, priv->dev);
4224
4225         /* Verify driver arguments */
4226         stmmac_verify_args();
4227
4228         /* Allocate workqueue */
4229         priv->wq = create_singlethread_workqueue("stmmac_wq");
4230         if (!priv->wq) {
4231                 dev_err(priv->device, "failed to create workqueue\n");
4232                 ret = -ENOMEM;
4233                 goto error_wq;
4234         }
4235
4236         INIT_WORK(&priv->service_task, stmmac_service_task);
4237
4238         /* Override with kernel parameters if supplied XXX CRS XXX
4239          * this needs to have multiple instances
4240          */
4241         if ((phyaddr >= 0) && (phyaddr <= 31))
4242                 priv->plat->phy_addr = phyaddr;
4243
4244         if (priv->plat->stmmac_rst) {
4245                 ret = reset_control_assert(priv->plat->stmmac_rst);
4246                 reset_control_deassert(priv->plat->stmmac_rst);
4247                 /* Some reset controllers have only reset callback instead of
4248                  * assert + deassert callbacks pair.
4249                  */
4250                 if (ret == -ENOTSUPP)
4251                         reset_control_reset(priv->plat->stmmac_rst);
4252         }
4253
4254         /* Init MAC and get the capabilities */
4255         ret = stmmac_hw_init(priv);
4256         if (ret)
4257                 goto error_hw_init;
4258
4259         /* Configure real RX and TX queues */
4260         netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4261         netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4262
4263         ndev->netdev_ops = &stmmac_netdev_ops;
4264
4265         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4266                             NETIF_F_RXCSUM;
4267
4268         ret = stmmac_tc_init(priv, priv);
4269         if (!ret) {
4270                 ndev->hw_features |= NETIF_F_HW_TC;
4271         }
4272
4273         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4274                 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4275                 priv->tso = true;
4276                 dev_info(priv->device, "TSO feature enabled\n");
4277         }
4278         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4279         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4280 #ifdef STMMAC_VLAN_TAG_USED
4281         /* Both mac100 and gmac support receive VLAN tag detection */
4282         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
4283 #endif
4284         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4285
4286         /* MTU range: 46 - hw-specific max */
4287         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4288         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4289                 ndev->max_mtu = JUMBO_LEN;
4290         else if (priv->plat->has_xgmac)
4291                 ndev->max_mtu = XGMAC_JUMBO_LEN;
4292         else
4293                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4294         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4295          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4296          */
4297         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4298             (priv->plat->maxmtu >= ndev->min_mtu))
4299                 ndev->max_mtu = priv->plat->maxmtu;
4300         else if (priv->plat->maxmtu < ndev->min_mtu)
4301                 dev_warn(priv->device,
4302                          "%s: warning: maxmtu having invalid value (%d)\n",
4303                          __func__, priv->plat->maxmtu);
4304
4305         if (flow_ctrl)
4306                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4307
4308         /* Setup channels NAPI */
4309         maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
4310
4311         for (queue = 0; queue < maxq; queue++) {
4312                 struct stmmac_channel *ch = &priv->channel[queue];
4313
4314                 ch->priv_data = priv;
4315                 ch->index = queue;
4316
4317                 if (queue < priv->plat->rx_queues_to_use)
4318                         ch->has_rx = true;
4319                 if (queue < priv->plat->tx_queues_to_use)
4320                         ch->has_tx = true;
4321
4322                 netif_napi_add(ndev, &ch->napi, stmmac_napi_poll,
4323                                NAPI_POLL_WEIGHT);
4324         }
4325
4326         mutex_init(&priv->lock);
4327
4328         /* If a specific clk_csr value is passed from the platform
4329          * this means that the CSR Clock Range selection cannot be
4330          * changed at run-time and it is fixed. Viceversa the driver'll try to
4331          * set the MDC clock dynamically according to the csr actual
4332          * clock input.
4333          */
4334         if (!priv->plat->clk_csr)
4335                 stmmac_clk_csr_set(priv);
4336         else
4337                 priv->clk_csr = priv->plat->clk_csr;
4338
4339         stmmac_check_pcs_mode(priv);
4340
4341         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4342             priv->hw->pcs != STMMAC_PCS_TBI &&
4343             priv->hw->pcs != STMMAC_PCS_RTBI) {
4344                 /* MDIO bus Registration */
4345                 ret = stmmac_mdio_register(ndev);
4346                 if (ret < 0) {
4347                         dev_err(priv->device,
4348                                 "%s: MDIO bus (id: %d) registration failed",
4349                                 __func__, priv->plat->bus_id);
4350                         goto error_mdio_register;
4351                 }
4352         }
4353
4354         ret = register_netdev(ndev);
4355         if (ret) {
4356                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4357                         __func__, ret);
4358                 goto error_netdev_register;
4359         }
4360
4361 #ifdef CONFIG_DEBUG_FS
4362         ret = stmmac_init_fs(ndev);
4363         if (ret < 0)
4364                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
4365                             __func__);
4366 #endif
4367
4368         return ret;
4369
4370 error_netdev_register:
4371         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4372             priv->hw->pcs != STMMAC_PCS_TBI &&
4373             priv->hw->pcs != STMMAC_PCS_RTBI)
4374                 stmmac_mdio_unregister(ndev);
4375 error_mdio_register:
4376         for (queue = 0; queue < maxq; queue++) {
4377                 struct stmmac_channel *ch = &priv->channel[queue];
4378
4379                 netif_napi_del(&ch->napi);
4380         }
4381 error_hw_init:
4382         destroy_workqueue(priv->wq);
4383 error_wq:
4384         free_netdev(ndev);
4385
4386         return ret;
4387 }
4388 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4389
4390 /**
4391  * stmmac_dvr_remove
4392  * @dev: device pointer
4393  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4394  * changes the link status, releases the DMA descriptor rings.
4395  */
4396 int stmmac_dvr_remove(struct device *dev)
4397 {
4398         struct net_device *ndev = dev_get_drvdata(dev);
4399         struct stmmac_priv *priv = netdev_priv(ndev);
4400
4401         netdev_info(priv->dev, "%s: removing driver", __func__);
4402
4403 #ifdef CONFIG_DEBUG_FS
4404         stmmac_exit_fs(ndev);
4405 #endif
4406         stmmac_stop_all_dma(priv);
4407
4408         stmmac_mac_set(priv, priv->ioaddr, false);
4409         netif_carrier_off(ndev);
4410         unregister_netdev(ndev);
4411         if (priv->plat->stmmac_rst)
4412                 reset_control_assert(priv->plat->stmmac_rst);
4413         clk_disable_unprepare(priv->plat->pclk);
4414         clk_disable_unprepare(priv->plat->stmmac_clk);
4415         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4416             priv->hw->pcs != STMMAC_PCS_TBI &&
4417             priv->hw->pcs != STMMAC_PCS_RTBI)
4418                 stmmac_mdio_unregister(ndev);
4419         destroy_workqueue(priv->wq);
4420         mutex_destroy(&priv->lock);
4421         free_netdev(ndev);
4422
4423         return 0;
4424 }
4425 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4426
4427 /**
4428  * stmmac_suspend - suspend callback
4429  * @dev: device pointer
4430  * Description: this is the function to suspend the device and it is called
4431  * by the platform driver to stop the network queue, release the resources,
4432  * program the PMT register (for WoL), clean and release driver resources.
4433  */
4434 int stmmac_suspend(struct device *dev)
4435 {
4436         struct net_device *ndev = dev_get_drvdata(dev);
4437         struct stmmac_priv *priv = netdev_priv(ndev);
4438
4439         if (!ndev || !netif_running(ndev))
4440                 return 0;
4441
4442         if (ndev->phydev)
4443                 phy_stop(ndev->phydev);
4444
4445         mutex_lock(&priv->lock);
4446
4447         netif_device_detach(ndev);
4448         stmmac_stop_all_queues(priv);
4449
4450         stmmac_disable_all_queues(priv);
4451
4452         /* Stop TX/RX DMA */
4453         stmmac_stop_all_dma(priv);
4454
4455         /* Enable Power down mode by programming the PMT regs */
4456         if (device_may_wakeup(priv->device)) {
4457                 stmmac_pmt(priv, priv->hw, priv->wolopts);
4458                 priv->irq_wake = 1;
4459         } else {
4460                 stmmac_mac_set(priv, priv->ioaddr, false);
4461                 pinctrl_pm_select_sleep_state(priv->device);
4462                 /* Disable clock in case of PWM is off */
4463                 clk_disable(priv->plat->pclk);
4464                 clk_disable(priv->plat->stmmac_clk);
4465         }
4466         mutex_unlock(&priv->lock);
4467
4468         priv->oldlink = false;
4469         priv->speed = SPEED_UNKNOWN;
4470         priv->oldduplex = DUPLEX_UNKNOWN;
4471         return 0;
4472 }
4473 EXPORT_SYMBOL_GPL(stmmac_suspend);
4474
4475 /**
4476  * stmmac_reset_queues_param - reset queue parameters
4477  * @dev: device pointer
4478  */
4479 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4480 {
4481         u32 rx_cnt = priv->plat->rx_queues_to_use;
4482         u32 tx_cnt = priv->plat->tx_queues_to_use;
4483         u32 queue;
4484
4485         for (queue = 0; queue < rx_cnt; queue++) {
4486                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4487
4488                 rx_q->cur_rx = 0;
4489                 rx_q->dirty_rx = 0;
4490         }
4491
4492         for (queue = 0; queue < tx_cnt; queue++) {
4493                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4494
4495                 tx_q->cur_tx = 0;
4496                 tx_q->dirty_tx = 0;
4497                 tx_q->mss = 0;
4498         }
4499 }
4500
4501 /**
4502  * stmmac_resume - resume callback
4503  * @dev: device pointer
4504  * Description: when resume this function is invoked to setup the DMA and CORE
4505  * in a usable state.
4506  */
4507 int stmmac_resume(struct device *dev)
4508 {
4509         struct net_device *ndev = dev_get_drvdata(dev);
4510         struct stmmac_priv *priv = netdev_priv(ndev);
4511
4512         if (!netif_running(ndev))
4513                 return 0;
4514
4515         /* Power Down bit, into the PM register, is cleared
4516          * automatically as soon as a magic packet or a Wake-up frame
4517          * is received. Anyway, it's better to manually clear
4518          * this bit because it can generate problems while resuming
4519          * from another devices (e.g. serial console).
4520          */
4521         if (device_may_wakeup(priv->device)) {
4522                 mutex_lock(&priv->lock);
4523                 stmmac_pmt(priv, priv->hw, 0);
4524                 mutex_unlock(&priv->lock);
4525                 priv->irq_wake = 0;
4526         } else {
4527                 pinctrl_pm_select_default_state(priv->device);
4528                 /* enable the clk previously disabled */
4529                 clk_enable(priv->plat->stmmac_clk);
4530                 clk_enable(priv->plat->pclk);
4531                 /* reset the phy so that it's ready */
4532                 if (priv->mii)
4533                         stmmac_mdio_reset(priv->mii);
4534         }
4535
4536         netif_device_attach(ndev);
4537
4538         mutex_lock(&priv->lock);
4539
4540         stmmac_reset_queues_param(priv);
4541
4542         stmmac_clear_descriptors(priv);
4543
4544         stmmac_hw_setup(ndev, false);
4545         stmmac_init_tx_coalesce(priv);
4546         stmmac_set_rx_mode(ndev);
4547
4548         stmmac_enable_all_queues(priv);
4549
4550         stmmac_start_all_queues(priv);
4551
4552         mutex_unlock(&priv->lock);
4553
4554         if (ndev->phydev)
4555                 phy_start(ndev->phydev);
4556
4557         return 0;
4558 }
4559 EXPORT_SYMBOL_GPL(stmmac_resume);
4560
4561 #ifndef MODULE
4562 static int __init stmmac_cmdline_opt(char *str)
4563 {
4564         char *opt;
4565
4566         if (!str || !*str)
4567                 return -EINVAL;
4568         while ((opt = strsep(&str, ",")) != NULL) {
4569                 if (!strncmp(opt, "debug:", 6)) {
4570                         if (kstrtoint(opt + 6, 0, &debug))
4571                                 goto err;
4572                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4573                         if (kstrtoint(opt + 8, 0, &phyaddr))
4574                                 goto err;
4575                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4576                         if (kstrtoint(opt + 7, 0, &buf_sz))
4577                                 goto err;
4578                 } else if (!strncmp(opt, "tc:", 3)) {
4579                         if (kstrtoint(opt + 3, 0, &tc))
4580                                 goto err;
4581                 } else if (!strncmp(opt, "watchdog:", 9)) {
4582                         if (kstrtoint(opt + 9, 0, &watchdog))
4583                                 goto err;
4584                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4585                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4586                                 goto err;
4587                 } else if (!strncmp(opt, "pause:", 6)) {
4588                         if (kstrtoint(opt + 6, 0, &pause))
4589                                 goto err;
4590                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4591                         if (kstrtoint(opt + 10, 0, &eee_timer))
4592                                 goto err;
4593                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4594                         if (kstrtoint(opt + 11, 0, &chain_mode))
4595                                 goto err;
4596                 }
4597         }
4598         return 0;
4599
4600 err:
4601         pr_err("%s: ERROR broken module parameter conversion", __func__);
4602         return -EINVAL;
4603 }
4604
4605 __setup("stmmaceth=", stmmac_cmdline_opt);
4606 #endif /* MODULE */
4607
4608 static int __init stmmac_init(void)
4609 {
4610 #ifdef CONFIG_DEBUG_FS
4611         /* Create debugfs main directory if it doesn't exist yet */
4612         if (!stmmac_fs_dir) {
4613                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4614
4615                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
4616                         pr_err("ERROR %s, debugfs create directory failed\n",
4617                                STMMAC_RESOURCE_NAME);
4618
4619                         return -ENOMEM;
4620                 }
4621         }
4622 #endif
4623
4624         return 0;
4625 }
4626
4627 static void __exit stmmac_exit(void)
4628 {
4629 #ifdef CONFIG_DEBUG_FS
4630         debugfs_remove_recursive(stmmac_fs_dir);
4631 #endif
4632 }
4633
4634 module_init(stmmac_init)
4635 module_exit(stmmac_exit)
4636
4637 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
4638 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
4639 MODULE_LICENSE("GPL");