drivers/net/ethernet/broadcom/genet/bcmgenet.c

   1 /*
   2  * Broadcom GENET (Gigabit Ethernet) controller driver
   3  *
   4  * Copyright (c) 2014 Broadcom Corporation
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #define pr_fmt(fmt)                             "bcmgenet: " fmt
  12
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/sched.h>
  16 #include <linux/types.h>
  17 #include <linux/fcntl.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/string.h>
  20 #include <linux/if_ether.h>
  21 #include <linux/init.h>
  22 #include <linux/errno.h>
  23 #include <linux/delay.h>
  24 #include <linux/platform_device.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/pm.h>
  27 #include <linux/clk.h>
  28 #include <linux/of.h>
  29 #include <linux/of_address.h>
  30 #include <linux/of_irq.h>
  31 #include <linux/of_net.h>
  32 #include <linux/of_platform.h>
  33 #include <net/arp.h>
  34
  35 #include <linux/mii.h>
  36 #include <linux/ethtool.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/inetdevice.h>
  39 #include <linux/etherdevice.h>
  40 #include <linux/skbuff.h>
  41 #include <linux/in.h>
  42 #include <linux/ip.h>
  43 #include <linux/ipv6.h>
  44 #include <linux/phy.h>
  45 #include <linux/platform_data/bcmgenet.h>
  46
  47 #include <asm/unaligned.h>
  48
  49 #include "bcmgenet.h"
  50
  51 /* Maximum number of hardware queues, downsized if needed */
  52 #define GENET_MAX_MQ_CNT        4
  53
  54 /* Default highest priority queue for multi queue support */
  55 #define GENET_Q0_PRIORITY       0
  56
  57 #define GENET_Q16_RX_BD_CNT     \
  58         (TOTAL_DESC - priv->hw_params->rx_queues * priv->hw_params->rx_bds_per_q)
  59 #define GENET_Q16_TX_BD_CNT     \
  60         (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q)
  61
  62 #define RX_BUF_LENGTH           2048
  63 #define SKB_ALIGNMENT           32
  64
  65 /* Tx/Rx DMA register offset, skip 256 descriptors */
  66 #define WORDS_PER_BD(p)         (p->hw_params->words_per_bd)
  67 #define DMA_DESC_SIZE           (WORDS_PER_BD(priv) * sizeof(u32))
  68
  69 #define GENET_TDMA_REG_OFF      (priv->hw_params->tdma_offset + \
  70                                 TOTAL_DESC * DMA_DESC_SIZE)
  71
  72 #define GENET_RDMA_REG_OFF      (priv->hw_params->rdma_offset + \
  73                                 TOTAL_DESC * DMA_DESC_SIZE)
  74
  75 static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv,
  76                                              void __iomem *d, u32 value)
  77 {
  78         __raw_writel(value, d + DMA_DESC_LENGTH_STATUS);
  79 }
  80
  81 static inline u32 dmadesc_get_length_status(struct bcmgenet_priv *priv,
  82                                             void __iomem *d)
  83 {
  84         return __raw_readl(d + DMA_DESC_LENGTH_STATUS);
  85 }
  86
  87 static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
  88                                     void __iomem *d,
  89                                     dma_addr_t addr)
  90 {
  91         __raw_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);
  92
  93         /* Register writes to GISB bus can take couple hundred nanoseconds
  94          * and are done for each packet, save these expensive writes unless
  95          * the platform is explicitly configured for 64-bits/LPAE.
  96          */
  97 #ifdef CONFIG_PHYS_ADDR_T_64BIT
  98         if (priv->hw_params->flags & GENET_HAS_40BITS)
  99                 __raw_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
 100 #endif
 101 }
 102
 103 /* Combined address + length/status setter */
 104 static inline void dmadesc_set(struct bcmgenet_priv *priv,
 105                                void __iomem *d, dma_addr_t addr, u32 val)
 106 {
 107         dmadesc_set_length_status(priv, d, val);
 108         dmadesc_set_addr(priv, d, addr);
 109 }
 110
 111 static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv,
 112                                           void __iomem *d)
 113 {
 114         dma_addr_t addr;
 115
 116         addr = __raw_readl(d + DMA_DESC_ADDRESS_LO);
 117
 118         /* Register writes to GISB bus can take couple hundred nanoseconds
 119          * and are done for each packet, save these expensive writes unless
 120          * the platform is explicitly configured for 64-bits/LPAE.
 121          */
 122 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 123         if (priv->hw_params->flags & GENET_HAS_40BITS)
 124                 addr |= (u64)__raw_readl(d + DMA_DESC_ADDRESS_HI) << 32;
 125 #endif
 126         return addr;
 127 }
 128
 129 #define GENET_VER_FMT   "%1d.%1d EPHY: 0x%04x"
 130
 131 #define GENET_MSG_DEFAULT       (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
 132                                 NETIF_MSG_LINK)
 133
 134 static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv)
 135 {
 136         if (GENET_IS_V1(priv))
 137                 return bcmgenet_rbuf_readl(priv, RBUF_FLUSH_CTRL_V1);
 138         else
 139                 return bcmgenet_sys_readl(priv, SYS_RBUF_FLUSH_CTRL);
 140 }
 141
 142 static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
 143 {
 144         if (GENET_IS_V1(priv))
 145                 bcmgenet_rbuf_writel(priv, val, RBUF_FLUSH_CTRL_V1);
 146         else
 147                 bcmgenet_sys_writel(priv, val, SYS_RBUF_FLUSH_CTRL);
 148 }
 149
 150 /* These macros are defined to deal with register map change
 151  * between GENET1.1 and GENET2. Only those currently being used
 152  * by driver are defined.
 153  */
 154 static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv)
 155 {
 156         if (GENET_IS_V1(priv))
 157                 return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1);
 158         else
 159                 return __raw_readl(priv->base +
 160                                 priv->hw_params->tbuf_offset + TBUF_CTRL);
 161 }
 162
 163 static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
 164 {
 165         if (GENET_IS_V1(priv))
 166                 bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1);
 167         else
 168                 __raw_writel(val, priv->base +
 169                                 priv->hw_params->tbuf_offset + TBUF_CTRL);
 170 }
 171
 172 static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv)
 173 {
 174         if (GENET_IS_V1(priv))
 175                 return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1);
 176         else
 177                 return __raw_readl(priv->base +
 178                                 priv->hw_params->tbuf_offset + TBUF_BP_MC);
 179 }
 180
 181 static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
 182 {
 183         if (GENET_IS_V1(priv))
 184                 bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1);
 185         else
 186                 __raw_writel(val, priv->base +
 187                                 priv->hw_params->tbuf_offset + TBUF_BP_MC);
 188 }
 189
 190 /* RX/TX DMA register accessors */
 191 enum dma_reg {
 192         DMA_RING_CFG = 0,
 193         DMA_CTRL,
 194         DMA_STATUS,
 195         DMA_SCB_BURST_SIZE,
 196         DMA_ARB_CTRL,
 197         DMA_PRIORITY_0,
 198         DMA_PRIORITY_1,
 199         DMA_PRIORITY_2,
 200 };
 201
 202 static const u8 bcmgenet_dma_regs_v3plus[] = {
 203         [DMA_RING_CFG]          = 0x00,
 204         [DMA_CTRL]              = 0x04,
 205         [DMA_STATUS]            = 0x08,
 206         [DMA_SCB_BURST_SIZE]    = 0x0C,
 207         [DMA_ARB_CTRL]          = 0x2C,
 208         [DMA_PRIORITY_0]        = 0x30,
 209         [DMA_PRIORITY_1]        = 0x34,
 210         [DMA_PRIORITY_2]        = 0x38,
 211 };
 212
 213 static const u8 bcmgenet_dma_regs_v2[] = {
 214         [DMA_RING_CFG]          = 0x00,
 215         [DMA_CTRL]              = 0x04,
 216         [DMA_STATUS]            = 0x08,
 217         [DMA_SCB_BURST_SIZE]    = 0x0C,
 218         [DMA_ARB_CTRL]          = 0x30,
 219         [DMA_PRIORITY_0]        = 0x34,
 220         [DMA_PRIORITY_1]        = 0x38,
 221         [DMA_PRIORITY_2]        = 0x3C,
 222 };
 223
 224 static const u8 bcmgenet_dma_regs_v1[] = {
 225         [DMA_CTRL]              = 0x00,
 226         [DMA_STATUS]            = 0x04,
 227         [DMA_SCB_BURST_SIZE]    = 0x0C,
 228         [DMA_ARB_CTRL]          = 0x30,
 229         [DMA_PRIORITY_0]        = 0x34,
 230         [DMA_PRIORITY_1]        = 0x38,
 231         [DMA_PRIORITY_2]        = 0x3C,
 232 };
 233
 234 /* Set at runtime once bcmgenet version is known */
 235 static const u8 *bcmgenet_dma_regs;
 236
 237 static inline struct bcmgenet_priv *dev_to_priv(struct device *dev)
 238 {
 239         return netdev_priv(dev_get_drvdata(dev));
 240 }
 241
 242 static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv,
 243                                       enum dma_reg r)
 244 {
 245         return __raw_readl(priv->base + GENET_TDMA_REG_OFF +
 246                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 247 }
 248
 249 static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv,
 250                                         u32 val, enum dma_reg r)
 251 {
 252         __raw_writel(val, priv->base + GENET_TDMA_REG_OFF +
 253                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 254 }
 255
 256 static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv,
 257                                       enum dma_reg r)
 258 {
 259         return __raw_readl(priv->base + GENET_RDMA_REG_OFF +
 260                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 261 }
 262
 263 static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv,
 264                                         u32 val, enum dma_reg r)
 265 {
 266         __raw_writel(val, priv->base + GENET_RDMA_REG_OFF +
 267                         DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 268 }
 269
 270 /* RDMA/TDMA ring registers and accessors
 271  * we merge the common fields and just prefix with T/D the registers
 272  * having different meaning depending on the direction
 273  */
 274 enum dma_ring_reg {
 275         TDMA_READ_PTR = 0,
 276         RDMA_WRITE_PTR = TDMA_READ_PTR,
 277         TDMA_READ_PTR_HI,
 278         RDMA_WRITE_PTR_HI = TDMA_READ_PTR_HI,
 279         TDMA_CONS_INDEX,
 280         RDMA_PROD_INDEX = TDMA_CONS_INDEX,
 281         TDMA_PROD_INDEX,
 282         RDMA_CONS_INDEX = TDMA_PROD_INDEX,
 283         DMA_RING_BUF_SIZE,
 284         DMA_START_ADDR,
 285         DMA_START_ADDR_HI,
 286         DMA_END_ADDR,
 287         DMA_END_ADDR_HI,
 288         DMA_MBUF_DONE_THRESH,
 289         TDMA_FLOW_PERIOD,
 290         RDMA_XON_XOFF_THRESH = TDMA_FLOW_PERIOD,
 291         TDMA_WRITE_PTR,
 292         RDMA_READ_PTR = TDMA_WRITE_PTR,
 293         TDMA_WRITE_PTR_HI,
 294         RDMA_READ_PTR_HI = TDMA_WRITE_PTR_HI
 295 };
 296
 297 /* GENET v4 supports 40-bits pointer addressing
 298  * for obvious reasons the LO and HI word parts
 299  * are contiguous, but this offsets the other
 300  * registers.
 301  */
 302 static const u8 genet_dma_ring_regs_v4[] = {
 303         [TDMA_READ_PTR]                 = 0x00,
 304         [TDMA_READ_PTR_HI]              = 0x04,
 305         [TDMA_CONS_INDEX]               = 0x08,
 306         [TDMA_PROD_INDEX]               = 0x0C,
 307         [DMA_RING_BUF_SIZE]             = 0x10,
 308         [DMA_START_ADDR]                = 0x14,
 309         [DMA_START_ADDR_HI]             = 0x18,
 310         [DMA_END_ADDR]                  = 0x1C,
 311         [DMA_END_ADDR_HI]               = 0x20,
 312         [DMA_MBUF_DONE_THRESH]          = 0x24,
 313         [TDMA_FLOW_PERIOD]              = 0x28,
 314         [TDMA_WRITE_PTR]                = 0x2C,
 315         [TDMA_WRITE_PTR_HI]             = 0x30,
 316 };
 317
 318 static const u8 genet_dma_ring_regs_v123[] = {
 319         [TDMA_READ_PTR]                 = 0x00,
 320         [TDMA_CONS_INDEX]               = 0x04,
 321         [TDMA_PROD_INDEX]               = 0x08,
 322         [DMA_RING_BUF_SIZE]             = 0x0C,
 323         [DMA_START_ADDR]                = 0x10,
 324         [DMA_END_ADDR]                  = 0x14,
 325         [DMA_MBUF_DONE_THRESH]          = 0x18,
 326         [TDMA_FLOW_PERIOD]              = 0x1C,
 327         [TDMA_WRITE_PTR]                = 0x20,
 328 };
 329
 330 /* Set at runtime once GENET version is known */
 331 static const u8 *genet_dma_ring_regs;
 332
 333 static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv,
 334                                            unsigned int ring,
 335                                            enum dma_ring_reg r)
 336 {
 337         return __raw_readl(priv->base + GENET_TDMA_REG_OFF +
 338                         (DMA_RING_SIZE * ring) +
 339                         genet_dma_ring_regs[r]);
 340 }
 341
 342 static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv,
 343                                              unsigned int ring, u32 val,
 344                                              enum dma_ring_reg r)
 345 {
 346         __raw_writel(val, priv->base + GENET_TDMA_REG_OFF +
 347                         (DMA_RING_SIZE * ring) +
 348                         genet_dma_ring_regs[r]);
 349 }
 350
 351 static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv,
 352                                            unsigned int ring,
 353                                            enum dma_ring_reg r)
 354 {
 355         return __raw_readl(priv->base + GENET_RDMA_REG_OFF +
 356                         (DMA_RING_SIZE * ring) +
 357                         genet_dma_ring_regs[r]);
 358 }
 359
 360 static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
 361                                              unsigned int ring, u32 val,
 362                                              enum dma_ring_reg r)
 363 {
 364         __raw_writel(val, priv->base + GENET_RDMA_REG_OFF +
 365                         (DMA_RING_SIZE * ring) +
 366                         genet_dma_ring_regs[r]);
 367 }
 368
 369 static int bcmgenet_get_settings(struct net_device *dev,
 370                                  struct ethtool_cmd *cmd)
 371 {
 372         struct bcmgenet_priv *priv = netdev_priv(dev);
 373
 374         if (!netif_running(dev))
 375                 return -EINVAL;
 376
 377         if (!priv->phydev)
 378                 return -ENODEV;
 379
 380         return phy_ethtool_gset(priv->phydev, cmd);
 381 }
 382
 383 static int bcmgenet_set_settings(struct net_device *dev,
 384                                  struct ethtool_cmd *cmd)
 385 {
 386         struct bcmgenet_priv *priv = netdev_priv(dev);
 387
 388         if (!netif_running(dev))
 389                 return -EINVAL;
 390
 391         if (!priv->phydev)
 392                 return -ENODEV;
 393
 394         return phy_ethtool_sset(priv->phydev, cmd);
 395 }
 396
 397 static int bcmgenet_set_rx_csum(struct net_device *dev,
 398                                 netdev_features_t wanted)
 399 {
 400         struct bcmgenet_priv *priv = netdev_priv(dev);
 401         u32 rbuf_chk_ctrl;
 402         bool rx_csum_en;
 403
 404         rx_csum_en = !!(wanted & NETIF_F_RXCSUM);
 405
 406         rbuf_chk_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CHK_CTRL);
 407
 408         /* enable rx checksumming */
 409         if (rx_csum_en)
 410                 rbuf_chk_ctrl |= RBUF_RXCHK_EN;
 411         else
 412                 rbuf_chk_ctrl &= ~RBUF_RXCHK_EN;
 413         priv->desc_rxchk_en = rx_csum_en;
 414
 415         /* If UniMAC forwards CRC, we need to skip over it to get
 416          * a valid CHK bit to be set in the per-packet status word
 417         */
 418         if (rx_csum_en && priv->crc_fwd_en)
 419                 rbuf_chk_ctrl |= RBUF_SKIP_FCS;
 420         else
 421                 rbuf_chk_ctrl &= ~RBUF_SKIP_FCS;
 422
 423         bcmgenet_rbuf_writel(priv, rbuf_chk_ctrl, RBUF_CHK_CTRL);
 424
 425         return 0;
 426 }
 427
 428 static int bcmgenet_set_tx_csum(struct net_device *dev,
 429                                 netdev_features_t wanted)
 430 {
 431         struct bcmgenet_priv *priv = netdev_priv(dev);
 432         bool desc_64b_en;
 433         u32 tbuf_ctrl, rbuf_ctrl;
 434
 435         tbuf_ctrl = bcmgenet_tbuf_ctrl_get(priv);
 436         rbuf_ctrl = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
 437
 438         desc_64b_en = !!(wanted & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));
 439
 440         /* enable 64 bytes descriptor in both directions (RBUF and TBUF) */
 441         if (desc_64b_en) {
 442                 tbuf_ctrl |= RBUF_64B_EN;
 443                 rbuf_ctrl |= RBUF_64B_EN;
 444         } else {
 445                 tbuf_ctrl &= ~RBUF_64B_EN;
 446                 rbuf_ctrl &= ~RBUF_64B_EN;
 447         }
 448         priv->desc_64b_en = desc_64b_en;
 449
 450         bcmgenet_tbuf_ctrl_set(priv, tbuf_ctrl);
 451         bcmgenet_rbuf_writel(priv, rbuf_ctrl, RBUF_CTRL);
 452
 453         return 0;
 454 }
 455
 456 static int bcmgenet_set_features(struct net_device *dev,
 457                                  netdev_features_t features)
 458 {
 459         netdev_features_t changed = features ^ dev->features;
 460         netdev_features_t wanted = dev->wanted_features;
 461         int ret = 0;
 462
 463         if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))
 464                 ret = bcmgenet_set_tx_csum(dev, wanted);
 465         if (changed & (NETIF_F_RXCSUM))
 466                 ret = bcmgenet_set_rx_csum(dev, wanted);
 467
 468         return ret;
 469 }
 470
 471 static u32 bcmgenet_get_msglevel(struct net_device *dev)
 472 {
 473         struct bcmgenet_priv *priv = netdev_priv(dev);
 474
 475         return priv->msg_enable;
 476 }
 477
 478 static void bcmgenet_set_msglevel(struct net_device *dev, u32 level)
 479 {
 480         struct bcmgenet_priv *priv = netdev_priv(dev);
 481
 482         priv->msg_enable = level;
 483 }
 484
 485 /* standard ethtool support functions. */
 486 enum bcmgenet_stat_type {
 487         BCMGENET_STAT_NETDEV = -1,
 488         BCMGENET_STAT_MIB_RX,
 489         BCMGENET_STAT_MIB_TX,
 490         BCMGENET_STAT_RUNT,
 491         BCMGENET_STAT_MISC,
 492         BCMGENET_STAT_SOFT,
 493 };
 494
 495 struct bcmgenet_stats {
 496         char stat_string[ETH_GSTRING_LEN];
 497         int stat_sizeof;
 498         int stat_offset;
 499         enum bcmgenet_stat_type type;
 500         /* reg offset from UMAC base for misc counters */
 501         u16 reg_offset;
 502 };
 503
 504 #define STAT_NETDEV(m) { \
 505         .stat_string = __stringify(m), \
 506         .stat_sizeof = sizeof(((struct net_device_stats *)0)->m), \
 507         .stat_offset = offsetof(struct net_device_stats, m), \
 508         .type = BCMGENET_STAT_NETDEV, \
 509 }
 510
 511 #define STAT_GENET_MIB(str, m, _type) { \
 512         .stat_string = str, \
 513         .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
 514         .stat_offset = offsetof(struct bcmgenet_priv, m), \
 515         .type = _type, \
 516 }
 517
 518 #define STAT_GENET_MIB_RX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_RX)
 519 #define STAT_GENET_MIB_TX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_TX)
 520 #define STAT_GENET_RUNT(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_RUNT)
 521 #define STAT_GENET_SOFT_MIB(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_SOFT)
 522
 523 #define STAT_GENET_MISC(str, m, offset) { \
 524         .stat_string = str, \
 525         .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
 526         .stat_offset = offsetof(struct bcmgenet_priv, m), \
 527         .type = BCMGENET_STAT_MISC, \
 528         .reg_offset = offset, \
 529 }
 530
 531
 532 /* There is a 0xC gap between the end of RX and beginning of TX stats and then
 533  * between the end of TX stats and the beginning of the RX RUNT
 534  */
 535 #define BCMGENET_STAT_OFFSET    0xc
 536
 537 /* Hardware counters must be kept in sync because the order/offset
 538  * is important here (order in structure declaration = order in hardware)
 539  */
 540 static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = {
 541         /* general stats */
 542         STAT_NETDEV(rx_packets),
 543         STAT_NETDEV(tx_packets),
 544         STAT_NETDEV(rx_bytes),
 545         STAT_NETDEV(tx_bytes),
 546         STAT_NETDEV(rx_errors),
 547         STAT_NETDEV(tx_errors),
 548         STAT_NETDEV(rx_dropped),
 549         STAT_NETDEV(tx_dropped),
 550         STAT_NETDEV(multicast),
 551         /* UniMAC RSV counters */
 552         STAT_GENET_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
 553         STAT_GENET_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
 554         STAT_GENET_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
 555         STAT_GENET_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
 556         STAT_GENET_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
 557         STAT_GENET_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
 558         STAT_GENET_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
 559         STAT_GENET_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
 560         STAT_GENET_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
 561         STAT_GENET_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
 562         STAT_GENET_MIB_RX("rx_pkts", mib.rx.pkt),
 563         STAT_GENET_MIB_RX("rx_bytes", mib.rx.bytes),
 564         STAT_GENET_MIB_RX("rx_multicast", mib.rx.mca),
 565         STAT_GENET_MIB_RX("rx_broadcast", mib.rx.bca),
 566         STAT_GENET_MIB_RX("rx_fcs", mib.rx.fcs),
 567         STAT_GENET_MIB_RX("rx_control", mib.rx.cf),
 568         STAT_GENET_MIB_RX("rx_pause", mib.rx.pf),
 569         STAT_GENET_MIB_RX("rx_unknown", mib.rx.uo),
 570         STAT_GENET_MIB_RX("rx_align", mib.rx.aln),
 571         STAT_GENET_MIB_RX("rx_outrange", mib.rx.flr),
 572         STAT_GENET_MIB_RX("rx_code", mib.rx.cde),
 573         STAT_GENET_MIB_RX("rx_carrier", mib.rx.fcr),
 574         STAT_GENET_MIB_RX("rx_oversize", mib.rx.ovr),
 575         STAT_GENET_MIB_RX("rx_jabber", mib.rx.jbr),
 576         STAT_GENET_MIB_RX("rx_mtu_err", mib.rx.mtue),
 577         STAT_GENET_MIB_RX("rx_good_pkts", mib.rx.pok),
 578         STAT_GENET_MIB_RX("rx_unicast", mib.rx.uc),
 579         STAT_GENET_MIB_RX("rx_ppp", mib.rx.ppp),
 580         STAT_GENET_MIB_RX("rx_crc", mib.rx.rcrc),
 581         /* UniMAC TSV counters */
 582         STAT_GENET_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
 583         STAT_GENET_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
 584         STAT_GENET_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
 585         STAT_GENET_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
 586         STAT_GENET_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
 587         STAT_GENET_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
 588         STAT_GENET_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
 589         STAT_GENET_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
 590         STAT_GENET_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
 591         STAT_GENET_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
 592         STAT_GENET_MIB_TX("tx_pkts", mib.tx.pkts),
 593         STAT_GENET_MIB_TX("tx_multicast", mib.tx.mca),
 594         STAT_GENET_MIB_TX("tx_broadcast", mib.tx.bca),
 595         STAT_GENET_MIB_TX("tx_pause", mib.tx.pf),
 596         STAT_GENET_MIB_TX("tx_control", mib.tx.cf),
 597         STAT_GENET_MIB_TX("tx_fcs_err", mib.tx.fcs),
 598         STAT_GENET_MIB_TX("tx_oversize", mib.tx.ovr),
 599         STAT_GENET_MIB_TX("tx_defer", mib.tx.drf),
 600         STAT_GENET_MIB_TX("tx_excess_defer", mib.tx.edf),
 601         STAT_GENET_MIB_TX("tx_single_col", mib.tx.scl),
 602         STAT_GENET_MIB_TX("tx_multi_col", mib.tx.mcl),
 603         STAT_GENET_MIB_TX("tx_late_col", mib.tx.lcl),
 604         STAT_GENET_MIB_TX("tx_excess_col", mib.tx.ecl),
 605         STAT_GENET_MIB_TX("tx_frags", mib.tx.frg),
 606         STAT_GENET_MIB_TX("tx_total_col", mib.tx.ncl),
 607         STAT_GENET_MIB_TX("tx_jabber", mib.tx.jbr),
 608         STAT_GENET_MIB_TX("tx_bytes", mib.tx.bytes),
 609         STAT_GENET_MIB_TX("tx_good_pkts", mib.tx.pok),
 610         STAT_GENET_MIB_TX("tx_unicast", mib.tx.uc),
 611         /* UniMAC RUNT counters */
 612         STAT_GENET_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
 613         STAT_GENET_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
 614         STAT_GENET_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
 615         STAT_GENET_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
 616         /* Misc UniMAC counters */
 617         STAT_GENET_MISC("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt,
 618                         UMAC_RBUF_OVFL_CNT),
 619         STAT_GENET_MISC("rbuf_err_cnt", mib.rbuf_err_cnt, UMAC_RBUF_ERR_CNT),
 620         STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT),
 621         STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
 622         STAT_GENET_SOFT_MIB("rx_dma_failed", mib.rx_dma_failed),
 623         STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed),
 624 };
 625
 626 #define BCMGENET_STATS_LEN      ARRAY_SIZE(bcmgenet_gstrings_stats)
 627
 628 static void bcmgenet_get_drvinfo(struct net_device *dev,
 629                                  struct ethtool_drvinfo *info)
 630 {
 631         strlcpy(info->driver, "bcmgenet", sizeof(info->driver));
 632         strlcpy(info->version, "v2.0", sizeof(info->version));
 633         info->n_stats = BCMGENET_STATS_LEN;
 634 }
 635
 636 static int bcmgenet_get_sset_count(struct net_device *dev, int string_set)
 637 {
 638         switch (string_set) {
 639         case ETH_SS_STATS:
 640                 return BCMGENET_STATS_LEN;
 641         default:
 642                 return -EOPNOTSUPP;
 643         }
 644 }
 645
 646 static void bcmgenet_get_strings(struct net_device *dev, u32 stringset,
 647                                  u8 *data)
 648 {
 649         int i;
 650
 651         switch (stringset) {
 652         case ETH_SS_STATS:
 653                 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 654                         memcpy(data + i * ETH_GSTRING_LEN,
 655                                bcmgenet_gstrings_stats[i].stat_string,
 656                                ETH_GSTRING_LEN);
 657                 }
 658                 break;
 659         }
 660 }
 661
 662 static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv)
 663 {
 664         int i, j = 0;
 665
 666         for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 667                 const struct bcmgenet_stats *s;
 668                 u8 offset = 0;
 669                 u32 val = 0;
 670                 char *p;
 671
 672                 s = &bcmgenet_gstrings_stats[i];
 673                 switch (s->type) {
 674                 case BCMGENET_STAT_NETDEV:
 675                 case BCMGENET_STAT_SOFT:
 676                         continue;
 677                 case BCMGENET_STAT_MIB_RX:
 678                 case BCMGENET_STAT_MIB_TX:
 679                 case BCMGENET_STAT_RUNT:
 680                         if (s->type != BCMGENET_STAT_MIB_RX)
 681                                 offset = BCMGENET_STAT_OFFSET;
 682                         val = bcmgenet_umac_readl(priv,
 683                                                   UMAC_MIB_START + j + offset);
 684                         break;
 685                 case BCMGENET_STAT_MISC:
 686                         val = bcmgenet_umac_readl(priv, s->reg_offset);
 687                         /* clear if overflowed */
 688                         if (val == ~0)
 689                                 bcmgenet_umac_writel(priv, 0, s->reg_offset);
 690                         break;
 691                 }
 692
 693                 j += s->stat_sizeof;
 694                 p = (char *)priv + s->stat_offset;
 695                 *(u32 *)p = val;
 696         }
 697 }
 698
 699 static void bcmgenet_get_ethtool_stats(struct net_device *dev,
 700                                        struct ethtool_stats *stats,
 701                                        u64 *data)
 702 {
 703         struct bcmgenet_priv *priv = netdev_priv(dev);
 704         int i;
 705
 706         if (netif_running(dev))
 707                 bcmgenet_update_mib_counters(priv);
 708
 709         for (i = 0; i < BCMGENET_STATS_LEN; i++) {
 710                 const struct bcmgenet_stats *s;
 711                 char *p;
 712
 713                 s = &bcmgenet_gstrings_stats[i];
 714                 if (s->type == BCMGENET_STAT_NETDEV)
 715                         p = (char *)&dev->stats;
 716                 else
 717                         p = (char *)priv;
 718                 p += s->stat_offset;
 719                 data[i] = *(u32 *)p;
 720         }
 721 }
 722
 723 static void bcmgenet_eee_enable_set(struct net_device *dev, bool enable)
 724 {
 725         struct bcmgenet_priv *priv = netdev_priv(dev);
 726         u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL;
 727         u32 reg;
 728
 729         if (enable && !priv->clk_eee_enabled) {
 730                 clk_prepare_enable(priv->clk_eee);
 731                 priv->clk_eee_enabled = true;
 732         }
 733
 734         reg = bcmgenet_umac_readl(priv, UMAC_EEE_CTRL);
 735         if (enable)
 736                 reg |= EEE_EN;
 737         else
 738                 reg &= ~EEE_EN;
 739         bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL);
 740
 741         /* Enable EEE and switch to a 27Mhz clock automatically */
 742         reg = __raw_readl(priv->base + off);
 743         if (enable)
 744                 reg |= TBUF_EEE_EN | TBUF_PM_EN;
 745         else
 746                 reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
 747         __raw_writel(reg, priv->base + off);
 748
 749         /* Do the same for thing for RBUF */
 750         reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
 751         if (enable)
 752                 reg |= RBUF_EEE_EN | RBUF_PM_EN;
 753         else
 754                 reg &= ~(RBUF_EEE_EN | RBUF_PM_EN);
 755         bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);
 756
 757         if (!enable && priv->clk_eee_enabled) {
 758                 clk_disable_unprepare(priv->clk_eee);
 759                 priv->clk_eee_enabled = false;
 760         }
 761
 762         priv->eee.eee_enabled = enable;
 763         priv->eee.eee_active = enable;
 764 }
 765
 766 static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e)
 767 {
 768         struct bcmgenet_priv *priv = netdev_priv(dev);
 769         struct ethtool_eee *p = &priv->eee;
 770
 771         if (GENET_IS_V1(priv))
 772                 return -EOPNOTSUPP;
 773
 774         e->eee_enabled = p->eee_enabled;
 775         e->eee_active = p->eee_active;
 776         e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER);
 777
 778         return phy_ethtool_get_eee(priv->phydev, e);
 779 }
 780
 781 static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e)
 782 {
 783         struct bcmgenet_priv *priv = netdev_priv(dev);
 784         struct ethtool_eee *p = &priv->eee;
 785         int ret = 0;
 786
 787         if (GENET_IS_V1(priv))
 788                 return -EOPNOTSUPP;
 789
 790         p->eee_enabled = e->eee_enabled;
 791
 792         if (!p->eee_enabled) {
 793                 bcmgenet_eee_enable_set(dev, false);
 794         } else {
 795                 ret = phy_init_eee(priv->phydev, 0);
 796                 if (ret) {
 797                         netif_err(priv, hw, dev, "EEE initialization failed\n");
 798                         return ret;
 799                 }
 800
 801                 bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER);
 802                 bcmgenet_eee_enable_set(dev, true);
 803         }
 804
 805         return phy_ethtool_set_eee(priv->phydev, e);
 806 }
 807
 808 static int bcmgenet_nway_reset(struct net_device *dev)
 809 {
 810         struct bcmgenet_priv *priv = netdev_priv(dev);
 811
 812         return genphy_restart_aneg(priv->phydev);
 813 }
 814
 815 /* standard ethtool support functions. */
 816 static struct ethtool_ops bcmgenet_ethtool_ops = {
 817         .get_strings            = bcmgenet_get_strings,
 818         .get_sset_count         = bcmgenet_get_sset_count,
 819         .get_ethtool_stats      = bcmgenet_get_ethtool_stats,
 820         .get_settings           = bcmgenet_get_settings,
 821         .set_settings           = bcmgenet_set_settings,
 822         .get_drvinfo            = bcmgenet_get_drvinfo,
 823         .get_link               = ethtool_op_get_link,
 824         .get_msglevel           = bcmgenet_get_msglevel,
 825         .set_msglevel           = bcmgenet_set_msglevel,
 826         .get_wol                = bcmgenet_get_wol,
 827         .set_wol                = bcmgenet_set_wol,
 828         .get_eee                = bcmgenet_get_eee,
 829         .set_eee                = bcmgenet_set_eee,
 830         .nway_reset             = bcmgenet_nway_reset,
 831 };
 832
 833 /* Power down the unimac, based on mode. */
 834 static void bcmgenet_power_down(struct bcmgenet_priv *priv,
 835                                 enum bcmgenet_power_mode mode)
 836 {
 837         u32 reg;
 838
 839         switch (mode) {
 840         case GENET_POWER_CABLE_SENSE:
 841                 phy_detach(priv->phydev);
 842                 break;
 843
 844         case GENET_POWER_WOL_MAGIC:
 845                 bcmgenet_wol_power_down_cfg(priv, mode);
 846                 break;
 847
 848         case GENET_POWER_PASSIVE:
 849                 /* Power down LED */
 850                 if (priv->hw_params->flags & GENET_HAS_EXT) {
 851                         reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
 852                         reg |= (EXT_PWR_DOWN_PHY |
 853                                 EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS);
 854                         bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
 855                 }
 856                 break;
 857         default:
 858                 break;
 859         }
 860 }
 861
 862 static void bcmgenet_power_up(struct bcmgenet_priv *priv,
 863                               enum bcmgenet_power_mode mode)
 864 {
 865         u32 reg;
 866
 867         if (!(priv->hw_params->flags & GENET_HAS_EXT))
 868                 return;
 869
 870         reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
 871
 872         switch (mode) {
 873         case GENET_POWER_PASSIVE:
 874                 reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_PHY |
 875                                 EXT_PWR_DOWN_BIAS);
 876                 /* fallthrough */
 877         case GENET_POWER_CABLE_SENSE:
 878                 /* enable APD */
 879                 reg |= EXT_PWR_DN_EN_LD;
 880                 break;
 881         case GENET_POWER_WOL_MAGIC:
 882                 bcmgenet_wol_power_up_cfg(priv, mode);
 883                 return;
 884         default:
 885                 break;
 886         }
 887
 888         bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
 889
 890         if (mode == GENET_POWER_PASSIVE)
 891                 bcmgenet_mii_reset(priv->dev);
 892 }
 893
 894 /* ioctl handle special commands that are not present in ethtool. */
 895 static int bcmgenet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 896 {
 897         struct bcmgenet_priv *priv = netdev_priv(dev);
 898         int val = 0;
 899
 900         if (!netif_running(dev))
 901                 return -EINVAL;
 902
 903         switch (cmd) {
 904         case SIOCGMIIPHY:
 905         case SIOCGMIIREG:
 906         case SIOCSMIIREG:
 907                 if (!priv->phydev)
 908                         val = -ENODEV;
 909                 else
 910                         val = phy_mii_ioctl(priv->phydev, rq, cmd);
 911                 break;
 912
 913         default:
 914                 val = -EINVAL;
 915                 break;
 916         }
 917
 918         return val;
 919 }
 920
 921 static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv,
 922                                          struct bcmgenet_tx_ring *ring)
 923 {
 924         struct enet_cb *tx_cb_ptr;
 925
 926         tx_cb_ptr = ring->cbs;
 927         tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
 928
 929         /* Advancing local write pointer */
 930         if (ring->write_ptr == ring->end_ptr)
 931                 ring->write_ptr = ring->cb_ptr;
 932         else
 933                 ring->write_ptr++;
 934
 935         return tx_cb_ptr;
 936 }
 937
 938 /* Simple helper to free a control block's resources */
 939 static void bcmgenet_free_cb(struct enet_cb *cb)
 940 {
 941         dev_kfree_skb_any(cb->skb);
 942         cb->skb = NULL;
 943         dma_unmap_addr_set(cb, dma_addr, 0);
 944 }
 945
 946 static inline void bcmgenet_tx_ring16_int_disable(struct bcmgenet_priv *priv,
 947                                                   struct bcmgenet_tx_ring *ring)
 948 {
 949         bcmgenet_intrl2_0_writel(priv,
 950                                  UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE,
 951                                  INTRL2_CPU_MASK_SET);
 952 }
 953
 954 static inline void bcmgenet_tx_ring16_int_enable(struct bcmgenet_priv *priv,
 955                                                  struct bcmgenet_tx_ring *ring)
 956 {
 957         bcmgenet_intrl2_0_writel(priv,
 958                                  UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE,
 959                                  INTRL2_CPU_MASK_CLEAR);
 960 }
 961
 962 static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_priv *priv,
 963                                                struct bcmgenet_tx_ring *ring)
 964 {
 965         bcmgenet_intrl2_1_writel(priv, (1 << ring->index),
 966                                  INTRL2_CPU_MASK_CLEAR);
 967         priv->int1_mask &= ~(1 << ring->index);
 968 }
 969
 970 static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_priv *priv,
 971                                                 struct bcmgenet_tx_ring *ring)
 972 {
 973         bcmgenet_intrl2_1_writel(priv, (1 << ring->index),
 974                                  INTRL2_CPU_MASK_SET);
 975         priv->int1_mask |= (1 << ring->index);
 976 }
 977
 978 /* Unlocked version of the reclaim routine */
 979 static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev,
 980                                           struct bcmgenet_tx_ring *ring)
 981 {
 982         struct bcmgenet_priv *priv = netdev_priv(dev);
 983         struct enet_cb *tx_cb_ptr;
 984         struct netdev_queue *txq;
 985         unsigned int pkts_compl = 0;
 986         unsigned int c_index;
 987         unsigned int txbds_ready;
 988         unsigned int txbds_processed = 0;
 989
 990         /* Compute how many buffers are transmitted since last xmit call */
 991         c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);
 992         c_index &= DMA_C_INDEX_MASK;
 993
 994         if (likely(c_index >= ring->c_index))
 995                 txbds_ready = c_index - ring->c_index;
 996         else
 997                 txbds_ready = (DMA_C_INDEX_MASK + 1) - ring->c_index + c_index;
 998
 999         netif_dbg(priv, tx_done, dev,
1000                   "%s ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
1001                   __func__, ring->index, ring->c_index, c_index, txbds_ready);
1002
1003         /* Reclaim transmitted buffers */
1004         while (txbds_processed < txbds_ready) {
1005                 tx_cb_ptr = &priv->tx_cbs[ring->clean_ptr];
1006                 if (tx_cb_ptr->skb) {
1007                         pkts_compl++;
1008                         dev->stats.tx_packets++;
1009                         dev->stats.tx_bytes += tx_cb_ptr->skb->len;
1010                         dma_unmap_single(&dev->dev,
1011                                          dma_unmap_addr(tx_cb_ptr, dma_addr),
1012                                          tx_cb_ptr->skb->len,
1013                                          DMA_TO_DEVICE);
1014                         bcmgenet_free_cb(tx_cb_ptr);
1015                 } else if (dma_unmap_addr(tx_cb_ptr, dma_addr)) {
1016                         dev->stats.tx_bytes +=
1017                                 dma_unmap_len(tx_cb_ptr, dma_len);
1018                         dma_unmap_page(&dev->dev,
1019                                        dma_unmap_addr(tx_cb_ptr, dma_addr),
1020                                        dma_unmap_len(tx_cb_ptr, dma_len),
1021                                        DMA_TO_DEVICE);
1022                         dma_unmap_addr_set(tx_cb_ptr, dma_addr, 0);
1023                 }
1024
1025                 txbds_processed++;
1026                 if (likely(ring->clean_ptr < ring->end_ptr))
1027                         ring->clean_ptr++;
1028                 else
1029                         ring->clean_ptr = ring->cb_ptr;
1030         }
1031
1032         ring->free_bds += txbds_processed;
1033         ring->c_index = (ring->c_index + txbds_processed) & DMA_C_INDEX_MASK;
1034
1035         if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {
1036                 txq = netdev_get_tx_queue(dev, ring->queue);
1037                 if (netif_tx_queue_stopped(txq))
1038                         netif_tx_wake_queue(txq);
1039         }
1040
1041         return pkts_compl;
1042 }
1043
1044 static unsigned int bcmgenet_tx_reclaim(struct net_device *dev,
1045                                 struct bcmgenet_tx_ring *ring)
1046 {
1047         unsigned int released;
1048         unsigned long flags;
1049
1050         spin_lock_irqsave(&ring->lock, flags);
1051         released = __bcmgenet_tx_reclaim(dev, ring);
1052         spin_unlock_irqrestore(&ring->lock, flags);
1053
1054         return released;
1055 }
1056
1057 static int bcmgenet_tx_poll(struct napi_struct *napi, int budget)
1058 {
1059         struct bcmgenet_tx_ring *ring =
1060                 container_of(napi, struct bcmgenet_tx_ring, napi);
1061         unsigned int work_done = 0;
1062
1063         work_done = bcmgenet_tx_reclaim(ring->priv->dev, ring);
1064
1065         if (work_done == 0) {
1066                 napi_complete(napi);
1067                 ring->int_enable(ring->priv, ring);
1068
1069                 return 0;
1070         }
1071
1072         return budget;
1073 }
1074
1075 static void bcmgenet_tx_reclaim_all(struct net_device *dev)
1076 {
1077         struct bcmgenet_priv *priv = netdev_priv(dev);
1078         int i;
1079
1080         if (netif_is_multiqueue(dev)) {
1081                 for (i = 0; i < priv->hw_params->tx_queues; i++)
1082                         bcmgenet_tx_reclaim(dev, &priv->tx_rings[i]);
1083         }
1084
1085         bcmgenet_tx_reclaim(dev, &priv->tx_rings[DESC_INDEX]);
1086 }
1087
1088 /* Transmits a single SKB (either head of a fragment or a single SKB)
1089  * caller must hold priv->lock
1090  */
1091 static int bcmgenet_xmit_single(struct net_device *dev,
1092                                 struct sk_buff *skb,
1093                                 u16 dma_desc_flags,
1094                                 struct bcmgenet_tx_ring *ring)
1095 {
1096         struct bcmgenet_priv *priv = netdev_priv(dev);
1097         struct device *kdev = &priv->pdev->dev;
1098         struct enet_cb *tx_cb_ptr;
1099         unsigned int skb_len;
1100         dma_addr_t mapping;
1101         u32 length_status;
1102         int ret;
1103
1104         tx_cb_ptr = bcmgenet_get_txcb(priv, ring);
1105
1106         if (unlikely(!tx_cb_ptr))
1107                 BUG();
1108
1109         tx_cb_ptr->skb = skb;
1110
1111         skb_len = skb_headlen(skb) < ETH_ZLEN ? ETH_ZLEN : skb_headlen(skb);
1112
1113         mapping = dma_map_single(kdev, skb->data, skb_len, DMA_TO_DEVICE);
1114         ret = dma_mapping_error(kdev, mapping);
1115         if (ret) {
1116                 priv->mib.tx_dma_failed++;
1117                 netif_err(priv, tx_err, dev, "Tx DMA map failed\n");
1118                 dev_kfree_skb(skb);
1119                 return ret;
1120         }
1121
1122         dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
1123         dma_unmap_len_set(tx_cb_ptr, dma_len, skb->len);
1124         length_status = (skb_len << DMA_BUFLENGTH_SHIFT) | dma_desc_flags |
1125                         (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT) |
1126                         DMA_TX_APPEND_CRC;
1127
1128         if (skb->ip_summed == CHECKSUM_PARTIAL)
1129                 length_status |= DMA_TX_DO_CSUM;
1130
1131         dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, length_status);
1132
1133         return 0;
1134 }
1135
1136 /* Transmit a SKB fragment */
1137 static int bcmgenet_xmit_frag(struct net_device *dev,
1138                               skb_frag_t *frag,
1139                               u16 dma_desc_flags,
1140                               struct bcmgenet_tx_ring *ring)
1141 {
1142         struct bcmgenet_priv *priv = netdev_priv(dev);
1143         struct device *kdev = &priv->pdev->dev;
1144         struct enet_cb *tx_cb_ptr;
1145         dma_addr_t mapping;
1146         int ret;
1147
1148         tx_cb_ptr = bcmgenet_get_txcb(priv, ring);
1149
1150         if (unlikely(!tx_cb_ptr))
1151                 BUG();
1152         tx_cb_ptr->skb = NULL;
1153
1154         mapping = skb_frag_dma_map(kdev, frag, 0,
1155                                    skb_frag_size(frag), DMA_TO_DEVICE);
1156         ret = dma_mapping_error(kdev, mapping);
1157         if (ret) {
1158                 priv->mib.tx_dma_failed++;
1159                 netif_err(priv, tx_err, dev, "%s: Tx DMA map failed\n",
1160                           __func__);
1161                 return ret;
1162         }
1163
1164         dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
1165         dma_unmap_len_set(tx_cb_ptr, dma_len, frag->size);
1166
1167         dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping,
1168                     (frag->size << DMA_BUFLENGTH_SHIFT) | dma_desc_flags |
1169                     (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT));
1170
1171         return 0;
1172 }
1173
1174 /* Reallocate the SKB to put enough headroom in front of it and insert
1175  * the transmit checksum offsets in the descriptors
1176  */
1177 static struct sk_buff *bcmgenet_put_tx_csum(struct net_device *dev,
1178                                             struct sk_buff *skb)
1179 {
1180         struct status_64 *status = NULL;
1181         struct sk_buff *new_skb;
1182         u16 offset;
1183         u8 ip_proto;
1184         u16 ip_ver;
1185         u32 tx_csum_info;
1186
1187         if (unlikely(skb_headroom(skb) < sizeof(*status))) {
1188                 /* If 64 byte status block enabled, must make sure skb has
1189                  * enough headroom for us to insert 64B status block.
1190                  */
1191                 new_skb = skb_realloc_headroom(skb, sizeof(*status));
1192                 dev_kfree_skb(skb);
1193                 if (!new_skb) {
1194                         dev->stats.tx_errors++;
1195                         dev->stats.tx_dropped++;
1196                         return NULL;
1197                 }
1198                 skb = new_skb;
1199         }
1200
1201         skb_push(skb, sizeof(*status));
1202         status = (struct status_64 *)skb->data;
1203
1204         if (skb->ip_summed  == CHECKSUM_PARTIAL) {
1205                 ip_ver = htons(skb->protocol);
1206                 switch (ip_ver) {
1207                 case ETH_P_IP:
1208                         ip_proto = ip_hdr(skb)->protocol;
1209                         break;
1210                 case ETH_P_IPV6:
1211                         ip_proto = ipv6_hdr(skb)->nexthdr;
1212                         break;
1213                 default:
1214                         return skb;
1215                 }
1216
1217                 offset = skb_checksum_start_offset(skb) - sizeof(*status);
1218                 tx_csum_info = (offset << STATUS_TX_CSUM_START_SHIFT) |
1219                                 (offset + skb->csum_offset);
1220
1221                 /* Set the length valid bit for TCP and UDP and just set
1222                  * the special UDP flag for IPv4, else just set to 0.
1223                  */
1224                 if (ip_proto == IPPROTO_TCP || ip_proto == IPPROTO_UDP) {
1225                         tx_csum_info |= STATUS_TX_CSUM_LV;
1226                         if (ip_proto == IPPROTO_UDP && ip_ver == ETH_P_IP)
1227                                 tx_csum_info |= STATUS_TX_CSUM_PROTO_UDP;
1228                 } else {
1229                         tx_csum_info = 0;
1230                 }
1231
1232                 status->tx_csum_info = tx_csum_info;
1233         }
1234
1235         return skb;
1236 }
1237
1238 static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
1239 {
1240         struct bcmgenet_priv *priv = netdev_priv(dev);
1241         struct bcmgenet_tx_ring *ring = NULL;
1242         struct netdev_queue *txq;
1243         unsigned long flags = 0;
1244         int nr_frags, index;
1245         u16 dma_desc_flags;
1246         int ret;
1247         int i;
1248
1249         index = skb_get_queue_mapping(skb);
1250         /* Mapping strategy:
1251          * queue_mapping = 0, unclassified, packet xmited through ring16
1252          * queue_mapping = 1, goes to ring 0. (highest priority queue
1253          * queue_mapping = 2, goes to ring 1.
1254          * queue_mapping = 3, goes to ring 2.
1255          * queue_mapping = 4, goes to ring 3.
1256          */
1257         if (index == 0)
1258                 index = DESC_INDEX;
1259         else
1260                 index -= 1;
1261
1262         nr_frags = skb_shinfo(skb)->nr_frags;
1263         ring = &priv->tx_rings[index];
1264         txq = netdev_get_tx_queue(dev, ring->queue);
1265
1266         spin_lock_irqsave(&ring->lock, flags);
1267         if (ring->free_bds <= nr_frags + 1) {
1268                 netif_tx_stop_queue(txq);
1269                 netdev_err(dev, "%s: tx ring %d full when queue %d awake\n",
1270                            __func__, index, ring->queue);
1271                 ret = NETDEV_TX_BUSY;
1272                 goto out;
1273         }
1274
1275         if (skb_padto(skb, ETH_ZLEN)) {
1276                 ret = NETDEV_TX_OK;
1277                 goto out;
1278         }
1279
1280         /* set the SKB transmit checksum */
1281         if (priv->desc_64b_en) {
1282                 skb = bcmgenet_put_tx_csum(dev, skb);
1283                 if (!skb) {
1284                         ret = NETDEV_TX_OK;
1285                         goto out;
1286                 }
1287         }
1288
1289         dma_desc_flags = DMA_SOP;
1290         if (nr_frags == 0)
1291                 dma_desc_flags |= DMA_EOP;
1292
1293         /* Transmit single SKB or head of fragment list */
1294         ret = bcmgenet_xmit_single(dev, skb, dma_desc_flags, ring);
1295         if (ret) {
1296                 ret = NETDEV_TX_OK;
1297                 goto out;
1298         }
1299
1300         /* xmit fragment */
1301         for (i = 0; i < nr_frags; i++) {
1302                 ret = bcmgenet_xmit_frag(dev,
1303                                          &skb_shinfo(skb)->frags[i],
1304                                          (i == nr_frags - 1) ? DMA_EOP : 0,
1305                                          ring);
1306                 if (ret) {
1307                         ret = NETDEV_TX_OK;
1308                         goto out;
1309                 }
1310         }
1311
1312         skb_tx_timestamp(skb);
1313
1314         /* Decrement total BD count and advance our write pointer */
1315         ring->free_bds -= nr_frags + 1;
1316         ring->prod_index += nr_frags + 1;
1317         ring->prod_index &= DMA_P_INDEX_MASK;
1318
1319         bcmgenet_tdma_ring_writel(priv, ring->index,
1320                                   ring->prod_index, TDMA_PROD_INDEX);
1321
1322         if (ring->free_bds <= (MAX_SKB_FRAGS + 1))
1323                 netif_tx_stop_queue(txq);
1324
1325 out:
1326         spin_unlock_irqrestore(&ring->lock, flags);
1327
1328         return ret;
1329 }
1330
1331 static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv,
1332                                           struct enet_cb *cb)
1333 {
1334         struct device *kdev = &priv->pdev->dev;
1335         struct sk_buff *skb;
1336         struct sk_buff *rx_skb;
1337         dma_addr_t mapping;
1338
1339         /* Allocate a new Rx skb */
1340         skb = netdev_alloc_skb(priv->dev, priv->rx_buf_len + SKB_ALIGNMENT);
1341         if (!skb) {
1342                 priv->mib.alloc_rx_buff_failed++;
1343                 netif_err(priv, rx_err, priv->dev,
1344                           "%s: Rx skb allocation failed\n", __func__);
1345                 return NULL;
1346         }
1347
1348         /* DMA-map the new Rx skb */
1349         mapping = dma_map_single(kdev, skb->data, priv->rx_buf_len,
1350                                  DMA_FROM_DEVICE);
1351         if (dma_mapping_error(kdev, mapping)) {
1352                 priv->mib.rx_dma_failed++;
1353                 dev_kfree_skb_any(skb);
1354                 netif_err(priv, rx_err, priv->dev,
1355                           "%s: Rx skb DMA mapping failed\n", __func__);
1356                 return NULL;
1357         }
1358
1359         /* Grab the current Rx skb from the ring and DMA-unmap it */
1360         rx_skb = cb->skb;
1361         if (likely(rx_skb))
1362                 dma_unmap_single(kdev, dma_unmap_addr(cb, dma_addr),
1363                                  priv->rx_buf_len, DMA_FROM_DEVICE);
1364
1365         /* Put the new Rx skb on the ring */
1366         cb->skb = skb;
1367         dma_unmap_addr_set(cb, dma_addr, mapping);
1368         dmadesc_set_addr(priv, cb->bd_addr, mapping);
1369
1370         /* Return the current Rx skb to caller */
1371         return rx_skb;
1372 }
1373
1374 /* bcmgenet_desc_rx - descriptor based rx process.
1375  * this could be called from bottom half, or from NAPI polling method.
1376  */
1377 static unsigned int bcmgenet_desc_rx(struct bcmgenet_priv *priv,
1378                                      unsigned int index,
1379                                      unsigned int budget)
1380 {
1381         struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];
1382         struct net_device *dev = priv->dev;
1383         struct enet_cb *cb;
1384         struct sk_buff *skb;
1385         u32 dma_length_status;
1386         unsigned long dma_flag;
1387         int len;
1388         unsigned int rxpktprocessed = 0, rxpkttoprocess;
1389         unsigned int p_index;
1390         unsigned int discards;
1391         unsigned int chksum_ok = 0;
1392
1393         p_index = bcmgenet_rdma_ring_readl(priv, index, RDMA_PROD_INDEX);
1394
1395         discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) &
1396                    DMA_P_INDEX_DISCARD_CNT_MASK;
1397         if (discards > ring->old_discards) {
1398                 discards = discards - ring->old_discards;
1399                 dev->stats.rx_missed_errors += discards;
1400                 dev->stats.rx_errors += discards;
1401                 ring->old_discards += discards;
1402
1403                 /* Clear HW register when we reach 75% of maximum 0xFFFF */
1404                 if (ring->old_discards >= 0xC000) {
1405                         ring->old_discards = 0;
1406                         bcmgenet_rdma_ring_writel(priv, index, 0,
1407                                                   RDMA_PROD_INDEX);
1408                 }
1409         }
1410
1411         p_index &= DMA_P_INDEX_MASK;
1412
1413         if (likely(p_index >= ring->c_index))
1414                 rxpkttoprocess = p_index - ring->c_index;
1415         else
1416                 rxpkttoprocess = (DMA_C_INDEX_MASK + 1) - ring->c_index +
1417                                  p_index;
1418
1419         netif_dbg(priv, rx_status, dev,
1420                   "RDMA: rxpkttoprocess=%d\n", rxpkttoprocess);
1421
1422         while ((rxpktprocessed < rxpkttoprocess) &&
1423                (rxpktprocessed < budget)) {
1424                 cb = &priv->rx_cbs[ring->read_ptr];
1425                 skb = bcmgenet_rx_refill(priv, cb);
1426
1427                 if (unlikely(!skb)) {
1428                         dev->stats.rx_dropped++;
1429                         dev->stats.rx_errors++;
1430                         goto next;
1431                 }
1432
1433                 if (!priv->desc_64b_en) {
1434                         dma_length_status =
1435                                 dmadesc_get_length_status(priv, cb->bd_addr);
1436                 } else {
1437                         struct status_64 *status;
1438
1439                         status = (struct status_64 *)skb->data;
1440                         dma_length_status = status->length_status;
1441                 }
1442
1443                 /* DMA flags and length are still valid no matter how
1444                  * we got the Receive Status Vector (64B RSB or register)
1445                  */
1446                 dma_flag = dma_length_status & 0xffff;
1447                 len = dma_length_status >> DMA_BUFLENGTH_SHIFT;
1448
1449                 netif_dbg(priv, rx_status, dev,
1450                           "%s:p_ind=%d c_ind=%d read_ptr=%d len_stat=0x%08x\n",
1451                           __func__, p_index, ring->c_index,
1452                           ring->read_ptr, dma_length_status);
1453
1454                 if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) {
1455                         netif_err(priv, rx_status, dev,
1456                                   "dropping fragmented packet!\n");
1457                         dev->stats.rx_dropped++;
1458                         dev->stats.rx_errors++;
1459                         dev_kfree_skb_any(skb);
1460                         goto next;
1461                 }
1462
1463                 /* report errors */
1464                 if (unlikely(dma_flag & (DMA_RX_CRC_ERROR |
1465                                                 DMA_RX_OV |
1466                                                 DMA_RX_NO |
1467                                                 DMA_RX_LG |
1468                                                 DMA_RX_RXER))) {
1469                         netif_err(priv, rx_status, dev, "dma_flag=0x%x\n",
1470                                   (unsigned int)dma_flag);
1471                         if (dma_flag & DMA_RX_CRC_ERROR)
1472                                 dev->stats.rx_crc_errors++;
1473                         if (dma_flag & DMA_RX_OV)
1474                                 dev->stats.rx_over_errors++;
1475                         if (dma_flag & DMA_RX_NO)
1476                                 dev->stats.rx_frame_errors++;
1477                         if (dma_flag & DMA_RX_LG)
1478                                 dev->stats.rx_length_errors++;
1479                         dev->stats.rx_dropped++;
1480                         dev->stats.rx_errors++;
1481                         dev_kfree_skb_any(skb);
1482                         goto next;
1483                 } /* error packet */
1484
1485                 chksum_ok = (dma_flag & priv->dma_rx_chk_bit) &&
1486                              priv->desc_rxchk_en;
1487
1488                 skb_put(skb, len);
1489                 if (priv->desc_64b_en) {
1490                         skb_pull(skb, 64);
1491                         len -= 64;
1492                 }
1493
1494                 if (likely(chksum_ok))
1495                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1496
1497                 /* remove hardware 2bytes added for IP alignment */
1498                 skb_pull(skb, 2);
1499                 len -= 2;
1500
1501                 if (priv->crc_fwd_en) {
1502                         skb_trim(skb, len - ETH_FCS_LEN);
1503                         len -= ETH_FCS_LEN;
1504                 }
1505
1506                 /*Finish setting up the received SKB and send it to the kernel*/
1507                 skb->protocol = eth_type_trans(skb, priv->dev);
1508                 dev->stats.rx_packets++;
1509                 dev->stats.rx_bytes += len;
1510                 if (dma_flag & DMA_RX_MULT)
1511                         dev->stats.multicast++;
1512
1513                 /* Notify kernel */
1514                 napi_gro_receive(&priv->napi, skb);
1515                 netif_dbg(priv, rx_status, dev, "pushed up to kernel\n");
1516
1517 next:
1518                 rxpktprocessed++;
1519                 if (likely(ring->read_ptr < ring->end_ptr))
1520                         ring->read_ptr++;
1521                 else
1522                         ring->read_ptr = ring->cb_ptr;
1523
1524                 ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;
1525                 bcmgenet_rdma_ring_writel(priv, index, ring->c_index, RDMA_CONS_INDEX);
1526         }
1527
1528         return rxpktprocessed;
1529 }
1530
1531 /* Assign skb to RX DMA descriptor. */
1532 static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv,
1533                                      struct bcmgenet_rx_ring *ring)
1534 {
1535         struct enet_cb *cb;
1536         struct sk_buff *skb;
1537         int i;
1538
1539         netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
1540
1541         /* loop here for each buffer needing assign */
1542         for (i = 0; i < ring->size; i++) {
1543                 cb = ring->cbs + i;
1544                 skb = bcmgenet_rx_refill(priv, cb);
1545                 if (skb)
1546                         dev_kfree_skb_any(skb);
1547                 if (!cb->skb)
1548                         return -ENOMEM;
1549         }
1550
1551         return 0;
1552 }
1553
1554 static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv)
1555 {
1556         struct enet_cb *cb;
1557         int i;
1558
1559         for (i = 0; i < priv->num_rx_bds; i++) {
1560                 cb = &priv->rx_cbs[i];
1561
1562                 if (dma_unmap_addr(cb, dma_addr)) {
1563                         dma_unmap_single(&priv->dev->dev,
1564                                          dma_unmap_addr(cb, dma_addr),
1565                                          priv->rx_buf_len, DMA_FROM_DEVICE);
1566                         dma_unmap_addr_set(cb, dma_addr, 0);
1567                 }
1568
1569                 if (cb->skb)
1570                         bcmgenet_free_cb(cb);
1571         }
1572 }
1573
1574 static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable)
1575 {
1576         u32 reg;
1577
1578         reg = bcmgenet_umac_readl(priv, UMAC_CMD);
1579         if (enable)
1580                 reg |= mask;
1581         else
1582                 reg &= ~mask;
1583         bcmgenet_umac_writel(priv, reg, UMAC_CMD);
1584
1585         /* UniMAC stops on a packet boundary, wait for a full-size packet
1586          * to be processed
1587          */
1588         if (enable == 0)
1589                 usleep_range(1000, 2000);
1590 }
1591
1592 static int reset_umac(struct bcmgenet_priv *priv)
1593 {
1594         struct device *kdev = &priv->pdev->dev;
1595         unsigned int timeout = 0;
1596         u32 reg;
1597
1598         /* 7358a0/7552a0: bad default in RBUF_FLUSH_CTRL.umac_sw_rst */
1599         bcmgenet_rbuf_ctrl_set(priv, 0);
1600         udelay(10);
1601
1602         /* disable MAC while updating its registers */
1603         bcmgenet_umac_writel(priv, 0, UMAC_CMD);
1604
1605         /* issue soft reset, wait for it to complete */
1606         bcmgenet_umac_writel(priv, CMD_SW_RESET, UMAC_CMD);
1607         while (timeout++ < 1000) {
1608                 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
1609                 if (!(reg & CMD_SW_RESET))
1610                         return 0;
1611
1612                 udelay(1);
1613         }
1614
1615         if (timeout == 1000) {
1616                 dev_err(kdev,
1617                         "timeout waiting for MAC to come out of reset\n");
1618                 return -ETIMEDOUT;
1619         }
1620
1621         return 0;
1622 }
1623
1624 static void bcmgenet_intr_disable(struct bcmgenet_priv *priv)
1625 {
1626         /* Mask all interrupts.*/
1627         bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
1628         bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
1629         bcmgenet_intrl2_0_writel(priv, 0, INTRL2_CPU_MASK_CLEAR);
1630         bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
1631         bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
1632         bcmgenet_intrl2_1_writel(priv, 0, INTRL2_CPU_MASK_CLEAR);
1633 }
1634
1635 static int init_umac(struct bcmgenet_priv *priv)
1636 {
1637         struct device *kdev = &priv->pdev->dev;
1638         int ret;
1639         u32 reg, cpu_mask_clear;
1640         int index;
1641
1642         dev_dbg(&priv->pdev->dev, "bcmgenet: init_umac\n");
1643
1644         ret = reset_umac(priv);
1645         if (ret)
1646                 return ret;
1647
1648         bcmgenet_umac_writel(priv, 0, UMAC_CMD);
1649         /* clear tx/rx counter */
1650         bcmgenet_umac_writel(priv,
1651                              MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT,
1652                              UMAC_MIB_CTRL);
1653         bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL);
1654
1655         bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
1656
1657         /* init rx registers, enable ip header optimization */
1658         reg = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
1659         reg |= RBUF_ALIGN_2B;
1660         bcmgenet_rbuf_writel(priv, reg, RBUF_CTRL);
1661
1662         if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv))
1663                 bcmgenet_rbuf_writel(priv, 1, RBUF_TBUF_SIZE_CTRL);
1664
1665         bcmgenet_intr_disable(priv);
1666
1667         cpu_mask_clear = UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_TXDMA_BDONE;
1668
1669         dev_dbg(kdev, "%s:Enabling RXDMA_BDONE interrupt\n", __func__);
1670
1671         /* Monitor cable plug/unplugged event for internal PHY */
1672         if (phy_is_internal(priv->phydev)) {
1673                 cpu_mask_clear |= (UMAC_IRQ_LINK_DOWN | UMAC_IRQ_LINK_UP);
1674         } else if (priv->ext_phy) {
1675                 cpu_mask_clear |= (UMAC_IRQ_LINK_DOWN | UMAC_IRQ_LINK_UP);
1676         } else if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
1677                 reg = bcmgenet_bp_mc_get(priv);
1678                 reg |= BIT(priv->hw_params->bp_in_en_shift);
1679
1680                 /* bp_mask: back pressure mask */
1681                 if (netif_is_multiqueue(priv->dev))
1682                         reg |= priv->hw_params->bp_in_mask;
1683                 else
1684                         reg &= ~priv->hw_params->bp_in_mask;
1685                 bcmgenet_bp_mc_set(priv, reg);
1686         }
1687
1688         /* Enable MDIO interrupts on GENET v3+ */
1689         if (priv->hw_params->flags & GENET_HAS_MDIO_INTR)
1690                 cpu_mask_clear |= UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR;
1691
1692         bcmgenet_intrl2_0_writel(priv, cpu_mask_clear, INTRL2_CPU_MASK_CLEAR);
1693
1694         for (index = 0; index < priv->hw_params->tx_queues; index++)
1695                 bcmgenet_intrl2_1_writel(priv, (1 << index),
1696                                          INTRL2_CPU_MASK_CLEAR);
1697
1698         /* Enable rx/tx engine.*/
1699         dev_dbg(kdev, "done init umac\n");
1700
1701         return 0;
1702 }
1703
1704 /* Initialize a Tx ring along with corresponding hardware registers */
1705 static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv,
1706                                   unsigned int index, unsigned int size,
1707                                   unsigned int start_ptr, unsigned int end_ptr)
1708 {
1709         struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];
1710         u32 words_per_bd = WORDS_PER_BD(priv);
1711         u32 flow_period_val = 0;
1712
1713         spin_lock_init(&ring->lock);
1714         ring->priv = priv;
1715         netif_napi_add(priv->dev, &ring->napi, bcmgenet_tx_poll, 64);
1716         ring->index = index;
1717         if (index == DESC_INDEX) {
1718                 ring->queue = 0;
1719                 ring->int_enable = bcmgenet_tx_ring16_int_enable;
1720                 ring->int_disable = bcmgenet_tx_ring16_int_disable;
1721         } else {
1722                 ring->queue = index + 1;
1723                 ring->int_enable = bcmgenet_tx_ring_int_enable;
1724                 ring->int_disable = bcmgenet_tx_ring_int_disable;
1725         }
1726         ring->cbs = priv->tx_cbs + start_ptr;
1727         ring->size = size;
1728         ring->clean_ptr = start_ptr;
1729         ring->c_index = 0;
1730         ring->free_bds = size;
1731         ring->write_ptr = start_ptr;
1732         ring->cb_ptr = start_ptr;
1733         ring->end_ptr = end_ptr - 1;
1734         ring->prod_index = 0;
1735
1736         /* Set flow period for ring != 16 */
1737         if (index != DESC_INDEX)
1738                 flow_period_val = ENET_MAX_MTU_SIZE << 16;
1739
1740         bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX);
1741         bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX);
1742         bcmgenet_tdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
1743         /* Disable rate control for now */
1744         bcmgenet_tdma_ring_writel(priv, index, flow_period_val,
1745                                   TDMA_FLOW_PERIOD);
1746         bcmgenet_tdma_ring_writel(priv, index,
1747                                   ((size << DMA_RING_SIZE_SHIFT) |
1748                                    RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
1749
1750         /* Set start and end address, read and write pointers */
1751         bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
1752                                   DMA_START_ADDR);
1753         bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
1754                                   TDMA_READ_PTR);
1755         bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
1756                                   TDMA_WRITE_PTR);
1757         bcmgenet_tdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
1758                                   DMA_END_ADDR);
1759
1760         napi_enable(&ring->napi);
1761 }
1762
1763 static void bcmgenet_fini_tx_ring(struct bcmgenet_priv *priv,
1764                                   unsigned int index)
1765 {
1766         struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];
1767
1768         napi_disable(&ring->napi);
1769         netif_napi_del(&ring->napi);
1770 }
1771
1772 /* Initialize a RDMA ring */
1773 static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
1774                                  unsigned int index, unsigned int size,
1775                                  unsigned int start_ptr, unsigned int end_ptr)
1776 {
1777         struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];
1778         u32 words_per_bd = WORDS_PER_BD(priv);
1779         int ret;
1780
1781         ring->index = index;
1782         ring->cbs = priv->rx_cbs + start_ptr;
1783         ring->size = size;
1784         ring->c_index = 0;
1785         ring->read_ptr = start_ptr;
1786         ring->cb_ptr = start_ptr;
1787         ring->end_ptr = end_ptr - 1;
1788
1789         ret = bcmgenet_alloc_rx_buffers(priv, ring);
1790         if (ret)
1791                 return ret;
1792
1793         bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX);
1794         bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX);
1795         bcmgenet_rdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
1796         bcmgenet_rdma_ring_writel(priv, index,
1797                                   ((size << DMA_RING_SIZE_SHIFT) |
1798                                    RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
1799         bcmgenet_rdma_ring_writel(priv, index,
1800                                   (DMA_FC_THRESH_LO <<
1801                                    DMA_XOFF_THRESHOLD_SHIFT) |
1802                                    DMA_FC_THRESH_HI, RDMA_XON_XOFF_THRESH);
1803
1804         /* Set start and end address, read and write pointers */
1805         bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
1806                                   DMA_START_ADDR);
1807         bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
1808                                   RDMA_READ_PTR);
1809         bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
1810                                   RDMA_WRITE_PTR);
1811         bcmgenet_rdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
1812                                   DMA_END_ADDR);
1813
1814         return ret;
1815 }
1816
1817 /* Initialize Tx queues
1818  *
1819  * Queues 0-3 are priority-based, each one has 32 descriptors,
1820  * with queue 0 being the highest priority queue.
1821  *
1822  * Queue 16 is the default Tx queue with
1823  * GENET_Q16_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors.
1824  *
1825  * The transmit control block pool is then partitioned as follows:
1826  * - Tx queue 0 uses tx_cbs[0..31]
1827  * - Tx queue 1 uses tx_cbs[32..63]
1828  * - Tx queue 2 uses tx_cbs[64..95]
1829  * - Tx queue 3 uses tx_cbs[96..127]
1830  * - Tx queue 16 uses tx_cbs[128..255]
1831  */
1832 static void bcmgenet_init_tx_queues(struct net_device *dev)
1833 {
1834         struct bcmgenet_priv *priv = netdev_priv(dev);
1835         u32 i, dma_enable;
1836         u32 dma_ctrl, ring_cfg;
1837         u32 dma_priority[3] = {0, 0, 0};
1838
1839         dma_ctrl = bcmgenet_tdma_readl(priv, DMA_CTRL);
1840         dma_enable = dma_ctrl & DMA_EN;
1841         dma_ctrl &= ~DMA_EN;
1842         bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);
1843
1844         dma_ctrl = 0;
1845         ring_cfg = 0;
1846
1847         /* Enable strict priority arbiter mode */
1848         bcmgenet_tdma_writel(priv, DMA_ARBITER_SP, DMA_ARB_CTRL);
1849
1850         /* Initialize Tx priority queues */
1851         for (i = 0; i < priv->hw_params->tx_queues; i++) {
1852                 bcmgenet_init_tx_ring(priv, i, priv->hw_params->tx_bds_per_q,
1853                                       i * priv->hw_params->tx_bds_per_q,
1854                                       (i + 1) * priv->hw_params->tx_bds_per_q);
1855                 ring_cfg |= (1 << i);
1856                 dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
1857                 dma_priority[DMA_PRIO_REG_INDEX(i)] |=
1858                         ((GENET_Q0_PRIORITY + i) << DMA_PRIO_REG_SHIFT(i));
1859         }
1860
1861         /* Initialize Tx default queue 16 */
1862         bcmgenet_init_tx_ring(priv, DESC_INDEX, GENET_Q16_TX_BD_CNT,
1863                               priv->hw_params->tx_queues *
1864                               priv->hw_params->tx_bds_per_q,
1865                               TOTAL_DESC);
1866         ring_cfg |= (1 << DESC_INDEX);
1867         dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));
1868         dma_priority[DMA_PRIO_REG_INDEX(DESC_INDEX)] |=
1869                 ((GENET_Q0_PRIORITY + priv->hw_params->tx_queues) <<
1870                  DMA_PRIO_REG_SHIFT(DESC_INDEX));
1871
1872         /* Set Tx queue priorities */
1873         bcmgenet_tdma_writel(priv, dma_priority[0], DMA_PRIORITY_0);
1874         bcmgenet_tdma_writel(priv, dma_priority[1], DMA_PRIORITY_1);
1875         bcmgenet_tdma_writel(priv, dma_priority[2], DMA_PRIORITY_2);
1876
1877         /* Enable Tx queues */
1878         bcmgenet_tdma_writel(priv, ring_cfg, DMA_RING_CFG);
1879
1880         /* Enable Tx DMA */
1881         if (dma_enable)
1882                 dma_ctrl |= DMA_EN;
1883         bcmgenet_tdma_writel(priv, dma_ctrl, DMA_CTRL);
1884 }
1885
1886 /* Initialize Rx queues
1887  *
1888  * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be
1889  * used to direct traffic to these queues.
1890  *
1891  * Queue 16 is the default Rx queue with GENET_Q16_RX_BD_CNT descriptors.
1892  */
1893 static int bcmgenet_init_rx_queues(struct net_device *dev)
1894 {
1895         struct bcmgenet_priv *priv = netdev_priv(dev);
1896         u32 i;
1897         u32 dma_enable;
1898         u32 dma_ctrl;
1899         u32 ring_cfg;
1900         int ret;
1901
1902         dma_ctrl = bcmgenet_rdma_readl(priv, DMA_CTRL);
1903         dma_enable = dma_ctrl & DMA_EN;
1904         dma_ctrl &= ~DMA_EN;
1905         bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);
1906
1907         dma_ctrl = 0;
1908         ring_cfg = 0;
1909
1910         /* Initialize Rx priority queues */
1911         for (i = 0; i < priv->hw_params->rx_queues; i++) {
1912                 ret = bcmgenet_init_rx_ring(priv, i,
1913                                             priv->hw_params->rx_bds_per_q,
1914                                             i * priv->hw_params->rx_bds_per_q,
1915                                             (i + 1) *
1916                                             priv->hw_params->rx_bds_per_q);
1917                 if (ret)
1918                         return ret;
1919
1920                 ring_cfg |= (1 << i);
1921                 dma_ctrl |= (1 << (i + DMA_RING_BUF_EN_SHIFT));
1922         }
1923
1924         /* Initialize Rx default queue 16 */
1925         ret = bcmgenet_init_rx_ring(priv, DESC_INDEX, GENET_Q16_RX_BD_CNT,
1926                                     priv->hw_params->rx_queues *
1927                                     priv->hw_params->rx_bds_per_q,
1928                                     TOTAL_DESC);
1929         if (ret)
1930                 return ret;
1931
1932         ring_cfg |= (1 << DESC_INDEX);
1933         dma_ctrl |= (1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT));
1934
1935         /* Enable rings */
1936         bcmgenet_rdma_writel(priv, ring_cfg, DMA_RING_CFG);
1937
1938         /* Configure ring as descriptor ring and re-enable DMA if enabled */
1939         if (dma_enable)
1940                 dma_ctrl |= DMA_EN;
1941         bcmgenet_rdma_writel(priv, dma_ctrl, DMA_CTRL);
1942
1943         return 0;
1944 }
1945
1946 static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv)
1947 {
1948         int ret = 0;
1949         int timeout = 0;
1950         u32 reg;
1951
1952         /* Disable TDMA to stop add more frames in TX DMA */
1953         reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
1954         reg &= ~DMA_EN;
1955         bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
1956
1957         /* Check TDMA status register to confirm TDMA is disabled */
1958         while (timeout++ < DMA_TIMEOUT_VAL) {
1959                 reg = bcmgenet_tdma_readl(priv, DMA_STATUS);
1960                 if (reg & DMA_DISABLED)
1961                         break;
1962
1963                 udelay(1);
1964         }
1965
1966         if (timeout == DMA_TIMEOUT_VAL) {
1967                 netdev_warn(priv->dev, "Timed out while disabling TX DMA\n");
1968                 ret = -ETIMEDOUT;
1969         }
1970
1971         /* Wait 10ms for packet drain in both tx and rx dma */
1972         usleep_range(10000, 20000);
1973
1974         /* Disable RDMA */
1975         reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
1976         reg &= ~DMA_EN;
1977         bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
1978
1979         timeout = 0;
1980         /* Check RDMA status register to confirm RDMA is disabled */
1981         while (timeout++ < DMA_TIMEOUT_VAL) {
1982                 reg = bcmgenet_rdma_readl(priv, DMA_STATUS);
1983                 if (reg & DMA_DISABLED)
1984                         break;
1985
1986                 udelay(1);
1987         }
1988
1989         if (timeout == DMA_TIMEOUT_VAL) {
1990                 netdev_warn(priv->dev, "Timed out while disabling RX DMA\n");
1991                 ret = -ETIMEDOUT;
1992         }
1993
1994         return ret;
1995 }
1996
1997 static void __bcmgenet_fini_dma(struct bcmgenet_priv *priv)
1998 {
1999         int i;
2000
2001         /* disable DMA */
2002         bcmgenet_dma_teardown(priv);
2003
2004         for (i = 0; i < priv->num_tx_bds; i++) {
2005                 if (priv->tx_cbs[i].skb != NULL) {
2006                         dev_kfree_skb(priv->tx_cbs[i].skb);
2007                         priv->tx_cbs[i].skb = NULL;
2008                 }
2009         }
2010
2011         bcmgenet_free_rx_buffers(priv);
2012         kfree(priv->rx_cbs);
2013         kfree(priv->tx_cbs);
2014 }
2015
2016 static void bcmgenet_fini_dma(struct bcmgenet_priv *priv)
2017 {
2018         int i;
2019
2020         bcmgenet_fini_tx_ring(priv, DESC_INDEX);
2021
2022         for (i = 0; i < priv->hw_params->tx_queues; i++)
2023                 bcmgenet_fini_tx_ring(priv, i);
2024
2025         __bcmgenet_fini_dma(priv);
2026 }
2027
2028 /* init_edma: Initialize DMA control register */
2029 static int bcmgenet_init_dma(struct bcmgenet_priv *priv)
2030 {
2031         int ret;
2032         unsigned int i;
2033         struct enet_cb *cb;
2034
2035         netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
2036
2037         /* Init rDma */
2038         bcmgenet_rdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);
2039
2040         /* Initialize common Rx ring structures */
2041         priv->rx_bds = priv->base + priv->hw_params->rdma_offset;
2042         priv->num_rx_bds = TOTAL_DESC;
2043         priv->rx_cbs = kcalloc(priv->num_rx_bds, sizeof(struct enet_cb),
2044                                GFP_KERNEL);
2045         if (!priv->rx_cbs)
2046                 return -ENOMEM;
2047
2048         for (i = 0; i < priv->num_rx_bds; i++) {
2049                 cb = priv->rx_cbs + i;
2050                 cb->bd_addr = priv->rx_bds + i * DMA_DESC_SIZE;
2051         }
2052
2053         /* Initialize Rx queues */
2054         ret = bcmgenet_init_rx_queues(priv->dev);
2055         if (ret) {
2056                 netdev_err(priv->dev, "failed to initialize Rx queues\n");
2057                 bcmgenet_free_rx_buffers(priv);
2058                 kfree(priv->rx_cbs);
2059                 return ret;
2060         }
2061
2062         /* Init tDma */
2063         bcmgenet_tdma_writel(priv, DMA_MAX_BURST_LENGTH, DMA_SCB_BURST_SIZE);
2064
2065         /* Initialize common TX ring structures */
2066         priv->tx_bds = priv->base + priv->hw_params->tdma_offset;
2067         priv->num_tx_bds = TOTAL_DESC;
2068         priv->tx_cbs = kcalloc(priv->num_tx_bds, sizeof(struct enet_cb),
2069                                GFP_KERNEL);
2070         if (!priv->tx_cbs) {
2071                 __bcmgenet_fini_dma(priv);
2072                 return -ENOMEM;
2073         }
2074
2075         for (i = 0; i < priv->num_tx_bds; i++) {
2076                 cb = priv->tx_cbs + i;
2077                 cb->bd_addr = priv->tx_bds + i * DMA_DESC_SIZE;
2078         }
2079
2080         /* Initialize Tx queues */
2081         bcmgenet_init_tx_queues(priv->dev);
2082
2083         return 0;
2084 }
2085
2086 /* NAPI polling method*/
2087 static int bcmgenet_poll(struct napi_struct *napi, int budget)
2088 {
2089         struct bcmgenet_priv *priv = container_of(napi,
2090                         struct bcmgenet_priv, napi);
2091         unsigned int work_done;
2092
2093         work_done = bcmgenet_desc_rx(priv, DESC_INDEX, budget);
2094
2095         if (work_done < budget) {
2096                 napi_complete(napi);
2097                 bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_BDONE,
2098                                          INTRL2_CPU_MASK_CLEAR);
2099         }
2100
2101         return work_done;
2102 }
2103
2104 /* Interrupt bottom half */
2105 static void bcmgenet_irq_task(struct work_struct *work)
2106 {
2107         struct bcmgenet_priv *priv = container_of(
2108                         work, struct bcmgenet_priv, bcmgenet_irq_work);
2109
2110         netif_dbg(priv, intr, priv->dev, "%s\n", __func__);
2111
2112         if (priv->irq0_stat & UMAC_IRQ_MPD_R) {
2113                 priv->irq0_stat &= ~UMAC_IRQ_MPD_R;
2114                 netif_dbg(priv, wol, priv->dev,
2115                           "magic packet detected, waking up\n");
2116                 bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC);
2117         }
2118
2119         /* Link UP/DOWN event */
2120         if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) &&
2121             (priv->irq0_stat & (UMAC_IRQ_LINK_UP|UMAC_IRQ_LINK_DOWN))) {
2122                 phy_mac_interrupt(priv->phydev,
2123                                   priv->irq0_stat & UMAC_IRQ_LINK_UP);
2124                 priv->irq0_stat &= ~(UMAC_IRQ_LINK_UP|UMAC_IRQ_LINK_DOWN);
2125         }
2126 }
2127
2128 /* bcmgenet_isr1: interrupt handler for ring buffer. */
2129 static irqreturn_t bcmgenet_isr1(int irq, void *dev_id)
2130 {
2131         struct bcmgenet_priv *priv = dev_id;
2132         struct bcmgenet_tx_ring *ring;
2133         unsigned int index;
2134
2135         /* Save irq status for bottom-half processing. */
2136         priv->irq1_stat =
2137                 bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) &
2138                 ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
2139         /* clear interrupts */
2140         bcmgenet_intrl2_1_writel(priv, priv->irq1_stat, INTRL2_CPU_CLEAR);
2141
2142         netif_dbg(priv, intr, priv->dev,
2143                   "%s: IRQ=0x%x\n", __func__, priv->irq1_stat);
2144
2145         /* Check the MBDONE interrupts.
2146          * packet is done, reclaim descriptors
2147          */
2148         for (index = 0; index < priv->hw_params->tx_queues; index++) {
2149                 if (!(priv->irq1_stat & BIT(index)))
2150                         continue;
2151
2152                 ring = &priv->tx_rings[index];
2153
2154                 if (likely(napi_schedule_prep(&ring->napi))) {
2155                         ring->int_disable(priv, ring);
2156                         __napi_schedule(&ring->napi);
2157                 }
2158         }
2159
2160         return IRQ_HANDLED;
2161 }
2162
2163 /* bcmgenet_isr0: Handle various interrupts. */
2164 static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
2165 {
2166         struct bcmgenet_priv *priv = dev_id;
2167
2168         /* Save irq status for bottom-half processing. */
2169         priv->irq0_stat =
2170                 bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) &
2171                 ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
2172         /* clear interrupts */
2173         bcmgenet_intrl2_0_writel(priv, priv->irq0_stat, INTRL2_CPU_CLEAR);
2174
2175         netif_dbg(priv, intr, priv->dev,
2176                   "IRQ=0x%x\n", priv->irq0_stat);
2177
2178         if (priv->irq0_stat & (UMAC_IRQ_RXDMA_BDONE | UMAC_IRQ_RXDMA_PDONE)) {
2179                 /* We use NAPI(software interrupt throttling, if
2180                  * Rx Descriptor throttling is not used.
2181                  * Disable interrupt, will be enabled in the poll method.
2182                  */
2183                 if (likely(napi_schedule_prep(&priv->napi))) {
2184                         bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_RXDMA_BDONE,
2185                                                  INTRL2_CPU_MASK_SET);
2186                         __napi_schedule(&priv->napi);
2187                 }
2188         }
2189         if (priv->irq0_stat &
2190                         (UMAC_IRQ_TXDMA_BDONE | UMAC_IRQ_TXDMA_PDONE)) {
2191                 struct bcmgenet_tx_ring *ring = &priv->tx_rings[DESC_INDEX];
2192
2193                 if (likely(napi_schedule_prep(&ring->napi))) {
2194                         ring->int_disable(priv, ring);
2195                         __napi_schedule(&ring->napi);
2196                 }
2197         }
2198         if (priv->irq0_stat & (UMAC_IRQ_PHY_DET_R |
2199                                 UMAC_IRQ_PHY_DET_F |
2200                                 UMAC_IRQ_LINK_UP |
2201                                 UMAC_IRQ_LINK_DOWN |
2202                                 UMAC_IRQ_HFB_SM |
2203                                 UMAC_IRQ_HFB_MM |
2204                                 UMAC_IRQ_MPD_R)) {
2205                 /* all other interested interrupts handled in bottom half */
2206                 schedule_work(&priv->bcmgenet_irq_work);
2207         }
2208
2209         if ((priv->hw_params->flags & GENET_HAS_MDIO_INTR) &&
2210             priv->irq0_stat & (UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR)) {
2211                 priv->irq0_stat &= ~(UMAC_IRQ_MDIO_DONE | UMAC_IRQ_MDIO_ERROR);
2212                 wake_up(&priv->wq);
2213         }
2214
2215         return IRQ_HANDLED;
2216 }
2217
2218 static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id)
2219 {
2220         struct bcmgenet_priv *priv = dev_id;
2221
2222         pm_wakeup_event(&priv->pdev->dev, 0);
2223
2224         return IRQ_HANDLED;
2225 }
2226
2227 static void bcmgenet_umac_reset(struct bcmgenet_priv *priv)
2228 {
2229         u32 reg;
2230
2231         reg = bcmgenet_rbuf_ctrl_get(priv);
2232         reg |= BIT(1);
2233         bcmgenet_rbuf_ctrl_set(priv, reg);
2234         udelay(10);
2235
2236         reg &= ~BIT(1);
2237         bcmgenet_rbuf_ctrl_set(priv, reg);
2238         udelay(10);
2239 }
2240
2241 static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv,
2242                                  unsigned char *addr)
2243 {
2244         bcmgenet_umac_writel(priv, (addr[0] << 24) | (addr[1] << 16) |
2245                         (addr[2] << 8) | addr[3], UMAC_MAC0);
2246         bcmgenet_umac_writel(priv, (addr[4] << 8) | addr[5], UMAC_MAC1);
2247 }
2248
2249 /* Returns a reusable dma control register value */
2250 static u32 bcmgenet_dma_disable(struct bcmgenet_priv *priv)
2251 {
2252         u32 reg;
2253         u32 dma_ctrl;
2254
2255         /* disable DMA */
2256         dma_ctrl = 1 << (DESC_INDEX + DMA_RING_BUF_EN_SHIFT) | DMA_EN;
2257         reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2258         reg &= ~dma_ctrl;
2259         bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2260
2261         reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2262         reg &= ~dma_ctrl;
2263         bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2264
2265         bcmgenet_umac_writel(priv, 1, UMAC_TX_FLUSH);
2266         udelay(10);
2267         bcmgenet_umac_writel(priv, 0, UMAC_TX_FLUSH);
2268
2269         return dma_ctrl;
2270 }
2271
2272 static void bcmgenet_enable_dma(struct bcmgenet_priv *priv, u32 dma_ctrl)
2273 {
2274         u32 reg;
2275
2276         reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2277         reg |= dma_ctrl;
2278         bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2279
2280         reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2281         reg |= dma_ctrl;
2282         bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2283 }
2284
2285 static void bcmgenet_netif_start(struct net_device *dev)
2286 {
2287         struct bcmgenet_priv *priv = netdev_priv(dev);
2288
2289         /* Start the network engine */
2290         napi_enable(&priv->napi);
2291
2292         umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, true);
2293
2294         if (phy_is_internal(priv->phydev))
2295                 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
2296
2297         netif_tx_start_all_queues(dev);
2298
2299         phy_start(priv->phydev);
2300 }
2301
2302 static int bcmgenet_open(struct net_device *dev)
2303 {
2304         struct bcmgenet_priv *priv = netdev_priv(dev);
2305         unsigned long dma_ctrl;
2306         u32 reg;
2307         int ret;
2308
2309         netif_dbg(priv, ifup, dev, "bcmgenet_open\n");
2310
2311         /* Turn on the clock */
2312         if (!IS_ERR(priv->clk))
2313                 clk_prepare_enable(priv->clk);
2314
2315         /* take MAC out of reset */
2316         bcmgenet_umac_reset(priv);
2317
2318         ret = init_umac(priv);
2319         if (ret)
2320                 goto err_clk_disable;
2321
2322         /* disable ethernet MAC while updating its registers */
2323         umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, false);
2324
2325         /* Make sure we reflect the value of CRC_CMD_FWD */
2326         reg = bcmgenet_umac_readl(priv, UMAC_CMD);
2327         priv->crc_fwd_en = !!(reg & CMD_CRC_FWD);
2328
2329         bcmgenet_set_hw_addr(priv, dev->dev_addr);
2330
2331         if (phy_is_internal(priv->phydev)) {
2332                 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
2333                 reg |= EXT_ENERGY_DET_MASK;
2334                 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
2335         }
2336
2337         /* Disable RX/TX DMA and flush TX queues */
2338         dma_ctrl = bcmgenet_dma_disable(priv);
2339
2340         /* Reinitialize TDMA and RDMA and SW housekeeping */
2341         ret = bcmgenet_init_dma(priv);
2342         if (ret) {
2343                 netdev_err(dev, "failed to initialize DMA\n");
2344                 goto err_fini_dma;
2345         }
2346
2347         /* Always enable ring 16 - descriptor ring */
2348         bcmgenet_enable_dma(priv, dma_ctrl);
2349
2350         ret = request_irq(priv->irq0, bcmgenet_isr0, IRQF_SHARED,
2351                           dev->name, priv);
2352         if (ret < 0) {
2353                 netdev_err(dev, "can't request IRQ %d\n", priv->irq0);
2354                 goto err_fini_dma;
2355         }
2356
2357         ret = request_irq(priv->irq1, bcmgenet_isr1, IRQF_SHARED,
2358                           dev->name, priv);
2359         if (ret < 0) {
2360                 netdev_err(dev, "can't request IRQ %d\n", priv->irq1);
2361                 goto err_irq0;
2362         }
2363
2364         /* Re-configure the port multiplexer towards the PHY device */
2365         bcmgenet_mii_config(priv->dev, false);
2366
2367         phy_connect_direct(dev, priv->phydev, bcmgenet_mii_setup,
2368                            priv->phy_interface);
2369
2370         bcmgenet_netif_start(dev);
2371
2372         return 0;
2373
2374 err_irq0:
2375         free_irq(priv->irq0, dev);
2376 err_fini_dma:
2377         bcmgenet_fini_dma(priv);
2378 err_clk_disable:
2379         if (!IS_ERR(priv->clk))
2380                 clk_disable_unprepare(priv->clk);
2381         return ret;
2382 }
2383
2384 static void bcmgenet_netif_stop(struct net_device *dev)
2385 {
2386         struct bcmgenet_priv *priv = netdev_priv(dev);
2387
2388         netif_tx_stop_all_queues(dev);
2389         napi_disable(&priv->napi);
2390         phy_stop(priv->phydev);
2391
2392         bcmgenet_intr_disable(priv);
2393
2394         /* Wait for pending work items to complete. Since interrupts are
2395          * disabled no new work will be scheduled.
2396          */
2397         cancel_work_sync(&priv->bcmgenet_irq_work);
2398
2399         priv->old_link = -1;
2400         priv->old_speed = -1;
2401         priv->old_duplex = -1;
2402         priv->old_pause = -1;
2403 }
2404
2405 static int bcmgenet_close(struct net_device *dev)
2406 {
2407         struct bcmgenet_priv *priv = netdev_priv(dev);
2408         int ret;
2409
2410         netif_dbg(priv, ifdown, dev, "bcmgenet_close\n");
2411
2412         bcmgenet_netif_stop(dev);
2413
2414         /* Really kill the PHY state machine and disconnect from it */
2415         phy_disconnect(priv->phydev);
2416
2417         /* Disable MAC receive */
2418         umac_enable_set(priv, CMD_RX_EN, false);
2419
2420         ret = bcmgenet_dma_teardown(priv);
2421         if (ret)
2422                 return ret;
2423
2424         /* Disable MAC transmit. TX DMA disabled have to done before this */
2425         umac_enable_set(priv, CMD_TX_EN, false);
2426
2427         /* tx reclaim */
2428         bcmgenet_tx_reclaim_all(dev);
2429         bcmgenet_fini_dma(priv);
2430
2431         free_irq(priv->irq0, priv);
2432         free_irq(priv->irq1, priv);
2433
2434         if (phy_is_internal(priv->phydev))
2435                 bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
2436
2437         if (!IS_ERR(priv->clk))
2438                 clk_disable_unprepare(priv->clk);
2439
2440         return 0;
2441 }
2442
2443 static void bcmgenet_timeout(struct net_device *dev)
2444 {
2445         struct bcmgenet_priv *priv = netdev_priv(dev);
2446
2447         netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n");
2448
2449         dev->trans_start = jiffies;
2450
2451         dev->stats.tx_errors++;
2452
2453         netif_tx_wake_all_queues(dev);
2454 }
2455
2456 #define MAX_MC_COUNT    16
2457
2458 static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv,
2459                                          unsigned char *addr,
2460                                          int *i,
2461                                          int *mc)
2462 {
2463         u32 reg;
2464
2465         bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1],
2466                              UMAC_MDF_ADDR + (*i * 4));
2467         bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 |
2468                              addr[4] << 8 | addr[5],
2469                              UMAC_MDF_ADDR + ((*i + 1) * 4));
2470         reg = bcmgenet_umac_readl(priv, UMAC_MDF_CTRL);
2471         reg |= (1 << (MAX_MC_COUNT - *mc));
2472         bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL);
2473         *i += 2;
2474         (*mc)++;
2475 }
2476
2477 static void bcmgenet_set_rx_mode(struct net_device *dev)
2478 {
2479         struct bcmgenet_priv *priv = netdev_priv(dev);
2480         struct netdev_hw_addr *ha;
2481         int i, mc;
2482         u32 reg;
2483
2484         netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags);
2485
2486         /* Promiscuous mode */
2487         reg = bcmgenet_umac_readl(priv, UMAC_CMD);
2488         if (dev->flags & IFF_PROMISC) {
2489                 reg |= CMD_PROMISC;
2490                 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
2491                 bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL);
2492                 return;
2493         } else {
2494                 reg &= ~CMD_PROMISC;
2495                 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
2496         }
2497
2498         /* UniMac doesn't support ALLMULTI */
2499         if (dev->flags & IFF_ALLMULTI) {
2500                 netdev_warn(dev, "ALLMULTI is not supported\n");
2501                 return;
2502         }
2503
2504         /* update MDF filter */
2505         i = 0;
2506         mc = 0;
2507         /* Broadcast */
2508         bcmgenet_set_mdf_addr(priv, dev->broadcast, &i, &mc);
2509         /* my own address.*/
2510         bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i, &mc);
2511         /* Unicast list*/
2512         if (netdev_uc_count(dev) > (MAX_MC_COUNT - mc))
2513                 return;
2514
2515         if (!netdev_uc_empty(dev))
2516                 netdev_for_each_uc_addr(ha, dev)
2517                         bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc);
2518         /* Multicast */
2519         if (netdev_mc_empty(dev) || netdev_mc_count(dev) >= (MAX_MC_COUNT - mc))
2520                 return;
2521
2522         netdev_for_each_mc_addr(ha, dev)
2523                 bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc);
2524 }
2525
2526 /* Set the hardware MAC address. */
2527 static int bcmgenet_set_mac_addr(struct net_device *dev, void *p)
2528 {
2529         struct sockaddr *addr = p;
2530
2531         /* Setting the MAC address at the hardware level is not possible
2532          * without disabling the UniMAC RX/TX enable bits.
2533          */
2534         if (netif_running(dev))
2535                 return -EBUSY;
2536
2537         ether_addr_copy(dev->dev_addr, addr->sa_data);
2538
2539         return 0;
2540 }
2541
2542 static const struct net_device_ops bcmgenet_netdev_ops = {
2543         .ndo_open               = bcmgenet_open,
2544         .ndo_stop               = bcmgenet_close,
2545         .ndo_start_xmit         = bcmgenet_xmit,
2546         .ndo_tx_timeout         = bcmgenet_timeout,
2547         .ndo_set_rx_mode        = bcmgenet_set_rx_mode,
2548         .ndo_set_mac_address    = bcmgenet_set_mac_addr,
2549         .ndo_do_ioctl           = bcmgenet_ioctl,
2550         .ndo_set_features       = bcmgenet_set_features,
2551 };
2552
2553 /* Array of GENET hardware parameters/characteristics */
2554 static struct bcmgenet_hw_params bcmgenet_hw_params[] = {
2555         [GENET_V1] = {
2556                 .tx_queues = 0,
2557                 .tx_bds_per_q = 0,
2558                 .rx_queues = 0,
2559                 .rx_bds_per_q = 0,
2560                 .bp_in_en_shift = 16,
2561                 .bp_in_mask = 0xffff,
2562                 .hfb_filter_cnt = 16,
2563                 .qtag_mask = 0x1F,
2564                 .hfb_offset = 0x1000,
2565                 .rdma_offset = 0x2000,
2566                 .tdma_offset = 0x3000,
2567                 .words_per_bd = 2,
2568         },
2569         [GENET_V2] = {
2570                 .tx_queues = 4,
2571                 .tx_bds_per_q = 32,
2572                 .rx_queues = 0,
2573                 .rx_bds_per_q = 0,
2574                 .bp_in_en_shift = 16,
2575                 .bp_in_mask = 0xffff,
2576                 .hfb_filter_cnt = 16,
2577                 .qtag_mask = 0x1F,
2578                 .tbuf_offset = 0x0600,
2579                 .hfb_offset = 0x1000,
2580                 .hfb_reg_offset = 0x2000,
2581                 .rdma_offset = 0x3000,
2582                 .tdma_offset = 0x4000,
2583                 .words_per_bd = 2,
2584                 .flags = GENET_HAS_EXT,
2585         },
2586         [GENET_V3] = {
2587                 .tx_queues = 4,
2588                 .tx_bds_per_q = 32,
2589                 .rx_queues = 0,
2590                 .rx_bds_per_q = 0,
2591                 .bp_in_en_shift = 17,
2592                 .bp_in_mask = 0x1ffff,
2593                 .hfb_filter_cnt = 48,
2594                 .qtag_mask = 0x3F,
2595                 .tbuf_offset = 0x0600,
2596                 .hfb_offset = 0x8000,
2597                 .hfb_reg_offset = 0xfc00,
2598                 .rdma_offset = 0x10000,
2599                 .tdma_offset = 0x11000,
2600                 .words_per_bd = 2,
2601                 .flags = GENET_HAS_EXT | GENET_HAS_MDIO_INTR,
2602         },
2603         [GENET_V4] = {
2604                 .tx_queues = 4,
2605                 .tx_bds_per_q = 32,
2606                 .rx_queues = 0,
2607                 .rx_bds_per_q = 0,
2608                 .bp_in_en_shift = 17,
2609                 .bp_in_mask = 0x1ffff,
2610                 .hfb_filter_cnt = 48,
2611                 .qtag_mask = 0x3F,
2612                 .tbuf_offset = 0x0600,
2613                 .hfb_offset = 0x8000,
2614                 .hfb_reg_offset = 0xfc00,
2615                 .rdma_offset = 0x2000,
2616                 .tdma_offset = 0x4000,
2617                 .words_per_bd = 3,
2618                 .flags = GENET_HAS_40BITS | GENET_HAS_EXT | GENET_HAS_MDIO_INTR,
2619         },
2620 };
2621
2622 /* Infer hardware parameters from the detected GENET version */
2623 static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv)
2624 {
2625         struct bcmgenet_hw_params *params;
2626         u32 reg;
2627         u8 major;
2628         u16 gphy_rev;
2629
2630         if (GENET_IS_V4(priv)) {
2631                 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
2632                 genet_dma_ring_regs = genet_dma_ring_regs_v4;
2633                 priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
2634                 priv->version = GENET_V4;
2635         } else if (GENET_IS_V3(priv)) {
2636                 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
2637                 genet_dma_ring_regs = genet_dma_ring_regs_v123;
2638                 priv->dma_rx_chk_bit = DMA_RX_CHK_V3PLUS;
2639                 priv->version = GENET_V3;
2640         } else if (GENET_IS_V2(priv)) {
2641                 bcmgenet_dma_regs = bcmgenet_dma_regs_v2;
2642                 genet_dma_ring_regs = genet_dma_ring_regs_v123;
2643                 priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
2644                 priv->version = GENET_V2;
2645         } else if (GENET_IS_V1(priv)) {
2646                 bcmgenet_dma_regs = bcmgenet_dma_regs_v1;
2647                 genet_dma_ring_regs = genet_dma_ring_regs_v123;
2648                 priv->dma_rx_chk_bit = DMA_RX_CHK_V12;
2649                 priv->version = GENET_V1;
2650         }
2651
2652         /* enum genet_version starts at 1 */
2653         priv->hw_params = &bcmgenet_hw_params[priv->version];
2654         params = priv->hw_params;
2655
2656         /* Read GENET HW version */
2657         reg = bcmgenet_sys_readl(priv, SYS_REV_CTRL);
2658         major = (reg >> 24 & 0x0f);
2659         if (major == 5)
2660                 major = 4;
2661         else if (major == 0)
2662                 major = 1;
2663         if (major != priv->version) {
2664                 dev_err(&priv->pdev->dev,
2665                         "GENET version mismatch, got: %d, configured for: %d\n",
2666                         major, priv->version);
2667         }
2668
2669         /* Print the GENET core version */
2670         dev_info(&priv->pdev->dev, "GENET " GENET_VER_FMT,
2671                  major, (reg >> 16) & 0x0f, reg & 0xffff);
2672
2673         /* Store the integrated PHY revision for the MDIO probing function
2674          * to pass this information to the PHY driver. The PHY driver expects
2675          * to find the PHY major revision in bits 15:8 while the GENET register
2676          * stores that information in bits 7:0, account for that.
2677          *
2678          * On newer chips, starting with PHY revision G0, a new scheme is
2679          * deployed similar to the Starfighter 2 switch with GPHY major
2680          * revision in bits 15:8 and patch level in bits 7:0. Major revision 0
2681          * is reserved as well as special value 0x01ff, we have a small
2682          * heuristic to check for the new GPHY revision and re-arrange things
2683          * so the GPHY driver is happy.
2684          */
2685         gphy_rev = reg & 0xffff;
2686
2687         /* This is the good old scheme, just GPHY major, no minor nor patch */
2688         if ((gphy_rev & 0xf0) != 0)
2689                 priv->gphy_rev = gphy_rev << 8;
2690
2691         /* This is the new scheme, GPHY major rolls over with 0x10 = rev G0 */
2692         else if ((gphy_rev & 0xff00) != 0)
2693                 priv->gphy_rev = gphy_rev;
2694
2695         /* This is reserved so should require special treatment */
2696         else if (gphy_rev == 0 || gphy_rev == 0x01ff) {
2697                 pr_warn("Invalid GPHY revision detected: 0x%04x\n", gphy_rev);
2698                 return;
2699         }
2700
2701 #ifdef CONFIG_PHYS_ADDR_T_64BIT
2702         if (!(params->flags & GENET_HAS_40BITS))
2703                 pr_warn("GENET does not support 40-bits PA\n");
2704 #endif
2705
2706         pr_debug("Configuration for version: %d\n"
2707                 "TXq: %1d, TXqBDs: %1d, RXq: %1d, RXqBDs: %1d\n"
2708                 "BP << en: %2d, BP msk: 0x%05x\n"
2709                 "HFB count: %2d, QTAQ msk: 0x%05x\n"
2710                 "TBUF: 0x%04x, HFB: 0x%04x, HFBreg: 0x%04x\n"
2711                 "RDMA: 0x%05x, TDMA: 0x%05x\n"
2712                 "Words/BD: %d\n",
2713                 priv->version,
2714                 params->tx_queues, params->tx_bds_per_q,
2715                 params->rx_queues, params->rx_bds_per_q,
2716                 params->bp_in_en_shift, params->bp_in_mask,
2717                 params->hfb_filter_cnt, params->qtag_mask,
2718                 params->tbuf_offset, params->hfb_offset,
2719                 params->hfb_reg_offset,
2720                 params->rdma_offset, params->tdma_offset,
2721                 params->words_per_bd);
2722 }
2723
2724 static const struct of_device_id bcmgenet_match[] = {
2725         { .compatible = "brcm,genet-v1", .data = (void *)GENET_V1 },
2726         { .compatible = "brcm,genet-v2", .data = (void *)GENET_V2 },
2727         { .compatible = "brcm,genet-v3", .data = (void *)GENET_V3 },
2728         { .compatible = "brcm,genet-v4", .data = (void *)GENET_V4 },
2729         { },
2730 };
2731
2732 static int bcmgenet_probe(struct platform_device *pdev)
2733 {
2734         struct bcmgenet_platform_data *pd = pdev->dev.platform_data;
2735         struct device_node *dn = pdev->dev.of_node;
2736         const struct of_device_id *of_id = NULL;
2737         struct bcmgenet_priv *priv;
2738         struct net_device *dev;
2739         const void *macaddr;
2740         struct resource *r;
2741         int err = -EIO;
2742
2743         /* Up to GENET_MAX_MQ_CNT + 1 TX queues and RX queues */
2744         dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1,
2745                                  GENET_MAX_MQ_CNT + 1);
2746         if (!dev) {
2747                 dev_err(&pdev->dev, "can't allocate net device\n");
2748                 return -ENOMEM;
2749         }
2750
2751         if (dn) {
2752                 of_id = of_match_node(bcmgenet_match, dn);
2753                 if (!of_id)
2754                         return -EINVAL;
2755         }
2756
2757         priv = netdev_priv(dev);
2758         priv->irq0 = platform_get_irq(pdev, 0);
2759         priv->irq1 = platform_get_irq(pdev, 1);
2760         priv->wol_irq = platform_get_irq(pdev, 2);
2761         if (!priv->irq0 || !priv->irq1) {
2762                 dev_err(&pdev->dev, "can't find IRQs\n");
2763                 err = -EINVAL;
2764                 goto err;
2765         }
2766
2767         if (dn) {
2768                 macaddr = of_get_mac_address(dn);
2769                 if (!macaddr) {
2770                         dev_err(&pdev->dev, "can't find MAC address\n");
2771                         err = -EINVAL;
2772                         goto err;
2773                 }
2774         } else {
2775                 macaddr = pd->mac_address;
2776         }
2777
2778         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2779         priv->base = devm_ioremap_resource(&pdev->dev, r);
2780         if (IS_ERR(priv->base)) {
2781                 err = PTR_ERR(priv->base);
2782                 goto err;
2783         }
2784
2785         SET_NETDEV_DEV(dev, &pdev->dev);
2786         dev_set_drvdata(&pdev->dev, dev);
2787         ether_addr_copy(dev->dev_addr, macaddr);
2788         dev->watchdog_timeo = 2 * HZ;
2789         dev->ethtool_ops = &bcmgenet_ethtool_ops;
2790         dev->netdev_ops = &bcmgenet_netdev_ops;
2791         netif_napi_add(dev, &priv->napi, bcmgenet_poll, 64);
2792
2793         priv->msg_enable = netif_msg_init(-1, GENET_MSG_DEFAULT);
2794
2795         /* Set hardware features */
2796         dev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM |
2797                 NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;
2798
2799         /* Request the WOL interrupt and advertise suspend if available */
2800         priv->wol_irq_disabled = true;
2801         err = devm_request_irq(&pdev->dev, priv->wol_irq, bcmgenet_wol_isr, 0,
2802                                dev->name, priv);
2803         if (!err)
2804                 device_set_wakeup_capable(&pdev->dev, 1);
2805
2806         /* Set the needed headroom to account for any possible
2807          * features enabling/disabling at runtime
2808          */
2809         dev->needed_headroom += 64;
2810
2811         netdev_boot_setup_check(dev);
2812
2813         priv->dev = dev;
2814         priv->pdev = pdev;
2815         if (of_id)
2816                 priv->version = (enum bcmgenet_version)of_id->data;
2817         else
2818                 priv->version = pd->genet_version;
2819
2820         priv->clk = devm_clk_get(&priv->pdev->dev, "enet");
2821         if (IS_ERR(priv->clk))
2822                 dev_warn(&priv->pdev->dev, "failed to get enet clock\n");
2823
2824         if (!IS_ERR(priv->clk))
2825                 clk_prepare_enable(priv->clk);
2826
2827         bcmgenet_set_hw_params(priv);
2828
2829         /* Mii wait queue */
2830         init_waitqueue_head(&priv->wq);
2831         /* Always use RX_BUF_LENGTH (2KB) buffer for all chips */
2832         priv->rx_buf_len = RX_BUF_LENGTH;
2833         INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task);
2834
2835         priv->clk_wol = devm_clk_get(&priv->pdev->dev, "enet-wol");
2836         if (IS_ERR(priv->clk_wol))
2837                 dev_warn(&priv->pdev->dev, "failed to get enet-wol clock\n");
2838
2839         priv->clk_eee = devm_clk_get(&priv->pdev->dev, "enet-eee");
2840         if (IS_ERR(priv->clk_eee)) {
2841                 dev_warn(&priv->pdev->dev, "failed to get enet-eee clock\n");
2842                 priv->clk_eee = NULL;
2843         }
2844
2845         err = reset_umac(priv);
2846         if (err)
2847                 goto err_clk_disable;
2848
2849         err = bcmgenet_mii_init(dev);
2850         if (err)
2851                 goto err_clk_disable;
2852
2853         /* setup number of real queues  + 1 (GENET_V1 has 0 hardware queues
2854          * just the ring 16 descriptor based TX
2855          */
2856         netif_set_real_num_tx_queues(priv->dev, priv->hw_params->tx_queues + 1);
2857         netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1);
2858
2859         /* libphy will determine the link state */
2860         netif_carrier_off(dev);
2861
2862         /* Turn off the main clock, WOL clock is handled separately */
2863         if (!IS_ERR(priv->clk))
2864                 clk_disable_unprepare(priv->clk);
2865
2866         err = register_netdev(dev);
2867         if (err)
2868                 goto err;
2869
2870         return err;
2871
2872 err_clk_disable:
2873         if (!IS_ERR(priv->clk))
2874                 clk_disable_unprepare(priv->clk);
2875 err:
2876         free_netdev(dev);
2877         return err;
2878 }
2879
2880 static int bcmgenet_remove(struct platform_device *pdev)
2881 {
2882         struct bcmgenet_priv *priv = dev_to_priv(&pdev->dev);
2883
2884         dev_set_drvdata(&pdev->dev, NULL);
2885         unregister_netdev(priv->dev);
2886         bcmgenet_mii_exit(priv->dev);
2887         free_netdev(priv->dev);
2888
2889         return 0;
2890 }
2891
2892 #ifdef CONFIG_PM_SLEEP
2893 static int bcmgenet_suspend(struct device *d)
2894 {
2895         struct net_device *dev = dev_get_drvdata(d);
2896         struct bcmgenet_priv *priv = netdev_priv(dev);
2897         int ret;
2898
2899         if (!netif_running(dev))
2900                 return 0;
2901
2902         bcmgenet_netif_stop(dev);
2903
2904         phy_suspend(priv->phydev);
2905
2906         netif_device_detach(dev);
2907
2908         /* Disable MAC receive */
2909         umac_enable_set(priv, CMD_RX_EN, false);
2910
2911         ret = bcmgenet_dma_teardown(priv);
2912         if (ret)
2913                 return ret;
2914
2915         /* Disable MAC transmit. TX DMA disabled have to done before this */
2916         umac_enable_set(priv, CMD_TX_EN, false);
2917
2918         /* tx reclaim */
2919         bcmgenet_tx_reclaim_all(dev);
2920         bcmgenet_fini_dma(priv);
2921
2922         /* Prepare the device for Wake-on-LAN and switch to the slow clock */
2923         if (device_may_wakeup(d) && priv->wolopts) {
2924                 bcmgenet_power_down(priv, GENET_POWER_WOL_MAGIC);
2925                 clk_prepare_enable(priv->clk_wol);
2926         }
2927
2928         /* Turn off the clocks */
2929         clk_disable_unprepare(priv->clk);
2930
2931         return 0;
2932 }
2933
2934 static int bcmgenet_resume(struct device *d)
2935 {
2936         struct net_device *dev = dev_get_drvdata(d);
2937         struct bcmgenet_priv *priv = netdev_priv(dev);
2938         unsigned long dma_ctrl;
2939         int ret;
2940         u32 reg;
2941
2942         if (!netif_running(dev))
2943                 return 0;
2944
2945         /* Turn on the clock */
2946         ret = clk_prepare_enable(priv->clk);
2947         if (ret)
2948                 return ret;
2949
2950         bcmgenet_umac_reset(priv);
2951
2952         ret = init_umac(priv);
2953         if (ret)
2954                 goto out_clk_disable;
2955
2956         /* From WOL-enabled suspend, switch to regular clock */
2957         if (priv->wolopts)
2958                 clk_disable_unprepare(priv->clk_wol);
2959
2960         phy_init_hw(priv->phydev);
2961         /* Speed settings must be restored */
2962         bcmgenet_mii_config(priv->dev, false);
2963
2964         /* disable ethernet MAC while updating its registers */
2965         umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, false);
2966
2967         bcmgenet_set_hw_addr(priv, dev->dev_addr);
2968
2969         if (phy_is_internal(priv->phydev)) {
2970                 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
2971                 reg |= EXT_ENERGY_DET_MASK;
2972                 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
2973         }
2974
2975         if (priv->wolopts)
2976                 bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC);
2977
2978         /* Disable RX/TX DMA and flush TX queues */
2979         dma_ctrl = bcmgenet_dma_disable(priv);
2980
2981         /* Reinitialize TDMA and RDMA and SW housekeeping */
2982         ret = bcmgenet_init_dma(priv);
2983         if (ret) {
2984                 netdev_err(dev, "failed to initialize DMA\n");
2985                 goto out_clk_disable;
2986         }
2987
2988         /* Always enable ring 16 - descriptor ring */
2989         bcmgenet_enable_dma(priv, dma_ctrl);
2990
2991         netif_device_attach(dev);
2992
2993         phy_resume(priv->phydev);
2994
2995         if (priv->eee.eee_enabled)
2996                 bcmgenet_eee_enable_set(dev, true);
2997
2998         bcmgenet_netif_start(dev);
2999
3000         return 0;
3001
3002 out_clk_disable:
3003         clk_disable_unprepare(priv->clk);
3004         return ret;
3005 }
3006 #endif /* CONFIG_PM_SLEEP */
3007
3008 static SIMPLE_DEV_PM_OPS(bcmgenet_pm_ops, bcmgenet_suspend, bcmgenet_resume);
3009
3010 static struct platform_driver bcmgenet_driver = {
3011         .probe  = bcmgenet_probe,
3012         .remove = bcmgenet_remove,
3013         .driver = {
3014                 .name   = "bcmgenet",
3015                 .of_match_table = bcmgenet_match,
3016                 .pm     = &bcmgenet_pm_ops,
3017         },
3018 };
3019 module_platform_driver(bcmgenet_driver);
3020
3021 MODULE_AUTHOR("Broadcom Corporation");
3022 MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver");
3023 MODULE_ALIAS("platform:bcmgenet");
3024 MODULE_LICENSE("GPL");