drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /* Copyright 2014-2016 Freescale Semiconductor Inc.
   3  * Copyright 2016-2017 NXP
   4  */
   5 #include <linux/init.h>
   6 #include <linux/module.h>
   7 #include <linux/platform_device.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/of_net.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/msi.h>
  12 #include <linux/kthread.h>
  13 #include <linux/iommu.h>
  14 #include <linux/net_tstamp.h>
  15 #include <linux/fsl/mc.h>
  16 #include <linux/bpf.h>
  17 #include <linux/bpf_trace.h>
  18 #include <net/sock.h>
  19
  20 #include "dpaa2-eth.h"
  21
  22 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files
  23  * using trace events only need to #include <trace/events/sched.h>
  24  */
  25 #define CREATE_TRACE_POINTS
  26 #include "dpaa2-eth-trace.h"
  27
  28 MODULE_LICENSE("Dual BSD/GPL");
  29 MODULE_AUTHOR("Freescale Semiconductor, Inc");
  30 MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver");
  31
  32 static void *dpaa2_iova_to_virt(struct iommu_domain *domain,
  33                                 dma_addr_t iova_addr)
  34 {
  35         phys_addr_t phys_addr;
  36
  37         phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
  38
  39         return phys_to_virt(phys_addr);
  40 }
  41
  42 static void validate_rx_csum(struct dpaa2_eth_priv *priv,
  43                              u32 fd_status,
  44                              struct sk_buff *skb)
  45 {
  46         skb_checksum_none_assert(skb);
  47
  48         /* HW checksum validation is disabled, nothing to do here */
  49         if (!(priv->net_dev->features & NETIF_F_RXCSUM))
  50                 return;
  51
  52         /* Read checksum validation bits */
  53         if (!((fd_status & DPAA2_FAS_L3CV) &&
  54               (fd_status & DPAA2_FAS_L4CV)))
  55                 return;
  56
  57         /* Inform the stack there's no need to compute L3/L4 csum anymore */
  58         skb->ip_summed = CHECKSUM_UNNECESSARY;
  59 }
  60
  61 /* Free a received FD.
  62  * Not to be used for Tx conf FDs or on any other paths.
  63  */
  64 static void free_rx_fd(struct dpaa2_eth_priv *priv,
  65                        const struct dpaa2_fd *fd,
  66                        void *vaddr)
  67 {
  68         struct device *dev = priv->net_dev->dev.parent;
  69         dma_addr_t addr = dpaa2_fd_get_addr(fd);
  70         u8 fd_format = dpaa2_fd_get_format(fd);
  71         struct dpaa2_sg_entry *sgt;
  72         void *sg_vaddr;
  73         int i;
  74
  75         /* If single buffer frame, just free the data buffer */
  76         if (fd_format == dpaa2_fd_single)
  77                 goto free_buf;
  78         else if (fd_format != dpaa2_fd_sg)
  79                 /* We don't support any other format */
  80                 return;
  81
  82         /* For S/G frames, we first need to free all SG entries
  83          * except the first one, which was taken care of already
  84          */
  85         sgt = vaddr + dpaa2_fd_get_offset(fd);
  86         for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
  87                 addr = dpaa2_sg_get_addr(&sgt[i]);
  88                 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
  89                 dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
  90                                DMA_BIDIRECTIONAL);
  91
  92                 free_pages((unsigned long)sg_vaddr, 0);
  93                 if (dpaa2_sg_is_final(&sgt[i]))
  94                         break;
  95         }
  96
  97 free_buf:
  98         free_pages((unsigned long)vaddr, 0);
  99 }
 100
 101 /* Build a linear skb based on a single-buffer frame descriptor */
 102 static struct sk_buff *build_linear_skb(struct dpaa2_eth_channel *ch,
 103                                         const struct dpaa2_fd *fd,
 104                                         void *fd_vaddr)
 105 {
 106         struct sk_buff *skb = NULL;
 107         u16 fd_offset = dpaa2_fd_get_offset(fd);
 108         u32 fd_length = dpaa2_fd_get_len(fd);
 109
 110         ch->buf_count--;
 111
 112         skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE);
 113         if (unlikely(!skb))
 114                 return NULL;
 115
 116         skb_reserve(skb, fd_offset);
 117         skb_put(skb, fd_length);
 118
 119         return skb;
 120 }
 121
 122 /* Build a non linear (fragmented) skb based on a S/G table */
 123 static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv,
 124                                       struct dpaa2_eth_channel *ch,
 125                                       struct dpaa2_sg_entry *sgt)
 126 {
 127         struct sk_buff *skb = NULL;
 128         struct device *dev = priv->net_dev->dev.parent;
 129         void *sg_vaddr;
 130         dma_addr_t sg_addr;
 131         u16 sg_offset;
 132         u32 sg_length;
 133         struct page *page, *head_page;
 134         int page_offset;
 135         int i;
 136
 137         for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
 138                 struct dpaa2_sg_entry *sge = &sgt[i];
 139
 140                 /* NOTE: We only support SG entries in dpaa2_sg_single format,
 141                  * but this is the only format we may receive from HW anyway
 142                  */
 143
 144                 /* Get the address and length from the S/G entry */
 145                 sg_addr = dpaa2_sg_get_addr(sge);
 146                 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr);
 147                 dma_unmap_page(dev, sg_addr, DPAA2_ETH_RX_BUF_SIZE,
 148                                DMA_BIDIRECTIONAL);
 149
 150                 sg_length = dpaa2_sg_get_len(sge);
 151
 152                 if (i == 0) {
 153                         /* We build the skb around the first data buffer */
 154                         skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE);
 155                         if (unlikely(!skb)) {
 156                                 /* Free the first SG entry now, since we already
 157                                  * unmapped it and obtained the virtual address
 158                                  */
 159                                 free_pages((unsigned long)sg_vaddr, 0);
 160
 161                                 /* We still need to subtract the buffers used
 162                                  * by this FD from our software counter
 163                                  */
 164                                 while (!dpaa2_sg_is_final(&sgt[i]) &&
 165                                        i < DPAA2_ETH_MAX_SG_ENTRIES)
 166                                         i++;
 167                                 break;
 168                         }
 169
 170                         sg_offset = dpaa2_sg_get_offset(sge);
 171                         skb_reserve(skb, sg_offset);
 172                         skb_put(skb, sg_length);
 173                 } else {
 174                         /* Rest of the data buffers are stored as skb frags */
 175                         page = virt_to_page(sg_vaddr);
 176                         head_page = virt_to_head_page(sg_vaddr);
 177
 178                         /* Offset in page (which may be compound).
 179                          * Data in subsequent SG entries is stored from the
 180                          * beginning of the buffer, so we don't need to add the
 181                          * sg_offset.
 182                          */
 183                         page_offset = ((unsigned long)sg_vaddr &
 184                                 (PAGE_SIZE - 1)) +
 185                                 (page_address(page) - page_address(head_page));
 186
 187                         skb_add_rx_frag(skb, i - 1, head_page, page_offset,
 188                                         sg_length, DPAA2_ETH_RX_BUF_SIZE);
 189                 }
 190
 191                 if (dpaa2_sg_is_final(sge))
 192                         break;
 193         }
 194
 195         WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT");
 196
 197         /* Count all data buffers + SG table buffer */
 198         ch->buf_count -= i + 2;
 199
 200         return skb;
 201 }
 202
 203 /* Free buffers acquired from the buffer pool or which were meant to
 204  * be released in the pool
 205  */
 206 static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count)
 207 {
 208         struct device *dev = priv->net_dev->dev.parent;
 209         void *vaddr;
 210         int i;
 211
 212         for (i = 0; i < count; i++) {
 213                 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]);
 214                 dma_unmap_page(dev, buf_array[i], DPAA2_ETH_RX_BUF_SIZE,
 215                                DMA_BIDIRECTIONAL);
 216                 free_pages((unsigned long)vaddr, 0);
 217         }
 218 }
 219
 220 static void xdp_release_buf(struct dpaa2_eth_priv *priv,
 221                             struct dpaa2_eth_channel *ch,
 222                             dma_addr_t addr)
 223 {
 224         int err;
 225
 226         ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr;
 227         if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD)
 228                 return;
 229
 230         while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid,
 231                                                ch->xdp.drop_bufs,
 232                                                ch->xdp.drop_cnt)) == -EBUSY)
 233                 cpu_relax();
 234
 235         if (err) {
 236                 free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt);
 237                 ch->buf_count -= ch->xdp.drop_cnt;
 238         }
 239
 240         ch->xdp.drop_cnt = 0;
 241 }
 242
 243 static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,
 244                        void *buf_start, u16 queue_id)
 245 {
 246         struct dpaa2_eth_fq *fq;
 247         struct dpaa2_faead *faead;
 248         u32 ctrl, frc;
 249         int i, err;
 250
 251         /* Mark the egress frame hardware annotation area as valid */
 252         frc = dpaa2_fd_get_frc(fd);
 253         dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);
 254         dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL);
 255
 256         /* Instruct hardware to release the FD buffer directly into
 257          * the buffer pool once transmission is completed, instead of
 258          * sending a Tx confirmation frame to us
 259          */
 260         ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV;
 261         faead = dpaa2_get_faead(buf_start, false);
 262         faead->ctrl = cpu_to_le32(ctrl);
 263         faead->conf_fqid = 0;
 264
 265         fq = &priv->fq[queue_id];
 266         for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
 267                 err = priv->enqueue(priv, fq, fd, 0);
 268                 if (err != -EBUSY)
 269                         break;
 270         }
 271
 272         return err;
 273 }
 274
 275 static u32 run_xdp(struct dpaa2_eth_priv *priv,
 276                    struct dpaa2_eth_channel *ch,
 277                    struct dpaa2_eth_fq *rx_fq,
 278                    struct dpaa2_fd *fd, void *vaddr)
 279 {
 280         dma_addr_t addr = dpaa2_fd_get_addr(fd);
 281         struct rtnl_link_stats64 *percpu_stats;
 282         struct bpf_prog *xdp_prog;
 283         struct xdp_buff xdp;
 284         u32 xdp_act = XDP_PASS;
 285         int err;
 286
 287         percpu_stats = this_cpu_ptr(priv->percpu_stats);
 288
 289         rcu_read_lock();
 290
 291         xdp_prog = READ_ONCE(ch->xdp.prog);
 292         if (!xdp_prog)
 293                 goto out;
 294
 295         xdp.data = vaddr + dpaa2_fd_get_offset(fd);
 296         xdp.data_end = xdp.data + dpaa2_fd_get_len(fd);
 297         xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
 298         xdp_set_data_meta_invalid(&xdp);
 299         xdp.rxq = &ch->xdp_rxq;
 300
 301         xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp);
 302
 303         /* xdp.data pointer may have changed */
 304         dpaa2_fd_set_offset(fd, xdp.data - vaddr);
 305         dpaa2_fd_set_len(fd, xdp.data_end - xdp.data);
 306
 307         switch (xdp_act) {
 308         case XDP_PASS:
 309                 break;
 310         case XDP_TX:
 311                 err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid);
 312                 if (err) {
 313                         xdp_release_buf(priv, ch, addr);
 314                         percpu_stats->tx_errors++;
 315                         ch->stats.xdp_tx_err++;
 316                 } else {
 317                         percpu_stats->tx_packets++;
 318                         percpu_stats->tx_bytes += dpaa2_fd_get_len(fd);
 319                         ch->stats.xdp_tx++;
 320                 }
 321                 break;
 322         default:
 323                 bpf_warn_invalid_xdp_action(xdp_act);
 324                 /* fall through */
 325         case XDP_ABORTED:
 326                 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
 327                 /* fall through */
 328         case XDP_DROP:
 329                 xdp_release_buf(priv, ch, addr);
 330                 ch->stats.xdp_drop++;
 331                 break;
 332         case XDP_REDIRECT:
 333                 dma_unmap_page(priv->net_dev->dev.parent, addr,
 334                                DPAA2_ETH_RX_BUF_SIZE, DMA_BIDIRECTIONAL);
 335                 ch->buf_count--;
 336                 xdp.data_hard_start = vaddr;
 337                 err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog);
 338                 if (unlikely(err))
 339                         ch->stats.xdp_drop++;
 340                 else
 341                         ch->stats.xdp_redirect++;
 342                 break;
 343         }
 344
 345         ch->xdp.res |= xdp_act;
 346 out:
 347         rcu_read_unlock();
 348         return xdp_act;
 349 }
 350
 351 /* Main Rx frame processing routine */
 352 static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv,
 353                          struct dpaa2_eth_channel *ch,
 354                          const struct dpaa2_fd *fd,
 355                          struct dpaa2_eth_fq *fq)
 356 {
 357         dma_addr_t addr = dpaa2_fd_get_addr(fd);
 358         u8 fd_format = dpaa2_fd_get_format(fd);
 359         void *vaddr;
 360         struct sk_buff *skb;
 361         struct rtnl_link_stats64 *percpu_stats;
 362         struct dpaa2_eth_drv_stats *percpu_extras;
 363         struct device *dev = priv->net_dev->dev.parent;
 364         struct dpaa2_fas *fas;
 365         void *buf_data;
 366         u32 status = 0;
 367         u32 xdp_act;
 368
 369         /* Tracing point */
 370         trace_dpaa2_rx_fd(priv->net_dev, fd);
 371
 372         vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
 373         dma_sync_single_for_cpu(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
 374                                 DMA_BIDIRECTIONAL);
 375
 376         fas = dpaa2_get_fas(vaddr, false);
 377         prefetch(fas);
 378         buf_data = vaddr + dpaa2_fd_get_offset(fd);
 379         prefetch(buf_data);
 380
 381         percpu_stats = this_cpu_ptr(priv->percpu_stats);
 382         percpu_extras = this_cpu_ptr(priv->percpu_extras);
 383
 384         if (fd_format == dpaa2_fd_single) {
 385                 xdp_act = run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr);
 386                 if (xdp_act != XDP_PASS) {
 387                         percpu_stats->rx_packets++;
 388                         percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
 389                         return;
 390                 }
 391
 392                 dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
 393                                DMA_BIDIRECTIONAL);
 394                 skb = build_linear_skb(ch, fd, vaddr);
 395         } else if (fd_format == dpaa2_fd_sg) {
 396                 WARN_ON(priv->xdp_prog);
 397
 398                 dma_unmap_page(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
 399                                DMA_BIDIRECTIONAL);
 400                 skb = build_frag_skb(priv, ch, buf_data);
 401                 free_pages((unsigned long)vaddr, 0);
 402                 percpu_extras->rx_sg_frames++;
 403                 percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd);
 404         } else {
 405                 /* We don't support any other format */
 406                 goto err_frame_format;
 407         }
 408
 409         if (unlikely(!skb))
 410                 goto err_build_skb;
 411
 412         prefetch(skb->data);
 413
 414         /* Get the timestamp value */
 415         if (priv->rx_tstamp) {
 416                 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
 417                 __le64 *ts = dpaa2_get_ts(vaddr, false);
 418                 u64 ns;
 419
 420                 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 421
 422                 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
 423                 shhwtstamps->hwtstamp = ns_to_ktime(ns);
 424         }
 425
 426         /* Check if we need to validate the L4 csum */
 427         if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) {
 428                 status = le32_to_cpu(fas->status);
 429                 validate_rx_csum(priv, status, skb);
 430         }
 431
 432         skb->protocol = eth_type_trans(skb, priv->net_dev);
 433         skb_record_rx_queue(skb, fq->flowid);
 434
 435         percpu_stats->rx_packets++;
 436         percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
 437
 438         napi_gro_receive(&ch->napi, skb);
 439
 440         return;
 441
 442 err_build_skb:
 443         free_rx_fd(priv, fd, vaddr);
 444 err_frame_format:
 445         percpu_stats->rx_dropped++;
 446 }
 447
 448 /* Consume all frames pull-dequeued into the store. This is the simplest way to
 449  * make sure we don't accidentally issue another volatile dequeue which would
 450  * overwrite (leak) frames already in the store.
 451  *
 452  * Observance of NAPI budget is not our concern, leaving that to the caller.
 453  */
 454 static int consume_frames(struct dpaa2_eth_channel *ch,
 455                           struct dpaa2_eth_fq **src)
 456 {
 457         struct dpaa2_eth_priv *priv = ch->priv;
 458         struct dpaa2_eth_fq *fq = NULL;
 459         struct dpaa2_dq *dq;
 460         const struct dpaa2_fd *fd;
 461         int cleaned = 0;
 462         int is_last;
 463
 464         do {
 465                 dq = dpaa2_io_store_next(ch->store, &is_last);
 466                 if (unlikely(!dq)) {
 467                         /* If we're here, we *must* have placed a
 468                          * volatile dequeue comnmand, so keep reading through
 469                          * the store until we get some sort of valid response
 470                          * token (either a valid frame or an "empty dequeue")
 471                          */
 472                         continue;
 473                 }
 474
 475                 fd = dpaa2_dq_fd(dq);
 476                 fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq);
 477
 478                 fq->consume(priv, ch, fd, fq);
 479                 cleaned++;
 480         } while (!is_last);
 481
 482         if (!cleaned)
 483                 return 0;
 484
 485         fq->stats.frames += cleaned;
 486
 487         /* A dequeue operation only pulls frames from a single queue
 488          * into the store. Return the frame queue as an out param.
 489          */
 490         if (src)
 491                 *src = fq;
 492
 493         return cleaned;
 494 }
 495
 496 /* Configure the egress frame annotation for timestamp update */
 497 static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start)
 498 {
 499         struct dpaa2_faead *faead;
 500         u32 ctrl, frc;
 501
 502         /* Mark the egress frame annotation area as valid */
 503         frc = dpaa2_fd_get_frc(fd);
 504         dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);
 505
 506         /* Set hardware annotation size */
 507         ctrl = dpaa2_fd_get_ctrl(fd);
 508         dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL);
 509
 510         /* enable UPD (update prepanded data) bit in FAEAD field of
 511          * hardware frame annotation area
 512          */
 513         ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD;
 514         faead = dpaa2_get_faead(buf_start, true);
 515         faead->ctrl = cpu_to_le32(ctrl);
 516 }
 517
 518 /* Create a frame descriptor based on a fragmented skb */
 519 static int build_sg_fd(struct dpaa2_eth_priv *priv,
 520                        struct sk_buff *skb,
 521                        struct dpaa2_fd *fd)
 522 {
 523         struct device *dev = priv->net_dev->dev.parent;
 524         void *sgt_buf = NULL;
 525         dma_addr_t addr;
 526         int nr_frags = skb_shinfo(skb)->nr_frags;
 527         struct dpaa2_sg_entry *sgt;
 528         int i, err;
 529         int sgt_buf_size;
 530         struct scatterlist *scl, *crt_scl;
 531         int num_sg;
 532         int num_dma_bufs;
 533         struct dpaa2_eth_swa *swa;
 534
 535         /* Create and map scatterlist.
 536          * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have
 537          * to go beyond nr_frags+1.
 538          * Note: We don't support chained scatterlists
 539          */
 540         if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1))
 541                 return -EINVAL;
 542
 543         scl = kcalloc(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC);
 544         if (unlikely(!scl))
 545                 return -ENOMEM;
 546
 547         sg_init_table(scl, nr_frags + 1);
 548         num_sg = skb_to_sgvec(skb, scl, 0, skb->len);
 549         num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
 550         if (unlikely(!num_dma_bufs)) {
 551                 err = -ENOMEM;
 552                 goto dma_map_sg_failed;
 553         }
 554
 555         /* Prepare the HW SGT structure */
 556         sgt_buf_size = priv->tx_data_offset +
 557                        sizeof(struct dpaa2_sg_entry) *  num_dma_bufs;
 558         sgt_buf = netdev_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN);
 559         if (unlikely(!sgt_buf)) {
 560                 err = -ENOMEM;
 561                 goto sgt_buf_alloc_failed;
 562         }
 563         sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN);
 564         memset(sgt_buf, 0, sgt_buf_size);
 565
 566         sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset);
 567
 568         /* Fill in the HW SGT structure.
 569          *
 570          * sgt_buf is zeroed out, so the following fields are implicit
 571          * in all sgt entries:
 572          *   - offset is 0
 573          *   - format is 'dpaa2_sg_single'
 574          */
 575         for_each_sg(scl, crt_scl, num_dma_bufs, i) {
 576                 dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl));
 577                 dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl));
 578         }
 579         dpaa2_sg_set_final(&sgt[i - 1], true);
 580
 581         /* Store the skb backpointer in the SGT buffer.
 582          * Fit the scatterlist and the number of buffers alongside the
 583          * skb backpointer in the software annotation area. We'll need
 584          * all of them on Tx Conf.
 585          */
 586         swa = (struct dpaa2_eth_swa *)sgt_buf;
 587         swa->type = DPAA2_ETH_SWA_SG;
 588         swa->sg.skb = skb;
 589         swa->sg.scl = scl;
 590         swa->sg.num_sg = num_sg;
 591         swa->sg.sgt_size = sgt_buf_size;
 592
 593         /* Separately map the SGT buffer */
 594         addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL);
 595         if (unlikely(dma_mapping_error(dev, addr))) {
 596                 err = -ENOMEM;
 597                 goto dma_map_single_failed;
 598         }
 599         dpaa2_fd_set_offset(fd, priv->tx_data_offset);
 600         dpaa2_fd_set_format(fd, dpaa2_fd_sg);
 601         dpaa2_fd_set_addr(fd, addr);
 602         dpaa2_fd_set_len(fd, skb->len);
 603         dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
 604
 605         if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
 606                 enable_tx_tstamp(fd, sgt_buf);
 607
 608         return 0;
 609
 610 dma_map_single_failed:
 611         skb_free_frag(sgt_buf);
 612 sgt_buf_alloc_failed:
 613         dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
 614 dma_map_sg_failed:
 615         kfree(scl);
 616         return err;
 617 }
 618
 619 /* Create a frame descriptor based on a linear skb */
 620 static int build_single_fd(struct dpaa2_eth_priv *priv,
 621                            struct sk_buff *skb,
 622                            struct dpaa2_fd *fd)
 623 {
 624         struct device *dev = priv->net_dev->dev.parent;
 625         u8 *buffer_start, *aligned_start;
 626         struct dpaa2_eth_swa *swa;
 627         dma_addr_t addr;
 628
 629         buffer_start = skb->data - dpaa2_eth_needed_headroom(priv, skb);
 630
 631         /* If there's enough room to align the FD address, do it.
 632          * It will help hardware optimize accesses.
 633          */
 634         aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN,
 635                                   DPAA2_ETH_TX_BUF_ALIGN);
 636         if (aligned_start >= skb->head)
 637                 buffer_start = aligned_start;
 638
 639         /* Store a backpointer to the skb at the beginning of the buffer
 640          * (in the private data area) such that we can release it
 641          * on Tx confirm
 642          */
 643         swa = (struct dpaa2_eth_swa *)buffer_start;
 644         swa->type = DPAA2_ETH_SWA_SINGLE;
 645         swa->single.skb = skb;
 646
 647         addr = dma_map_single(dev, buffer_start,
 648                               skb_tail_pointer(skb) - buffer_start,
 649                               DMA_BIDIRECTIONAL);
 650         if (unlikely(dma_mapping_error(dev, addr)))
 651                 return -ENOMEM;
 652
 653         dpaa2_fd_set_addr(fd, addr);
 654         dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start));
 655         dpaa2_fd_set_len(fd, skb->len);
 656         dpaa2_fd_set_format(fd, dpaa2_fd_single);
 657         dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
 658
 659         if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
 660                 enable_tx_tstamp(fd, buffer_start);
 661
 662         return 0;
 663 }
 664
 665 /* FD freeing routine on the Tx path
 666  *
 667  * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb
 668  * back-pointed to is also freed.
 669  * This can be called either from dpaa2_eth_tx_conf() or on the error path of
 670  * dpaa2_eth_tx().
 671  */
 672 static void free_tx_fd(const struct dpaa2_eth_priv *priv,
 673                        struct dpaa2_eth_fq *fq,
 674                        const struct dpaa2_fd *fd, bool in_napi)
 675 {
 676         struct device *dev = priv->net_dev->dev.parent;
 677         dma_addr_t fd_addr;
 678         struct sk_buff *skb = NULL;
 679         unsigned char *buffer_start;
 680         struct dpaa2_eth_swa *swa;
 681         u8 fd_format = dpaa2_fd_get_format(fd);
 682         u32 fd_len = dpaa2_fd_get_len(fd);
 683
 684         fd_addr = dpaa2_fd_get_addr(fd);
 685         buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr);
 686         swa = (struct dpaa2_eth_swa *)buffer_start;
 687
 688         if (fd_format == dpaa2_fd_single) {
 689                 if (swa->type == DPAA2_ETH_SWA_SINGLE) {
 690                         skb = swa->single.skb;
 691                         /* Accessing the skb buffer is safe before dma unmap,
 692                          * because we didn't map the actual skb shell.
 693                          */
 694                         dma_unmap_single(dev, fd_addr,
 695                                          skb_tail_pointer(skb) - buffer_start,
 696                                          DMA_BIDIRECTIONAL);
 697                 } else {
 698                         WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type");
 699                         dma_unmap_single(dev, fd_addr, swa->xdp.dma_size,
 700                                          DMA_BIDIRECTIONAL);
 701                 }
 702         } else if (fd_format == dpaa2_fd_sg) {
 703                 skb = swa->sg.skb;
 704
 705                 /* Unmap the scatterlist */
 706                 dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg,
 707                              DMA_BIDIRECTIONAL);
 708                 kfree(swa->sg.scl);
 709
 710                 /* Unmap the SGT buffer */
 711                 dma_unmap_single(dev, fd_addr, swa->sg.sgt_size,
 712                                  DMA_BIDIRECTIONAL);
 713         } else {
 714                 netdev_dbg(priv->net_dev, "Invalid FD format\n");
 715                 return;
 716         }
 717
 718         if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) {
 719                 fq->dq_frames++;
 720                 fq->dq_bytes += fd_len;
 721         }
 722
 723         if (swa->type == DPAA2_ETH_SWA_XDP) {
 724                 xdp_return_frame(swa->xdp.xdpf);
 725                 return;
 726         }
 727
 728         /* Get the timestamp value */
 729         if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
 730                 struct skb_shared_hwtstamps shhwtstamps;
 731                 __le64 *ts = dpaa2_get_ts(buffer_start, true);
 732                 u64 ns;
 733
 734                 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 735
 736                 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
 737                 shhwtstamps.hwtstamp = ns_to_ktime(ns);
 738                 skb_tstamp_tx(skb, &shhwtstamps);
 739         }
 740
 741         /* Free SGT buffer allocated on tx */
 742         if (fd_format != dpaa2_fd_single)
 743                 skb_free_frag(buffer_start);
 744
 745         /* Move on with skb release */
 746         napi_consume_skb(skb, in_napi);
 747 }
 748
 749 static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev)
 750 {
 751         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
 752         struct dpaa2_fd fd;
 753         struct rtnl_link_stats64 *percpu_stats;
 754         struct dpaa2_eth_drv_stats *percpu_extras;
 755         struct dpaa2_eth_fq *fq;
 756         struct netdev_queue *nq;
 757         u16 queue_mapping;
 758         unsigned int needed_headroom;
 759         u32 fd_len;
 760         int err, i;
 761
 762         percpu_stats = this_cpu_ptr(priv->percpu_stats);
 763         percpu_extras = this_cpu_ptr(priv->percpu_extras);
 764
 765         needed_headroom = dpaa2_eth_needed_headroom(priv, skb);
 766         if (skb_headroom(skb) < needed_headroom) {
 767                 struct sk_buff *ns;
 768
 769                 ns = skb_realloc_headroom(skb, needed_headroom);
 770                 if (unlikely(!ns)) {
 771                         percpu_stats->tx_dropped++;
 772                         goto err_alloc_headroom;
 773                 }
 774                 percpu_extras->tx_reallocs++;
 775
 776                 if (skb->sk)
 777                         skb_set_owner_w(ns, skb->sk);
 778
 779                 dev_kfree_skb(skb);
 780                 skb = ns;
 781         }
 782
 783         /* We'll be holding a back-reference to the skb until Tx Confirmation;
 784          * we don't want that overwritten by a concurrent Tx with a cloned skb.
 785          */
 786         skb = skb_unshare(skb, GFP_ATOMIC);
 787         if (unlikely(!skb)) {
 788                 /* skb_unshare() has already freed the skb */
 789                 percpu_stats->tx_dropped++;
 790                 return NETDEV_TX_OK;
 791         }
 792
 793         /* Setup the FD fields */
 794         memset(&fd, 0, sizeof(fd));
 795
 796         if (skb_is_nonlinear(skb)) {
 797                 err = build_sg_fd(priv, skb, &fd);
 798                 percpu_extras->tx_sg_frames++;
 799                 percpu_extras->tx_sg_bytes += skb->len;
 800         } else {
 801                 err = build_single_fd(priv, skb, &fd);
 802         }
 803
 804         if (unlikely(err)) {
 805                 percpu_stats->tx_dropped++;
 806                 goto err_build_fd;
 807         }
 808
 809         /* Tracing point */
 810         trace_dpaa2_tx_fd(net_dev, &fd);
 811
 812         /* TxConf FQ selection relies on queue id from the stack.
 813          * In case of a forwarded frame from another DPNI interface, we choose
 814          * a queue affined to the same core that processed the Rx frame
 815          */
 816         queue_mapping = skb_get_queue_mapping(skb);
 817         fq = &priv->fq[queue_mapping];
 818
 819         fd_len = dpaa2_fd_get_len(&fd);
 820         nq = netdev_get_tx_queue(net_dev, queue_mapping);
 821         netdev_tx_sent_queue(nq, fd_len);
 822
 823         /* Everything that happens after this enqueues might race with
 824          * the Tx confirmation callback for this frame
 825          */
 826         for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
 827                 err = priv->enqueue(priv, fq, &fd, 0);
 828                 if (err != -EBUSY)
 829                         break;
 830         }
 831         percpu_extras->tx_portal_busy += i;
 832         if (unlikely(err < 0)) {
 833                 percpu_stats->tx_errors++;
 834                 /* Clean up everything, including freeing the skb */
 835                 free_tx_fd(priv, fq, &fd, false);
 836                 netdev_tx_completed_queue(nq, 1, fd_len);
 837         } else {
 838                 percpu_stats->tx_packets++;
 839                 percpu_stats->tx_bytes += fd_len;
 840         }
 841
 842         return NETDEV_TX_OK;
 843
 844 err_build_fd:
 845 err_alloc_headroom:
 846         dev_kfree_skb(skb);
 847
 848         return NETDEV_TX_OK;
 849 }
 850
 851 /* Tx confirmation frame processing routine */
 852 static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv,
 853                               struct dpaa2_eth_channel *ch __always_unused,
 854                               const struct dpaa2_fd *fd,
 855                               struct dpaa2_eth_fq *fq)
 856 {
 857         struct rtnl_link_stats64 *percpu_stats;
 858         struct dpaa2_eth_drv_stats *percpu_extras;
 859         u32 fd_len = dpaa2_fd_get_len(fd);
 860         u32 fd_errors;
 861
 862         /* Tracing point */
 863         trace_dpaa2_tx_conf_fd(priv->net_dev, fd);
 864
 865         percpu_extras = this_cpu_ptr(priv->percpu_extras);
 866         percpu_extras->tx_conf_frames++;
 867         percpu_extras->tx_conf_bytes += fd_len;
 868
 869         /* Check frame errors in the FD field */
 870         fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK;
 871         free_tx_fd(priv, fq, fd, true);
 872
 873         if (likely(!fd_errors))
 874                 return;
 875
 876         if (net_ratelimit())
 877                 netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n",
 878                            fd_errors);
 879
 880         percpu_stats = this_cpu_ptr(priv->percpu_stats);
 881         /* Tx-conf logically pertains to the egress path. */
 882         percpu_stats->tx_errors++;
 883 }
 884
 885 static int set_rx_csum(struct dpaa2_eth_priv *priv, bool enable)
 886 {
 887         int err;
 888
 889         err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
 890                                DPNI_OFF_RX_L3_CSUM, enable);
 891         if (err) {
 892                 netdev_err(priv->net_dev,
 893                            "dpni_set_offload(RX_L3_CSUM) failed\n");
 894                 return err;
 895         }
 896
 897         err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
 898                                DPNI_OFF_RX_L4_CSUM, enable);
 899         if (err) {
 900                 netdev_err(priv->net_dev,
 901                            "dpni_set_offload(RX_L4_CSUM) failed\n");
 902                 return err;
 903         }
 904
 905         return 0;
 906 }
 907
 908 static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable)
 909 {
 910         int err;
 911
 912         err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
 913                                DPNI_OFF_TX_L3_CSUM, enable);
 914         if (err) {
 915                 netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n");
 916                 return err;
 917         }
 918
 919         err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
 920                                DPNI_OFF_TX_L4_CSUM, enable);
 921         if (err) {
 922                 netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n");
 923                 return err;
 924         }
 925
 926         return 0;
 927 }
 928
 929 /* Perform a single release command to add buffers
 930  * to the specified buffer pool
 931  */
 932 static int add_bufs(struct dpaa2_eth_priv *priv,
 933                     struct dpaa2_eth_channel *ch, u16 bpid)
 934 {
 935         struct device *dev = priv->net_dev->dev.parent;
 936         u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
 937         struct page *page;
 938         dma_addr_t addr;
 939         int i, err;
 940
 941         for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) {
 942                 /* Allocate buffer visible to WRIOP + skb shared info +
 943                  * alignment padding
 944                  */
 945                 /* allocate one page for each Rx buffer. WRIOP sees
 946                  * the entire page except for a tailroom reserved for
 947                  * skb shared info
 948                  */
 949                 page = dev_alloc_pages(0);
 950                 if (!page)
 951                         goto err_alloc;
 952
 953                 addr = dma_map_page(dev, page, 0, DPAA2_ETH_RX_BUF_SIZE,
 954                                     DMA_BIDIRECTIONAL);
 955                 if (unlikely(dma_mapping_error(dev, addr)))
 956                         goto err_map;
 957
 958                 buf_array[i] = addr;
 959
 960                 /* tracing point */
 961                 trace_dpaa2_eth_buf_seed(priv->net_dev,
 962                                          page, DPAA2_ETH_RX_BUF_RAW_SIZE,
 963                                          addr, DPAA2_ETH_RX_BUF_SIZE,
 964                                          bpid);
 965         }
 966
 967 release_bufs:
 968         /* In case the portal is busy, retry until successful */
 969         while ((err = dpaa2_io_service_release(ch->dpio, bpid,
 970                                                buf_array, i)) == -EBUSY)
 971                 cpu_relax();
 972
 973         /* If release command failed, clean up and bail out;
 974          * not much else we can do about it
 975          */
 976         if (err) {
 977                 free_bufs(priv, buf_array, i);
 978                 return 0;
 979         }
 980
 981         return i;
 982
 983 err_map:
 984         __free_pages(page, 0);
 985 err_alloc:
 986         /* If we managed to allocate at least some buffers,
 987          * release them to hardware
 988          */
 989         if (i)
 990                 goto release_bufs;
 991
 992         return 0;
 993 }
 994
 995 static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid)
 996 {
 997         int i, j;
 998         int new_count;
 999
1000         /* This is the lazy seeding of Rx buffer pools.
1001          * dpaa2_add_bufs() is also used on the Rx hotpath and calls
1002          * napi_alloc_frag(). The trouble with that is that it in turn ends up
1003          * calling this_cpu_ptr(), which mandates execution in atomic context.
1004          * Rather than splitting up the code, do a one-off preempt disable.
1005          */
1006         preempt_disable();
1007         for (j = 0; j < priv->num_channels; j++) {
1008                 for (i = 0; i < DPAA2_ETH_NUM_BUFS;
1009                      i += DPAA2_ETH_BUFS_PER_CMD) {
1010                         new_count = add_bufs(priv, priv->channel[j], bpid);
1011                         priv->channel[j]->buf_count += new_count;
1012
1013                         if (new_count < DPAA2_ETH_BUFS_PER_CMD) {
1014                                 preempt_enable();
1015                                 return -ENOMEM;
1016                         }
1017                 }
1018         }
1019         preempt_enable();
1020
1021         return 0;
1022 }
1023
1024 /**
1025  * Drain the specified number of buffers from the DPNI's private buffer pool.
1026  * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD
1027  */
1028 static void drain_bufs(struct dpaa2_eth_priv *priv, int count)
1029 {
1030         u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
1031         int ret;
1032
1033         do {
1034                 ret = dpaa2_io_service_acquire(NULL, priv->bpid,
1035                                                buf_array, count);
1036                 if (ret < 0) {
1037                         netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n");
1038                         return;
1039                 }
1040                 free_bufs(priv, buf_array, ret);
1041         } while (ret);
1042 }
1043
1044 static void drain_pool(struct dpaa2_eth_priv *priv)
1045 {
1046         int i;
1047
1048         drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD);
1049         drain_bufs(priv, 1);
1050
1051         for (i = 0; i < priv->num_channels; i++)
1052                 priv->channel[i]->buf_count = 0;
1053 }
1054
1055 /* Function is called from softirq context only, so we don't need to guard
1056  * the access to percpu count
1057  */
1058 static int refill_pool(struct dpaa2_eth_priv *priv,
1059                        struct dpaa2_eth_channel *ch,
1060                        u16 bpid)
1061 {
1062         int new_count;
1063
1064         if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH))
1065                 return 0;
1066
1067         do {
1068                 new_count = add_bufs(priv, ch, bpid);
1069                 if (unlikely(!new_count)) {
1070                         /* Out of memory; abort for now, we'll try later on */
1071                         break;
1072                 }
1073                 ch->buf_count += new_count;
1074         } while (ch->buf_count < DPAA2_ETH_NUM_BUFS);
1075
1076         if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS))
1077                 return -ENOMEM;
1078
1079         return 0;
1080 }
1081
1082 static int pull_channel(struct dpaa2_eth_channel *ch)
1083 {
1084         int err;
1085         int dequeues = -1;
1086
1087         /* Retry while portal is busy */
1088         do {
1089                 err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id,
1090                                                     ch->store);
1091                 dequeues++;
1092                 cpu_relax();
1093         } while (err == -EBUSY);
1094
1095         ch->stats.dequeue_portal_busy += dequeues;
1096         if (unlikely(err))
1097                 ch->stats.pull_err++;
1098
1099         return err;
1100 }
1101
1102 /* NAPI poll routine
1103  *
1104  * Frames are dequeued from the QMan channel associated with this NAPI context.
1105  * Rx, Tx confirmation and (if configured) Rx error frames all count
1106  * towards the NAPI budget.
1107  */
1108 static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
1109 {
1110         struct dpaa2_eth_channel *ch;
1111         struct dpaa2_eth_priv *priv;
1112         int rx_cleaned = 0, txconf_cleaned = 0;
1113         struct dpaa2_eth_fq *fq, *txc_fq = NULL;
1114         struct netdev_queue *nq;
1115         int store_cleaned, work_done;
1116         int err;
1117
1118         ch = container_of(napi, struct dpaa2_eth_channel, napi);
1119         ch->xdp.res = 0;
1120         priv = ch->priv;
1121
1122         do {
1123                 err = pull_channel(ch);
1124                 if (unlikely(err))
1125                         break;
1126
1127                 /* Refill pool if appropriate */
1128                 refill_pool(priv, ch, priv->bpid);
1129
1130                 store_cleaned = consume_frames(ch, &fq);
1131                 if (!store_cleaned)
1132                         break;
1133                 if (fq->type == DPAA2_RX_FQ) {
1134                         rx_cleaned += store_cleaned;
1135                 } else {
1136                         txconf_cleaned += store_cleaned;
1137                         /* We have a single Tx conf FQ on this channel */
1138                         txc_fq = fq;
1139                 }
1140
1141                 /* If we either consumed the whole NAPI budget with Rx frames
1142                  * or we reached the Tx confirmations threshold, we're done.
1143                  */
1144                 if (rx_cleaned >= budget ||
1145                     txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) {
1146                         work_done = budget;
1147                         goto out;
1148                 }
1149         } while (store_cleaned);
1150
1151         /* We didn't consume the entire budget, so finish napi and
1152          * re-enable data availability notifications
1153          */
1154         napi_complete_done(napi, rx_cleaned);
1155         do {
1156                 err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx);
1157                 cpu_relax();
1158         } while (err == -EBUSY);
1159         WARN_ONCE(err, "CDAN notifications rearm failed on core %d",
1160                   ch->nctx.desired_cpu);
1161
1162         work_done = max(rx_cleaned, 1);
1163
1164 out:
1165         if (txc_fq && txc_fq->dq_frames) {
1166                 nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid);
1167                 netdev_tx_completed_queue(nq, txc_fq->dq_frames,
1168                                           txc_fq->dq_bytes);
1169                 txc_fq->dq_frames = 0;
1170                 txc_fq->dq_bytes = 0;
1171         }
1172
1173         if (ch->xdp.res & XDP_REDIRECT)
1174                 xdp_do_flush_map();
1175
1176         return work_done;
1177 }
1178
1179 static void enable_ch_napi(struct dpaa2_eth_priv *priv)
1180 {
1181         struct dpaa2_eth_channel *ch;
1182         int i;
1183
1184         for (i = 0; i < priv->num_channels; i++) {
1185                 ch = priv->channel[i];
1186                 napi_enable(&ch->napi);
1187         }
1188 }
1189
1190 static void disable_ch_napi(struct dpaa2_eth_priv *priv)
1191 {
1192         struct dpaa2_eth_channel *ch;
1193         int i;
1194
1195         for (i = 0; i < priv->num_channels; i++) {
1196                 ch = priv->channel[i];
1197                 napi_disable(&ch->napi);
1198         }
1199 }
1200
1201 static int link_state_update(struct dpaa2_eth_priv *priv)
1202 {
1203         struct dpni_link_state state = {0};
1204         int err;
1205
1206         err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state);
1207         if (unlikely(err)) {
1208                 netdev_err(priv->net_dev,
1209                            "dpni_get_link_state() failed\n");
1210                 return err;
1211         }
1212
1213         /* Chech link state; speed / duplex changes are not treated yet */
1214         if (priv->link_state.up == state.up)
1215                 return 0;
1216
1217         priv->link_state = state;
1218         if (state.up) {
1219                 netif_carrier_on(priv->net_dev);
1220                 netif_tx_start_all_queues(priv->net_dev);
1221         } else {
1222                 netif_tx_stop_all_queues(priv->net_dev);
1223                 netif_carrier_off(priv->net_dev);
1224         }
1225
1226         netdev_info(priv->net_dev, "Link Event: state %s\n",
1227                     state.up ? "up" : "down");
1228
1229         return 0;
1230 }
1231
1232 static int dpaa2_eth_open(struct net_device *net_dev)
1233 {
1234         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1235         int err;
1236
1237         err = seed_pool(priv, priv->bpid);
1238         if (err) {
1239                 /* Not much to do; the buffer pool, though not filled up,
1240                  * may still contain some buffers which would enable us
1241                  * to limp on.
1242                  */
1243                 netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n",
1244                            priv->dpbp_dev->obj_desc.id, priv->bpid);
1245         }
1246
1247         /* We'll only start the txqs when the link is actually ready; make sure
1248          * we don't race against the link up notification, which may come
1249          * immediately after dpni_enable();
1250          */
1251         netif_tx_stop_all_queues(net_dev);
1252         enable_ch_napi(priv);
1253         /* Also, explicitly set carrier off, otherwise netif_carrier_ok() will
1254          * return true and cause 'ip link show' to report the LOWER_UP flag,
1255          * even though the link notification wasn't even received.
1256          */
1257         netif_carrier_off(net_dev);
1258
1259         err = dpni_enable(priv->mc_io, 0, priv->mc_token);
1260         if (err < 0) {
1261                 netdev_err(net_dev, "dpni_enable() failed\n");
1262                 goto enable_err;
1263         }
1264
1265         /* If the DPMAC object has already processed the link up interrupt,
1266          * we have to learn the link state ourselves.
1267          */
1268         err = link_state_update(priv);
1269         if (err < 0) {
1270                 netdev_err(net_dev, "Can't update link state\n");
1271                 goto link_state_err;
1272         }
1273
1274         return 0;
1275
1276 link_state_err:
1277 enable_err:
1278         disable_ch_napi(priv);
1279         drain_pool(priv);
1280         return err;
1281 }
1282
1283 /* Total number of in-flight frames on ingress queues */
1284 static u32 ingress_fq_count(struct dpaa2_eth_priv *priv)
1285 {
1286         struct dpaa2_eth_fq *fq;
1287         u32 fcnt = 0, bcnt = 0, total = 0;
1288         int i, err;
1289
1290         for (i = 0; i < priv->num_fqs; i++) {
1291                 fq = &priv->fq[i];
1292                 err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt);
1293                 if (err) {
1294                         netdev_warn(priv->net_dev, "query_fq_count failed");
1295                         break;
1296                 }
1297                 total += fcnt;
1298         }
1299
1300         return total;
1301 }
1302
1303 static void wait_for_fq_empty(struct dpaa2_eth_priv *priv)
1304 {
1305         int retries = 10;
1306         u32 pending;
1307
1308         do {
1309                 pending = ingress_fq_count(priv);
1310                 if (pending)
1311                         msleep(100);
1312         } while (pending && --retries);
1313 }
1314
1315 static int dpaa2_eth_stop(struct net_device *net_dev)
1316 {
1317         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1318         int dpni_enabled = 0;
1319         int retries = 10;
1320
1321         netif_tx_stop_all_queues(net_dev);
1322         netif_carrier_off(net_dev);
1323
1324         /* On dpni_disable(), the MC firmware will:
1325          * - stop MAC Rx and wait for all Rx frames to be enqueued to software
1326          * - cut off WRIOP dequeues from egress FQs and wait until transmission
1327          * of all in flight Tx frames is finished (and corresponding Tx conf
1328          * frames are enqueued back to software)
1329          *
1330          * Before calling dpni_disable(), we wait for all Tx frames to arrive
1331          * on WRIOP. After it finishes, wait until all remaining frames on Rx
1332          * and Tx conf queues are consumed on NAPI poll.
1333          */
1334         msleep(500);
1335
1336         do {
1337                 dpni_disable(priv->mc_io, 0, priv->mc_token);
1338                 dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled);
1339                 if (dpni_enabled)
1340                         /* Allow the hardware some slack */
1341                         msleep(100);
1342         } while (dpni_enabled && --retries);
1343         if (!retries) {
1344                 netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n");
1345                 /* Must go on and disable NAPI nonetheless, so we don't crash at
1346                  * the next "ifconfig up"
1347                  */
1348         }
1349
1350         wait_for_fq_empty(priv);
1351         disable_ch_napi(priv);
1352
1353         /* Empty the buffer pool */
1354         drain_pool(priv);
1355
1356         return 0;
1357 }
1358
1359 static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr)
1360 {
1361         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1362         struct device *dev = net_dev->dev.parent;
1363         int err;
1364
1365         err = eth_mac_addr(net_dev, addr);
1366         if (err < 0) {
1367                 dev_err(dev, "eth_mac_addr() failed (%d)\n", err);
1368                 return err;
1369         }
1370
1371         err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
1372                                         net_dev->dev_addr);
1373         if (err) {
1374                 dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err);
1375                 return err;
1376         }
1377
1378         return 0;
1379 }
1380
1381 /** Fill in counters maintained by the GPP driver. These may be different from
1382  * the hardware counters obtained by ethtool.
1383  */
1384 static void dpaa2_eth_get_stats(struct net_device *net_dev,
1385                                 struct rtnl_link_stats64 *stats)
1386 {
1387         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1388         struct rtnl_link_stats64 *percpu_stats;
1389         u64 *cpustats;
1390         u64 *netstats = (u64 *)stats;
1391         int i, j;
1392         int num = sizeof(struct rtnl_link_stats64) / sizeof(u64);
1393
1394         for_each_possible_cpu(i) {
1395                 percpu_stats = per_cpu_ptr(priv->percpu_stats, i);
1396                 cpustats = (u64 *)percpu_stats;
1397                 for (j = 0; j < num; j++)
1398                         netstats[j] += cpustats[j];
1399         }
1400 }
1401
1402 /* Copy mac unicast addresses from @net_dev to @priv.
1403  * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable.
1404  */
1405 static void add_uc_hw_addr(const struct net_device *net_dev,
1406                            struct dpaa2_eth_priv *priv)
1407 {
1408         struct netdev_hw_addr *ha;
1409         int err;
1410
1411         netdev_for_each_uc_addr(ha, net_dev) {
1412                 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token,
1413                                         ha->addr);
1414                 if (err)
1415                         netdev_warn(priv->net_dev,
1416                                     "Could not add ucast MAC %pM to the filtering table (err %d)\n",
1417                                     ha->addr, err);
1418         }
1419 }
1420
1421 /* Copy mac multicast addresses from @net_dev to @priv
1422  * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable.
1423  */
1424 static void add_mc_hw_addr(const struct net_device *net_dev,
1425                            struct dpaa2_eth_priv *priv)
1426 {
1427         struct netdev_hw_addr *ha;
1428         int err;
1429
1430         netdev_for_each_mc_addr(ha, net_dev) {
1431                 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token,
1432                                         ha->addr);
1433                 if (err)
1434                         netdev_warn(priv->net_dev,
1435                                     "Could not add mcast MAC %pM to the filtering table (err %d)\n",
1436                                     ha->addr, err);
1437         }
1438 }
1439
1440 static void dpaa2_eth_set_rx_mode(struct net_device *net_dev)
1441 {
1442         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1443         int uc_count = netdev_uc_count(net_dev);
1444         int mc_count = netdev_mc_count(net_dev);
1445         u8 max_mac = priv->dpni_attrs.mac_filter_entries;
1446         u32 options = priv->dpni_attrs.options;
1447         u16 mc_token = priv->mc_token;
1448         struct fsl_mc_io *mc_io = priv->mc_io;
1449         int err;
1450
1451         /* Basic sanity checks; these probably indicate a misconfiguration */
1452         if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0)
1453                 netdev_info(net_dev,
1454                             "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n",
1455                             max_mac);
1456
1457         /* Force promiscuous if the uc or mc counts exceed our capabilities. */
1458         if (uc_count > max_mac) {
1459                 netdev_info(net_dev,
1460                             "Unicast addr count reached %d, max allowed is %d; forcing promisc\n",
1461                             uc_count, max_mac);
1462                 goto force_promisc;
1463         }
1464         if (mc_count + uc_count > max_mac) {
1465                 netdev_info(net_dev,
1466                             "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n",
1467                             uc_count + mc_count, max_mac);
1468                 goto force_mc_promisc;
1469         }
1470
1471         /* Adjust promisc settings due to flag combinations */
1472         if (net_dev->flags & IFF_PROMISC)
1473                 goto force_promisc;
1474         if (net_dev->flags & IFF_ALLMULTI) {
1475                 /* First, rebuild unicast filtering table. This should be done
1476                  * in promisc mode, in order to avoid frame loss while we
1477                  * progressively add entries to the table.
1478                  * We don't know whether we had been in promisc already, and
1479                  * making an MC call to find out is expensive; so set uc promisc
1480                  * nonetheless.
1481                  */
1482                 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1483                 if (err)
1484                         netdev_warn(net_dev, "Can't set uc promisc\n");
1485
1486                 /* Actual uc table reconstruction. */
1487                 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0);
1488                 if (err)
1489                         netdev_warn(net_dev, "Can't clear uc filters\n");
1490                 add_uc_hw_addr(net_dev, priv);
1491
1492                 /* Finally, clear uc promisc and set mc promisc as requested. */
1493                 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0);
1494                 if (err)
1495                         netdev_warn(net_dev, "Can't clear uc promisc\n");
1496                 goto force_mc_promisc;
1497         }
1498
1499         /* Neither unicast, nor multicast promisc will be on... eventually.
1500          * For now, rebuild mac filtering tables while forcing both of them on.
1501          */
1502         err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1503         if (err)
1504                 netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err);
1505         err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1);
1506         if (err)
1507                 netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err);
1508
1509         /* Actual mac filtering tables reconstruction */
1510         err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1);
1511         if (err)
1512                 netdev_warn(net_dev, "Can't clear mac filters\n");
1513         add_mc_hw_addr(net_dev, priv);
1514         add_uc_hw_addr(net_dev, priv);
1515
1516         /* Now we can clear both ucast and mcast promisc, without risking
1517          * to drop legitimate frames anymore.
1518          */
1519         err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0);
1520         if (err)
1521                 netdev_warn(net_dev, "Can't clear ucast promisc\n");
1522         err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0);
1523         if (err)
1524                 netdev_warn(net_dev, "Can't clear mcast promisc\n");
1525
1526         return;
1527
1528 force_promisc:
1529         err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1530         if (err)
1531                 netdev_warn(net_dev, "Can't set ucast promisc\n");
1532 force_mc_promisc:
1533         err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1);
1534         if (err)
1535                 netdev_warn(net_dev, "Can't set mcast promisc\n");
1536 }
1537
1538 static int dpaa2_eth_set_features(struct net_device *net_dev,
1539                                   netdev_features_t features)
1540 {
1541         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1542         netdev_features_t changed = features ^ net_dev->features;
1543         bool enable;
1544         int err;
1545
1546         if (changed & NETIF_F_RXCSUM) {
1547                 enable = !!(features & NETIF_F_RXCSUM);
1548                 err = set_rx_csum(priv, enable);
1549                 if (err)
1550                         return err;
1551         }
1552
1553         if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
1554                 enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));
1555                 err = set_tx_csum(priv, enable);
1556                 if (err)
1557                         return err;
1558         }
1559
1560         return 0;
1561 }
1562
1563 static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1564 {
1565         struct dpaa2_eth_priv *priv = netdev_priv(dev);
1566         struct hwtstamp_config config;
1567
1568         if (copy_from_user(&config, rq->ifr_data, sizeof(config)))
1569                 return -EFAULT;
1570
1571         switch (config.tx_type) {
1572         case HWTSTAMP_TX_OFF:
1573                 priv->tx_tstamp = false;
1574                 break;
1575         case HWTSTAMP_TX_ON:
1576                 priv->tx_tstamp = true;
1577                 break;
1578         default:
1579                 return -ERANGE;
1580         }
1581
1582         if (config.rx_filter == HWTSTAMP_FILTER_NONE) {
1583                 priv->rx_tstamp = false;
1584         } else {
1585                 priv->rx_tstamp = true;
1586                 /* TS is set for all frame types, not only those requested */
1587                 config.rx_filter = HWTSTAMP_FILTER_ALL;
1588         }
1589
1590         return copy_to_user(rq->ifr_data, &config, sizeof(config)) ?
1591                         -EFAULT : 0;
1592 }
1593
1594 static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1595 {
1596         if (cmd == SIOCSHWTSTAMP)
1597                 return dpaa2_eth_ts_ioctl(dev, rq, cmd);
1598
1599         return -EINVAL;
1600 }
1601
1602 static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu)
1603 {
1604         int mfl, linear_mfl;
1605
1606         mfl = DPAA2_ETH_L2_MAX_FRM(mtu);
1607         linear_mfl = DPAA2_ETH_RX_BUF_SIZE - DPAA2_ETH_RX_HWA_SIZE -
1608                      dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM;
1609
1610         if (mfl > linear_mfl) {
1611                 netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n",
1612                             linear_mfl - VLAN_ETH_HLEN);
1613                 return false;
1614         }
1615
1616         return true;
1617 }
1618
1619 static int set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp)
1620 {
1621         int mfl, err;
1622
1623         /* We enforce a maximum Rx frame length based on MTU only if we have
1624          * an XDP program attached (in order to avoid Rx S/G frames).
1625          * Otherwise, we accept all incoming frames as long as they are not
1626          * larger than maximum size supported in hardware
1627          */
1628         if (has_xdp)
1629                 mfl = DPAA2_ETH_L2_MAX_FRM(mtu);
1630         else
1631                 mfl = DPAA2_ETH_MFL;
1632
1633         err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl);
1634         if (err) {
1635                 netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n");
1636                 return err;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu)
1643 {
1644         struct dpaa2_eth_priv *priv = netdev_priv(dev);
1645         int err;
1646
1647         if (!priv->xdp_prog)
1648                 goto out;
1649
1650         if (!xdp_mtu_valid(priv, new_mtu))
1651                 return -EINVAL;
1652
1653         err = set_rx_mfl(priv, new_mtu, true);
1654         if (err)
1655                 return err;
1656
1657 out:
1658         dev->mtu = new_mtu;
1659         return 0;
1660 }
1661
1662 static int update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp)
1663 {
1664         struct dpni_buffer_layout buf_layout = {0};
1665         int err;
1666
1667         err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token,
1668                                      DPNI_QUEUE_RX, &buf_layout);
1669         if (err) {
1670                 netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n");
1671                 return err;
1672         }
1673
1674         /* Reserve extra headroom for XDP header size changes */
1675         buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) +
1676                                     (has_xdp ? XDP_PACKET_HEADROOM : 0);
1677         buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM;
1678         err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
1679                                      DPNI_QUEUE_RX, &buf_layout);
1680         if (err) {
1681                 netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n");
1682                 return err;
1683         }
1684
1685         return 0;
1686 }
1687
1688 static int setup_xdp(struct net_device *dev, struct bpf_prog *prog)
1689 {
1690         struct dpaa2_eth_priv *priv = netdev_priv(dev);
1691         struct dpaa2_eth_channel *ch;
1692         struct bpf_prog *old;
1693         bool up, need_update;
1694         int i, err;
1695
1696         if (prog && !xdp_mtu_valid(priv, dev->mtu))
1697                 return -EINVAL;
1698
1699         if (prog) {
1700                 prog = bpf_prog_add(prog, priv->num_channels);
1701                 if (IS_ERR(prog))
1702                         return PTR_ERR(prog);
1703         }
1704
1705         up = netif_running(dev);
1706         need_update = (!!priv->xdp_prog != !!prog);
1707
1708         if (up)
1709                 dpaa2_eth_stop(dev);
1710
1711         /* While in xdp mode, enforce a maximum Rx frame size based on MTU.
1712          * Also, when switching between xdp/non-xdp modes we need to reconfigure
1713          * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop,
1714          * so we are sure no old format buffers will be used from now on.
1715          */
1716         if (need_update) {
1717                 err = set_rx_mfl(priv, dev->mtu, !!prog);
1718                 if (err)
1719                         goto out_err;
1720                 err = update_rx_buffer_headroom(priv, !!prog);
1721                 if (err)
1722                         goto out_err;
1723         }
1724
1725         old = xchg(&priv->xdp_prog, prog);
1726         if (old)
1727                 bpf_prog_put(old);
1728
1729         for (i = 0; i < priv->num_channels; i++) {
1730                 ch = priv->channel[i];
1731                 old = xchg(&ch->xdp.prog, prog);
1732                 if (old)
1733                         bpf_prog_put(old);
1734         }
1735
1736         if (up) {
1737                 err = dpaa2_eth_open(dev);
1738                 if (err)
1739                         return err;
1740         }
1741
1742         return 0;
1743
1744 out_err:
1745         if (prog)
1746                 bpf_prog_sub(prog, priv->num_channels);
1747         if (up)
1748                 dpaa2_eth_open(dev);
1749
1750         return err;
1751 }
1752
1753 static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1754 {
1755         struct dpaa2_eth_priv *priv = netdev_priv(dev);
1756
1757         switch (xdp->command) {
1758         case XDP_SETUP_PROG:
1759                 return setup_xdp(dev, xdp->prog);
1760         case XDP_QUERY_PROG:
1761                 xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
1762                 break;
1763         default:
1764                 return -EINVAL;
1765         }
1766
1767         return 0;
1768 }
1769
1770 static int dpaa2_eth_xdp_xmit_frame(struct net_device *net_dev,
1771                                     struct xdp_frame *xdpf)
1772 {
1773         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1774         struct device *dev = net_dev->dev.parent;
1775         struct rtnl_link_stats64 *percpu_stats;
1776         struct dpaa2_eth_drv_stats *percpu_extras;
1777         unsigned int needed_headroom;
1778         struct dpaa2_eth_swa *swa;
1779         struct dpaa2_eth_fq *fq;
1780         struct dpaa2_fd fd;
1781         void *buffer_start, *aligned_start;
1782         dma_addr_t addr;
1783         int err, i;
1784
1785         /* We require a minimum headroom to be able to transmit the frame.
1786          * Otherwise return an error and let the original net_device handle it
1787          */
1788         needed_headroom = dpaa2_eth_needed_headroom(priv, NULL);
1789         if (xdpf->headroom < needed_headroom)
1790                 return -EINVAL;
1791
1792         percpu_stats = this_cpu_ptr(priv->percpu_stats);
1793         percpu_extras = this_cpu_ptr(priv->percpu_extras);
1794
1795         /* Setup the FD fields */
1796         memset(&fd, 0, sizeof(fd));
1797
1798         /* Align FD address, if possible */
1799         buffer_start = xdpf->data - needed_headroom;
1800         aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN,
1801                                   DPAA2_ETH_TX_BUF_ALIGN);
1802         if (aligned_start >= xdpf->data - xdpf->headroom)
1803                 buffer_start = aligned_start;
1804
1805         swa = (struct dpaa2_eth_swa *)buffer_start;
1806         /* fill in necessary fields here */
1807         swa->type = DPAA2_ETH_SWA_XDP;
1808         swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start;
1809         swa->xdp.xdpf = xdpf;
1810
1811         addr = dma_map_single(dev, buffer_start,
1812                               swa->xdp.dma_size,
1813                               DMA_BIDIRECTIONAL);
1814         if (unlikely(dma_mapping_error(dev, addr))) {
1815                 percpu_stats->tx_dropped++;
1816                 return -ENOMEM;
1817         }
1818
1819         dpaa2_fd_set_addr(&fd, addr);
1820         dpaa2_fd_set_offset(&fd, xdpf->data - buffer_start);
1821         dpaa2_fd_set_len(&fd, xdpf->len);
1822         dpaa2_fd_set_format(&fd, dpaa2_fd_single);
1823         dpaa2_fd_set_ctrl(&fd, FD_CTRL_PTA);
1824
1825         fq = &priv->fq[smp_processor_id() % dpaa2_eth_queue_count(priv)];
1826         for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
1827                 err = priv->enqueue(priv, fq, &fd, 0);
1828                 if (err != -EBUSY)
1829                         break;
1830         }
1831         percpu_extras->tx_portal_busy += i;
1832         if (unlikely(err < 0)) {
1833                 percpu_stats->tx_errors++;
1834                 /* let the Rx device handle the cleanup */
1835                 return err;
1836         }
1837
1838         percpu_stats->tx_packets++;
1839         percpu_stats->tx_bytes += dpaa2_fd_get_len(&fd);
1840
1841         return 0;
1842 }
1843
1844 static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n,
1845                               struct xdp_frame **frames, u32 flags)
1846 {
1847         int drops = 0;
1848         int i, err;
1849
1850         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1851                 return -EINVAL;
1852
1853         if (!netif_running(net_dev))
1854                 return -ENETDOWN;
1855
1856         for (i = 0; i < n; i++) {
1857                 struct xdp_frame *xdpf = frames[i];
1858
1859                 err = dpaa2_eth_xdp_xmit_frame(net_dev, xdpf);
1860                 if (err) {
1861                         xdp_return_frame_rx_napi(xdpf);
1862                         drops++;
1863                 }
1864         }
1865
1866         return n - drops;
1867 }
1868
1869 static const struct net_device_ops dpaa2_eth_ops = {
1870         .ndo_open = dpaa2_eth_open,
1871         .ndo_start_xmit = dpaa2_eth_tx,
1872         .ndo_stop = dpaa2_eth_stop,
1873         .ndo_set_mac_address = dpaa2_eth_set_addr,
1874         .ndo_get_stats64 = dpaa2_eth_get_stats,
1875         .ndo_set_rx_mode = dpaa2_eth_set_rx_mode,
1876         .ndo_set_features = dpaa2_eth_set_features,
1877         .ndo_do_ioctl = dpaa2_eth_ioctl,
1878         .ndo_change_mtu = dpaa2_eth_change_mtu,
1879         .ndo_bpf = dpaa2_eth_xdp,
1880         .ndo_xdp_xmit = dpaa2_eth_xdp_xmit,
1881 };
1882
1883 static void cdan_cb(struct dpaa2_io_notification_ctx *ctx)
1884 {
1885         struct dpaa2_eth_channel *ch;
1886
1887         ch = container_of(ctx, struct dpaa2_eth_channel, nctx);
1888
1889         /* Update NAPI statistics */
1890         ch->stats.cdan++;
1891
1892         napi_schedule_irqoff(&ch->napi);
1893 }
1894
1895 /* Allocate and configure a DPCON object */
1896 static struct fsl_mc_device *setup_dpcon(struct dpaa2_eth_priv *priv)
1897 {
1898         struct fsl_mc_device *dpcon;
1899         struct device *dev = priv->net_dev->dev.parent;
1900         struct dpcon_attr attrs;
1901         int err;
1902
1903         err = fsl_mc_object_allocate(to_fsl_mc_device(dev),
1904                                      FSL_MC_POOL_DPCON, &dpcon);
1905         if (err) {
1906                 if (err == -ENXIO)
1907                         err = -EPROBE_DEFER;
1908                 else
1909                         dev_info(dev, "Not enough DPCONs, will go on as-is\n");
1910                 return ERR_PTR(err);
1911         }
1912
1913         err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle);
1914         if (err) {
1915                 dev_err(dev, "dpcon_open() failed\n");
1916                 goto free;
1917         }
1918
1919         err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle);
1920         if (err) {
1921                 dev_err(dev, "dpcon_reset() failed\n");
1922                 goto close;
1923         }
1924
1925         err = dpcon_get_attributes(priv->mc_io, 0, dpcon->mc_handle, &attrs);
1926         if (err) {
1927                 dev_err(dev, "dpcon_get_attributes() failed\n");
1928                 goto close;
1929         }
1930
1931         err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle);
1932         if (err) {
1933                 dev_err(dev, "dpcon_enable() failed\n");
1934                 goto close;
1935         }
1936
1937         return dpcon;
1938
1939 close:
1940         dpcon_close(priv->mc_io, 0, dpcon->mc_handle);
1941 free:
1942         fsl_mc_object_free(dpcon);
1943
1944         return NULL;
1945 }
1946
1947 static void free_dpcon(struct dpaa2_eth_priv *priv,
1948                        struct fsl_mc_device *dpcon)
1949 {
1950         dpcon_disable(priv->mc_io, 0, dpcon->mc_handle);
1951         dpcon_close(priv->mc_io, 0, dpcon->mc_handle);
1952         fsl_mc_object_free(dpcon);
1953 }
1954
1955 static struct dpaa2_eth_channel *
1956 alloc_channel(struct dpaa2_eth_priv *priv)
1957 {
1958         struct dpaa2_eth_channel *channel;
1959         struct dpcon_attr attr;
1960         struct device *dev = priv->net_dev->dev.parent;
1961         int err;
1962
1963         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
1964         if (!channel)
1965                 return NULL;
1966
1967         channel->dpcon = setup_dpcon(priv);
1968         if (IS_ERR_OR_NULL(channel->dpcon)) {
1969                 err = PTR_ERR(channel->dpcon);
1970                 goto err_setup;
1971         }
1972
1973         err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle,
1974                                    &attr);
1975         if (err) {
1976                 dev_err(dev, "dpcon_get_attributes() failed\n");
1977                 goto err_get_attr;
1978         }
1979
1980         channel->dpcon_id = attr.id;
1981         channel->ch_id = attr.qbman_ch_id;
1982         channel->priv = priv;
1983
1984         return channel;
1985
1986 err_get_attr:
1987         free_dpcon(priv, channel->dpcon);
1988 err_setup:
1989         kfree(channel);
1990         return ERR_PTR(err);
1991 }
1992
1993 static void free_channel(struct dpaa2_eth_priv *priv,
1994                          struct dpaa2_eth_channel *channel)
1995 {
1996         free_dpcon(priv, channel->dpcon);
1997         kfree(channel);
1998 }
1999
2000 /* DPIO setup: allocate and configure QBMan channels, setup core affinity
2001  * and register data availability notifications
2002  */
2003 static int setup_dpio(struct dpaa2_eth_priv *priv)
2004 {
2005         struct dpaa2_io_notification_ctx *nctx;
2006         struct dpaa2_eth_channel *channel;
2007         struct dpcon_notification_cfg dpcon_notif_cfg;
2008         struct device *dev = priv->net_dev->dev.parent;
2009         int i, err;
2010
2011         /* We want the ability to spread ingress traffic (RX, TX conf) to as
2012          * many cores as possible, so we need one channel for each core
2013          * (unless there's fewer queues than cores, in which case the extra
2014          * channels would be wasted).
2015          * Allocate one channel per core and register it to the core's
2016          * affine DPIO. If not enough channels are available for all cores
2017          * or if some cores don't have an affine DPIO, there will be no
2018          * ingress frame processing on those cores.
2019          */
2020         cpumask_clear(&priv->dpio_cpumask);
2021         for_each_online_cpu(i) {
2022                 /* Try to allocate a channel */
2023                 channel = alloc_channel(priv);
2024                 if (IS_ERR_OR_NULL(channel)) {
2025                         err = PTR_ERR(channel);
2026                         if (err != -EPROBE_DEFER)
2027                                 dev_info(dev,
2028                                          "No affine channel for cpu %d and above\n", i);
2029                         goto err_alloc_ch;
2030                 }
2031
2032                 priv->channel[priv->num_channels] = channel;
2033
2034                 nctx = &channel->nctx;
2035                 nctx->is_cdan = 1;
2036                 nctx->cb = cdan_cb;
2037                 nctx->id = channel->ch_id;
2038                 nctx->desired_cpu = i;
2039
2040                 /* Register the new context */
2041                 channel->dpio = dpaa2_io_service_select(i);
2042                 err = dpaa2_io_service_register(channel->dpio, nctx, dev);
2043                 if (err) {
2044                         dev_dbg(dev, "No affine DPIO for cpu %d\n", i);
2045                         /* If no affine DPIO for this core, there's probably
2046                          * none available for next cores either. Signal we want
2047                          * to retry later, in case the DPIO devices weren't
2048                          * probed yet.
2049                          */
2050                         err = -EPROBE_DEFER;
2051                         goto err_service_reg;
2052                 }
2053
2054                 /* Register DPCON notification with MC */
2055                 dpcon_notif_cfg.dpio_id = nctx->dpio_id;
2056                 dpcon_notif_cfg.priority = 0;
2057                 dpcon_notif_cfg.user_ctx = nctx->qman64;
2058                 err = dpcon_set_notification(priv->mc_io, 0,
2059                                              channel->dpcon->mc_handle,
2060                                              &dpcon_notif_cfg);
2061                 if (err) {
2062                         dev_err(dev, "dpcon_set_notification failed()\n");
2063                         goto err_set_cdan;
2064                 }
2065
2066                 /* If we managed to allocate a channel and also found an affine
2067                  * DPIO for this core, add it to the final mask
2068                  */
2069                 cpumask_set_cpu(i, &priv->dpio_cpumask);
2070                 priv->num_channels++;
2071
2072                 /* Stop if we already have enough channels to accommodate all
2073                  * RX and TX conf queues
2074                  */
2075                 if (priv->num_channels == priv->dpni_attrs.num_queues)
2076                         break;
2077         }
2078
2079         return 0;
2080
2081 err_set_cdan:
2082         dpaa2_io_service_deregister(channel->dpio, nctx, dev);
2083 err_service_reg:
2084         free_channel(priv, channel);
2085 err_alloc_ch:
2086         if (err == -EPROBE_DEFER)
2087                 return err;
2088
2089         if (cpumask_empty(&priv->dpio_cpumask)) {
2090                 dev_err(dev, "No cpu with an affine DPIO/DPCON\n");
2091                 return -ENODEV;
2092         }
2093
2094         dev_info(dev, "Cores %*pbl available for processing ingress traffic\n",
2095                  cpumask_pr_args(&priv->dpio_cpumask));
2096
2097         return 0;
2098 }
2099
2100 static void free_dpio(struct dpaa2_eth_priv *priv)
2101 {
2102         struct device *dev = priv->net_dev->dev.parent;
2103         struct dpaa2_eth_channel *ch;
2104         int i;
2105
2106         /* deregister CDAN notifications and free channels */
2107         for (i = 0; i < priv->num_channels; i++) {
2108                 ch = priv->channel[i];
2109                 dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev);
2110                 free_channel(priv, ch);
2111         }
2112 }
2113
2114 static struct dpaa2_eth_channel *get_affine_channel(struct dpaa2_eth_priv *priv,
2115                                                     int cpu)
2116 {
2117         struct device *dev = priv->net_dev->dev.parent;
2118         int i;
2119
2120         for (i = 0; i < priv->num_channels; i++)
2121                 if (priv->channel[i]->nctx.desired_cpu == cpu)
2122                         return priv->channel[i];
2123
2124         /* We should never get here. Issue a warning and return
2125          * the first channel, because it's still better than nothing
2126          */
2127         dev_warn(dev, "No affine channel found for cpu %d\n", cpu);
2128
2129         return priv->channel[0];
2130 }
2131
2132 static void set_fq_affinity(struct dpaa2_eth_priv *priv)
2133 {
2134         struct device *dev = priv->net_dev->dev.parent;
2135         struct cpumask xps_mask;
2136         struct dpaa2_eth_fq *fq;
2137         int rx_cpu, txc_cpu;
2138         int i, err;
2139
2140         /* For each FQ, pick one channel/CPU to deliver frames to.
2141          * This may well change at runtime, either through irqbalance or
2142          * through direct user intervention.
2143          */
2144         rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask);
2145
2146         for (i = 0; i < priv->num_fqs; i++) {
2147                 fq = &priv->fq[i];
2148                 switch (fq->type) {
2149                 case DPAA2_RX_FQ:
2150                         fq->target_cpu = rx_cpu;
2151                         rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask);
2152                         if (rx_cpu >= nr_cpu_ids)
2153                                 rx_cpu = cpumask_first(&priv->dpio_cpumask);
2154                         break;
2155                 case DPAA2_TX_CONF_FQ:
2156                         fq->target_cpu = txc_cpu;
2157
2158                         /* Tell the stack to affine to txc_cpu the Tx queue
2159                          * associated with the confirmation one
2160                          */
2161                         cpumask_clear(&xps_mask);
2162                         cpumask_set_cpu(txc_cpu, &xps_mask);
2163                         err = netif_set_xps_queue(priv->net_dev, &xps_mask,
2164                                                   fq->flowid);
2165                         if (err)
2166                                 dev_err(dev, "Error setting XPS queue\n");
2167
2168                         txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask);
2169                         if (txc_cpu >= nr_cpu_ids)
2170                                 txc_cpu = cpumask_first(&priv->dpio_cpumask);
2171                         break;
2172                 default:
2173                         dev_err(dev, "Unknown FQ type: %d\n", fq->type);
2174                 }
2175                 fq->channel = get_affine_channel(priv, fq->target_cpu);
2176         }
2177 }
2178
2179 static void setup_fqs(struct dpaa2_eth_priv *priv)
2180 {
2181         int i;
2182
2183         /* We have one TxConf FQ per Tx flow.
2184          * The number of Tx and Rx queues is the same.
2185          * Tx queues come first in the fq array.
2186          */
2187         for (i = 0; i < dpaa2_eth_queue_count(priv); i++) {
2188                 priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ;
2189                 priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf;
2190                 priv->fq[priv->num_fqs++].flowid = (u16)i;
2191         }
2192
2193         for (i = 0; i < dpaa2_eth_queue_count(priv); i++) {
2194                 priv->fq[priv->num_fqs].type = DPAA2_RX_FQ;
2195                 priv->fq[priv->num_fqs].consume = dpaa2_eth_rx;
2196                 priv->fq[priv->num_fqs++].flowid = (u16)i;
2197         }
2198
2199         /* For each FQ, decide on which core to process incoming frames */
2200         set_fq_affinity(priv);
2201 }
2202
2203 /* Allocate and configure one buffer pool for each interface */
2204 static int setup_dpbp(struct dpaa2_eth_priv *priv)
2205 {
2206         int err;
2207         struct fsl_mc_device *dpbp_dev;
2208         struct device *dev = priv->net_dev->dev.parent;
2209         struct dpbp_attr dpbp_attrs;
2210
2211         err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP,
2212                                      &dpbp_dev);
2213         if (err) {
2214                 if (err == -ENXIO)
2215                         err = -EPROBE_DEFER;
2216                 else
2217                         dev_err(dev, "DPBP device allocation failed\n");
2218                 return err;
2219         }
2220
2221         priv->dpbp_dev = dpbp_dev;
2222
2223         err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id,
2224                         &dpbp_dev->mc_handle);
2225         if (err) {
2226                 dev_err(dev, "dpbp_open() failed\n");
2227                 goto err_open;
2228         }
2229
2230         err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle);
2231         if (err) {
2232                 dev_err(dev, "dpbp_reset() failed\n");
2233                 goto err_reset;
2234         }
2235
2236         err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle);
2237         if (err) {
2238                 dev_err(dev, "dpbp_enable() failed\n");
2239                 goto err_enable;
2240         }
2241
2242         err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle,
2243                                   &dpbp_attrs);
2244         if (err) {
2245                 dev_err(dev, "dpbp_get_attributes() failed\n");
2246                 goto err_get_attr;
2247         }
2248         priv->bpid = dpbp_attrs.bpid;
2249
2250         return 0;
2251
2252 err_get_attr:
2253         dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle);
2254 err_enable:
2255 err_reset:
2256         dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle);
2257 err_open:
2258         fsl_mc_object_free(dpbp_dev);
2259
2260         return err;
2261 }
2262
2263 static void free_dpbp(struct dpaa2_eth_priv *priv)
2264 {
2265         drain_pool(priv);
2266         dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle);
2267         dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle);
2268         fsl_mc_object_free(priv->dpbp_dev);
2269 }
2270
2271 static int set_buffer_layout(struct dpaa2_eth_priv *priv)
2272 {
2273         struct device *dev = priv->net_dev->dev.parent;
2274         struct dpni_buffer_layout buf_layout = {0};
2275         u16 rx_buf_align;
2276         int err;
2277
2278         /* We need to check for WRIOP version 1.0.0, but depending on the MC
2279          * version, this number is not always provided correctly on rev1.
2280          * We need to check for both alternatives in this situation.
2281          */
2282         if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) ||
2283             priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0))
2284                 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1;
2285         else
2286                 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN;
2287
2288         /* tx buffer */
2289         buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE;
2290         buf_layout.pass_timestamp = true;
2291         buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE |
2292                              DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2293         err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2294                                      DPNI_QUEUE_TX, &buf_layout);
2295         if (err) {
2296                 dev_err(dev, "dpni_set_buffer_layout(TX) failed\n");
2297                 return err;
2298         }
2299
2300         /* tx-confirm buffer */
2301         buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2302         err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2303                                      DPNI_QUEUE_TX_CONFIRM, &buf_layout);
2304         if (err) {
2305                 dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n");
2306                 return err;
2307         }
2308
2309         /* Now that we've set our tx buffer layout, retrieve the minimum
2310          * required tx data offset.
2311          */
2312         err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token,
2313                                       &priv->tx_data_offset);
2314         if (err) {
2315                 dev_err(dev, "dpni_get_tx_data_offset() failed\n");
2316                 return err;
2317         }
2318
2319         if ((priv->tx_data_offset % 64) != 0)
2320                 dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n",
2321                          priv->tx_data_offset);
2322
2323         /* rx buffer */
2324         buf_layout.pass_frame_status = true;
2325         buf_layout.pass_parser_result = true;
2326         buf_layout.data_align = rx_buf_align;
2327         buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv);
2328         buf_layout.private_data_size = 0;
2329         buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT |
2330                              DPNI_BUF_LAYOUT_OPT_FRAME_STATUS |
2331                              DPNI_BUF_LAYOUT_OPT_DATA_ALIGN |
2332                              DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM |
2333                              DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2334         err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2335                                      DPNI_QUEUE_RX, &buf_layout);
2336         if (err) {
2337                 dev_err(dev, "dpni_set_buffer_layout(RX) failed\n");
2338                 return err;
2339         }
2340
2341         return 0;
2342 }
2343
2344 #define DPNI_ENQUEUE_FQID_VER_MAJOR     7
2345 #define DPNI_ENQUEUE_FQID_VER_MINOR     9
2346
2347 static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv,
2348                                        struct dpaa2_eth_fq *fq,
2349                                        struct dpaa2_fd *fd, u8 prio)
2350 {
2351         return dpaa2_io_service_enqueue_qd(fq->channel->dpio,
2352                                            priv->tx_qdid, prio,
2353                                            fq->tx_qdbin, fd);
2354 }
2355
2356 static inline int dpaa2_eth_enqueue_fq(struct dpaa2_eth_priv *priv,
2357                                        struct dpaa2_eth_fq *fq,
2358                                        struct dpaa2_fd *fd,
2359                                        u8 prio __always_unused)
2360 {
2361         return dpaa2_io_service_enqueue_fq(fq->channel->dpio,
2362                                            fq->tx_fqid, fd);
2363 }
2364
2365 static void set_enqueue_mode(struct dpaa2_eth_priv *priv)
2366 {
2367         if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR,
2368                                    DPNI_ENQUEUE_FQID_VER_MINOR) < 0)
2369                 priv->enqueue = dpaa2_eth_enqueue_qd;
2370         else
2371                 priv->enqueue = dpaa2_eth_enqueue_fq;
2372 }
2373
2374 /* Configure the DPNI object this interface is associated with */
2375 static int setup_dpni(struct fsl_mc_device *ls_dev)
2376 {
2377         struct device *dev = &ls_dev->dev;
2378         struct dpaa2_eth_priv *priv;
2379         struct net_device *net_dev;
2380         int err;
2381
2382         net_dev = dev_get_drvdata(dev);
2383         priv = netdev_priv(net_dev);
2384
2385         /* get a handle for the DPNI object */
2386         err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token);
2387         if (err) {
2388                 dev_err(dev, "dpni_open() failed\n");
2389                 return err;
2390         }
2391
2392         /* Check if we can work with this DPNI object */
2393         err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major,
2394                                    &priv->dpni_ver_minor);
2395         if (err) {
2396                 dev_err(dev, "dpni_get_api_version() failed\n");
2397                 goto close;
2398         }
2399         if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) {
2400                 dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n",
2401                         priv->dpni_ver_major, priv->dpni_ver_minor,
2402                         DPNI_VER_MAJOR, DPNI_VER_MINOR);
2403                 err = -ENOTSUPP;
2404                 goto close;
2405         }
2406
2407         ls_dev->mc_io = priv->mc_io;
2408         ls_dev->mc_handle = priv->mc_token;
2409
2410         err = dpni_reset(priv->mc_io, 0, priv->mc_token);
2411         if (err) {
2412                 dev_err(dev, "dpni_reset() failed\n");
2413                 goto close;
2414         }
2415
2416         err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token,
2417                                   &priv->dpni_attrs);
2418         if (err) {
2419                 dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err);
2420                 goto close;
2421         }
2422
2423         err = set_buffer_layout(priv);
2424         if (err)
2425                 goto close;
2426
2427         set_enqueue_mode(priv);
2428
2429         priv->cls_rules = devm_kzalloc(dev, sizeof(struct dpaa2_eth_cls_rule) *
2430                                        dpaa2_eth_fs_count(priv), GFP_KERNEL);
2431         if (!priv->cls_rules)
2432                 goto close;
2433
2434         return 0;
2435
2436 close:
2437         dpni_close(priv->mc_io, 0, priv->mc_token);
2438
2439         return err;
2440 }
2441
2442 static void free_dpni(struct dpaa2_eth_priv *priv)
2443 {
2444         int err;
2445
2446         err = dpni_reset(priv->mc_io, 0, priv->mc_token);
2447         if (err)
2448                 netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n",
2449                             err);
2450
2451         dpni_close(priv->mc_io, 0, priv->mc_token);
2452 }
2453
2454 static int setup_rx_flow(struct dpaa2_eth_priv *priv,
2455                          struct dpaa2_eth_fq *fq)
2456 {
2457         struct device *dev = priv->net_dev->dev.parent;
2458         struct dpni_queue queue;
2459         struct dpni_queue_id qid;
2460         struct dpni_taildrop td;
2461         int err;
2462
2463         err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2464                              DPNI_QUEUE_RX, 0, fq->flowid, &queue, &qid);
2465         if (err) {
2466                 dev_err(dev, "dpni_get_queue(RX) failed\n");
2467                 return err;
2468         }
2469
2470         fq->fqid = qid.fqid;
2471
2472         queue.destination.id = fq->channel->dpcon_id;
2473         queue.destination.type = DPNI_DEST_DPCON;
2474         queue.destination.priority = 1;
2475         queue.user_context = (u64)(uintptr_t)fq;
2476         queue.flc.stash_control = 1;
2477         queue.flc.value &= 0xFFFFFFFFFFFFFFC0;
2478         /* 01 01 00 - data, annotation, flow context */
2479         queue.flc.value |= 0x14;
2480         err = dpni_set_queue(priv->mc_io, 0, priv->mc_token,
2481                              DPNI_QUEUE_RX, 0, fq->flowid,
2482                              DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST |
2483                              DPNI_QUEUE_OPT_FLC,
2484                              &queue);
2485         if (err) {
2486                 dev_err(dev, "dpni_set_queue(RX) failed\n");
2487                 return err;
2488         }
2489
2490         td.enable = 1;
2491         td.threshold = DPAA2_ETH_TAILDROP_THRESH;
2492         err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, DPNI_CP_QUEUE,
2493                                 DPNI_QUEUE_RX, 0, fq->flowid, &td);
2494         if (err) {
2495                 dev_err(dev, "dpni_set_threshold() failed\n");
2496                 return err;
2497         }
2498
2499         /* xdp_rxq setup */
2500         err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev,
2501                                fq->flowid);
2502         if (err) {
2503                 dev_err(dev, "xdp_rxq_info_reg failed\n");
2504                 return err;
2505         }
2506
2507         err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq,
2508                                          MEM_TYPE_PAGE_ORDER0, NULL);
2509         if (err) {
2510                 dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n");
2511                 return err;
2512         }
2513
2514         return 0;
2515 }
2516
2517 static int setup_tx_flow(struct dpaa2_eth_priv *priv,
2518                          struct dpaa2_eth_fq *fq)
2519 {
2520         struct device *dev = priv->net_dev->dev.parent;
2521         struct dpni_queue queue;
2522         struct dpni_queue_id qid;
2523         int err;
2524
2525         err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2526                              DPNI_QUEUE_TX, 0, fq->flowid, &queue, &qid);
2527         if (err) {
2528                 dev_err(dev, "dpni_get_queue(TX) failed\n");
2529                 return err;
2530         }
2531
2532         fq->tx_qdbin = qid.qdbin;
2533         fq->tx_fqid = qid.fqid;
2534
2535         err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2536                              DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid,
2537                              &queue, &qid);
2538         if (err) {
2539                 dev_err(dev, "dpni_get_queue(TX_CONF) failed\n");
2540                 return err;
2541         }
2542
2543         fq->fqid = qid.fqid;
2544
2545         queue.destination.id = fq->channel->dpcon_id;
2546         queue.destination.type = DPNI_DEST_DPCON;
2547         queue.destination.priority = 0;
2548         queue.user_context = (u64)(uintptr_t)fq;
2549         err = dpni_set_queue(priv->mc_io, 0, priv->mc_token,
2550                              DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid,
2551                              DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST,
2552                              &queue);
2553         if (err) {
2554                 dev_err(dev, "dpni_set_queue(TX_CONF) failed\n");
2555                 return err;
2556         }
2557
2558         return 0;
2559 }
2560
2561 /* Supported header fields for Rx hash distribution key */
2562 static const struct dpaa2_eth_dist_fields dist_fields[] = {
2563         {
2564                 /* L2 header */
2565                 .rxnfc_field = RXH_L2DA,
2566                 .cls_prot = NET_PROT_ETH,
2567                 .cls_field = NH_FLD_ETH_DA,
2568                 .size = 6,
2569         }, {
2570                 .cls_prot = NET_PROT_ETH,
2571                 .cls_field = NH_FLD_ETH_SA,
2572                 .size = 6,
2573         }, {
2574                 /* This is the last ethertype field parsed:
2575                  * depending on frame format, it can be the MAC ethertype
2576                  * or the VLAN etype.
2577                  */
2578                 .cls_prot = NET_PROT_ETH,
2579                 .cls_field = NH_FLD_ETH_TYPE,
2580                 .size = 2,
2581         }, {
2582                 /* VLAN header */
2583                 .rxnfc_field = RXH_VLAN,
2584                 .cls_prot = NET_PROT_VLAN,
2585                 .cls_field = NH_FLD_VLAN_TCI,
2586                 .size = 2,
2587         }, {
2588                 /* IP header */
2589                 .rxnfc_field = RXH_IP_SRC,
2590                 .cls_prot = NET_PROT_IP,
2591                 .cls_field = NH_FLD_IP_SRC,
2592                 .size = 4,
2593         }, {
2594                 .rxnfc_field = RXH_IP_DST,
2595                 .cls_prot = NET_PROT_IP,
2596                 .cls_field = NH_FLD_IP_DST,
2597                 .size = 4,
2598         }, {
2599                 .rxnfc_field = RXH_L3_PROTO,
2600                 .cls_prot = NET_PROT_IP,
2601                 .cls_field = NH_FLD_IP_PROTO,
2602                 .size = 1,
2603         }, {
2604                 /* Using UDP ports, this is functionally equivalent to raw
2605                  * byte pairs from L4 header.
2606                  */
2607                 .rxnfc_field = RXH_L4_B_0_1,
2608                 .cls_prot = NET_PROT_UDP,
2609                 .cls_field = NH_FLD_UDP_PORT_SRC,
2610                 .size = 2,
2611         }, {
2612                 .rxnfc_field = RXH_L4_B_2_3,
2613                 .cls_prot = NET_PROT_UDP,
2614                 .cls_field = NH_FLD_UDP_PORT_DST,
2615                 .size = 2,
2616         },
2617 };
2618
2619 /* Configure the Rx hash key using the legacy API */
2620 static int config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2621 {
2622         struct device *dev = priv->net_dev->dev.parent;
2623         struct dpni_rx_tc_dist_cfg dist_cfg;
2624         int err;
2625
2626         memset(&dist_cfg, 0, sizeof(dist_cfg));
2627
2628         dist_cfg.key_cfg_iova = key;
2629         dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2630         dist_cfg.dist_mode = DPNI_DIST_MODE_HASH;
2631
2632         err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 0, &dist_cfg);
2633         if (err)
2634                 dev_err(dev, "dpni_set_rx_tc_dist failed\n");
2635
2636         return err;
2637 }
2638
2639 /* Configure the Rx hash key using the new API */
2640 static int config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2641 {
2642         struct device *dev = priv->net_dev->dev.parent;
2643         struct dpni_rx_dist_cfg dist_cfg;
2644         int err;
2645
2646         memset(&dist_cfg, 0, sizeof(dist_cfg));
2647
2648         dist_cfg.key_cfg_iova = key;
2649         dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2650         dist_cfg.enable = 1;
2651
2652         err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg);
2653         if (err)
2654                 dev_err(dev, "dpni_set_rx_hash_dist failed\n");
2655
2656         return err;
2657 }
2658
2659 /* Configure the Rx flow classification key */
2660 static int config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2661 {
2662         struct device *dev = priv->net_dev->dev.parent;
2663         struct dpni_rx_dist_cfg dist_cfg;
2664         int err;
2665
2666         memset(&dist_cfg, 0, sizeof(dist_cfg));
2667
2668         dist_cfg.key_cfg_iova = key;
2669         dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2670         dist_cfg.enable = 1;
2671
2672         err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg);
2673         if (err)
2674                 dev_err(dev, "dpni_set_rx_fs_dist failed\n");
2675
2676         return err;
2677 }
2678
2679 /* Size of the Rx flow classification key */
2680 int dpaa2_eth_cls_key_size(void)
2681 {
2682         int i, size = 0;
2683
2684         for (i = 0; i < ARRAY_SIZE(dist_fields); i++)
2685                 size += dist_fields[i].size;
2686
2687         return size;
2688 }
2689
2690 /* Offset of header field in Rx classification key */
2691 int dpaa2_eth_cls_fld_off(int prot, int field)
2692 {
2693         int i, off = 0;
2694
2695         for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
2696                 if (dist_fields[i].cls_prot == prot &&
2697                     dist_fields[i].cls_field == field)
2698                         return off;
2699                 off += dist_fields[i].size;
2700         }
2701
2702         WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n");
2703         return 0;
2704 }
2705
2706 /* Set Rx distribution (hash or flow classification) key
2707  * flags is a combination of RXH_ bits
2708  */
2709 static int dpaa2_eth_set_dist_key(struct net_device *net_dev,
2710                                   enum dpaa2_eth_rx_dist type, u64 flags)
2711 {
2712         struct device *dev = net_dev->dev.parent;
2713         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
2714         struct dpkg_profile_cfg cls_cfg;
2715         u32 rx_hash_fields = 0;
2716         dma_addr_t key_iova;
2717         u8 *dma_mem;
2718         int i;
2719         int err = 0;
2720
2721         memset(&cls_cfg, 0, sizeof(cls_cfg));
2722
2723         for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
2724                 struct dpkg_extract *key =
2725                         &cls_cfg.extracts[cls_cfg.num_extracts];
2726
2727                 /* For Rx hashing key we set only the selected fields.
2728                  * For Rx flow classification key we set all supported fields
2729                  */
2730                 if (type == DPAA2_ETH_RX_DIST_HASH) {
2731                         if (!(flags & dist_fields[i].rxnfc_field))
2732                                 continue;
2733                         rx_hash_fields |= dist_fields[i].rxnfc_field;
2734                 }
2735
2736                 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) {
2737                         dev_err(dev, "error adding key extraction rule, too many rules?\n");
2738                         return -E2BIG;
2739                 }
2740
2741                 key->type = DPKG_EXTRACT_FROM_HDR;
2742                 key->extract.from_hdr.prot = dist_fields[i].cls_prot;
2743                 key->extract.from_hdr.type = DPKG_FULL_FIELD;
2744                 key->extract.from_hdr.field = dist_fields[i].cls_field;
2745                 cls_cfg.num_extracts++;
2746         }
2747
2748         dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL);
2749         if (!dma_mem)
2750                 return -ENOMEM;
2751
2752         err = dpni_prepare_key_cfg(&cls_cfg, dma_mem);
2753         if (err) {
2754                 dev_err(dev, "dpni_prepare_key_cfg error %d\n", err);
2755                 goto free_key;
2756         }
2757
2758         /* Prepare for setting the rx dist */
2759         key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE,
2760                                   DMA_TO_DEVICE);
2761         if (dma_mapping_error(dev, key_iova)) {
2762                 dev_err(dev, "DMA mapping failed\n");
2763                 err = -ENOMEM;
2764                 goto free_key;
2765         }
2766
2767         if (type == DPAA2_ETH_RX_DIST_HASH) {
2768                 if (dpaa2_eth_has_legacy_dist(priv))
2769                         err = config_legacy_hash_key(priv, key_iova);
2770                 else
2771                         err = config_hash_key(priv, key_iova);
2772         } else {
2773                 err = config_cls_key(priv, key_iova);
2774         }
2775
2776         dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE,
2777                          DMA_TO_DEVICE);
2778         if (!err && type == DPAA2_ETH_RX_DIST_HASH)
2779                 priv->rx_hash_fields = rx_hash_fields;
2780
2781 free_key:
2782         kfree(dma_mem);
2783         return err;
2784 }
2785
2786 int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags)
2787 {
2788         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
2789
2790         if (!dpaa2_eth_hash_enabled(priv))
2791                 return -EOPNOTSUPP;
2792
2793         return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, flags);
2794 }
2795
2796 static int dpaa2_eth_set_cls(struct dpaa2_eth_priv *priv)
2797 {
2798         struct device *dev = priv->net_dev->dev.parent;
2799
2800         /* Check if we actually support Rx flow classification */
2801         if (dpaa2_eth_has_legacy_dist(priv)) {
2802                 dev_dbg(dev, "Rx cls not supported by current MC version\n");
2803                 return -EOPNOTSUPP;
2804         }
2805
2806         if (priv->dpni_attrs.options & DPNI_OPT_NO_FS ||
2807             !(priv->dpni_attrs.options & DPNI_OPT_HAS_KEY_MASKING)) {
2808                 dev_dbg(dev, "Rx cls disabled in DPNI options\n");
2809                 return -EOPNOTSUPP;
2810         }
2811
2812         if (!dpaa2_eth_hash_enabled(priv)) {
2813                 dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n");
2814                 return -EOPNOTSUPP;
2815         }
2816
2817         priv->rx_cls_enabled = 1;
2818
2819         return dpaa2_eth_set_dist_key(priv->net_dev, DPAA2_ETH_RX_DIST_CLS, 0);
2820 }
2821
2822 /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs,
2823  * frame queues and channels
2824  */
2825 static int bind_dpni(struct dpaa2_eth_priv *priv)
2826 {
2827         struct net_device *net_dev = priv->net_dev;
2828         struct device *dev = net_dev->dev.parent;
2829         struct dpni_pools_cfg pools_params;
2830         struct dpni_error_cfg err_cfg;
2831         int err = 0;
2832         int i;
2833
2834         pools_params.num_dpbp = 1;
2835         pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id;
2836         pools_params.pools[0].backup_pool = 0;
2837         pools_params.pools[0].buffer_size = DPAA2_ETH_RX_BUF_SIZE;
2838         err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
2839         if (err) {
2840                 dev_err(dev, "dpni_set_pools() failed\n");
2841                 return err;
2842         }
2843
2844         /* have the interface implicitly distribute traffic based on
2845          * the default hash key
2846          */
2847         err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT);
2848         if (err && err != -EOPNOTSUPP)
2849                 dev_err(dev, "Failed to configure hashing\n");
2850
2851         /* Configure the flow classification key; it includes all
2852          * supported header fields and cannot be modified at runtime
2853          */
2854         err = dpaa2_eth_set_cls(priv);
2855         if (err && err != -EOPNOTSUPP)
2856                 dev_err(dev, "Failed to configure Rx classification key\n");
2857
2858         /* Configure handling of error frames */
2859         err_cfg.errors = DPAA2_FAS_RX_ERR_MASK;
2860         err_cfg.set_frame_annotation = 1;
2861         err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD;
2862         err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token,
2863                                        &err_cfg);
2864         if (err) {
2865                 dev_err(dev, "dpni_set_errors_behavior failed\n");
2866                 return err;
2867         }
2868
2869         /* Configure Rx and Tx conf queues to generate CDANs */
2870         for (i = 0; i < priv->num_fqs; i++) {
2871                 switch (priv->fq[i].type) {
2872                 case DPAA2_RX_FQ:
2873                         err = setup_rx_flow(priv, &priv->fq[i]);
2874                         break;
2875                 case DPAA2_TX_CONF_FQ:
2876                         err = setup_tx_flow(priv, &priv->fq[i]);
2877                         break;
2878                 default:
2879                         dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type);
2880                         return -EINVAL;
2881                 }
2882                 if (err)
2883                         return err;
2884         }
2885
2886         err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token,
2887                             DPNI_QUEUE_TX, &priv->tx_qdid);
2888         if (err) {
2889                 dev_err(dev, "dpni_get_qdid() failed\n");
2890                 return err;
2891         }
2892
2893         return 0;
2894 }
2895
2896 /* Allocate rings for storing incoming frame descriptors */
2897 static int alloc_rings(struct dpaa2_eth_priv *priv)
2898 {
2899         struct net_device *net_dev = priv->net_dev;
2900         struct device *dev = net_dev->dev.parent;
2901         int i;
2902
2903         for (i = 0; i < priv->num_channels; i++) {
2904                 priv->channel[i]->store =
2905                         dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev);
2906                 if (!priv->channel[i]->store) {
2907                         netdev_err(net_dev, "dpaa2_io_store_create() failed\n");
2908                         goto err_ring;
2909                 }
2910         }
2911
2912         return 0;
2913
2914 err_ring:
2915         for (i = 0; i < priv->num_channels; i++) {
2916                 if (!priv->channel[i]->store)
2917                         break;
2918                 dpaa2_io_store_destroy(priv->channel[i]->store);
2919         }
2920
2921         return -ENOMEM;
2922 }
2923
2924 static void free_rings(struct dpaa2_eth_priv *priv)
2925 {
2926         int i;
2927
2928         for (i = 0; i < priv->num_channels; i++)
2929                 dpaa2_io_store_destroy(priv->channel[i]->store);
2930 }
2931
2932 static int set_mac_addr(struct dpaa2_eth_priv *priv)
2933 {
2934         struct net_device *net_dev = priv->net_dev;
2935         struct device *dev = net_dev->dev.parent;
2936         u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN];
2937         int err;
2938
2939         /* Get firmware address, if any */
2940         err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr);
2941         if (err) {
2942                 dev_err(dev, "dpni_get_port_mac_addr() failed\n");
2943                 return err;
2944         }
2945
2946         /* Get DPNI attributes address, if any */
2947         err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
2948                                         dpni_mac_addr);
2949         if (err) {
2950                 dev_err(dev, "dpni_get_primary_mac_addr() failed\n");
2951                 return err;
2952         }
2953
2954         /* First check if firmware has any address configured by bootloader */
2955         if (!is_zero_ether_addr(mac_addr)) {
2956                 /* If the DPMAC addr != DPNI addr, update it */
2957                 if (!ether_addr_equal(mac_addr, dpni_mac_addr)) {
2958                         err = dpni_set_primary_mac_addr(priv->mc_io, 0,
2959                                                         priv->mc_token,
2960                                                         mac_addr);
2961                         if (err) {
2962                                 dev_err(dev, "dpni_set_primary_mac_addr() failed\n");
2963                                 return err;
2964                         }
2965                 }
2966                 memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len);
2967         } else if (is_zero_ether_addr(dpni_mac_addr)) {
2968                 /* No MAC address configured, fill in net_dev->dev_addr
2969                  * with a random one
2970                  */
2971                 eth_hw_addr_random(net_dev);
2972                 dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n");
2973
2974                 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
2975                                                 net_dev->dev_addr);
2976                 if (err) {
2977                         dev_err(dev, "dpni_set_primary_mac_addr() failed\n");
2978                         return err;
2979                 }
2980
2981                 /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all
2982                  * practical purposes, this will be our "permanent" mac address,
2983                  * at least until the next reboot. This move will also permit
2984                  * register_netdevice() to properly fill up net_dev->perm_addr.
2985                  */
2986                 net_dev->addr_assign_type = NET_ADDR_PERM;
2987         } else {
2988                 /* NET_ADDR_PERM is default, all we have to do is
2989                  * fill in the device addr.
2990                  */
2991                 memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len);
2992         }
2993
2994         return 0;
2995 }
2996
2997 static int netdev_init(struct net_device *net_dev)
2998 {
2999         struct device *dev = net_dev->dev.parent;
3000         struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
3001         u32 options = priv->dpni_attrs.options;
3002         u64 supported = 0, not_supported = 0;
3003         u8 bcast_addr[ETH_ALEN];
3004         u8 num_queues;
3005         int err;
3006
3007         net_dev->netdev_ops = &dpaa2_eth_ops;
3008         net_dev->ethtool_ops = &dpaa2_ethtool_ops;
3009
3010         err = set_mac_addr(priv);
3011         if (err)
3012                 return err;
3013
3014         /* Explicitly add the broadcast address to the MAC filtering table */
3015         eth_broadcast_addr(bcast_addr);
3016         err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr);
3017         if (err) {
3018                 dev_err(dev, "dpni_add_mac_addr() failed\n");
3019                 return err;
3020         }
3021
3022         /* Set MTU upper limit; lower limit is 68B (default value) */
3023         net_dev->max_mtu = DPAA2_ETH_MAX_MTU;
3024         err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token,
3025                                         DPAA2_ETH_MFL);
3026         if (err) {
3027                 dev_err(dev, "dpni_set_max_frame_length() failed\n");
3028                 return err;
3029         }
3030
3031         /* Set actual number of queues in the net device */
3032         num_queues = dpaa2_eth_queue_count(priv);
3033         err = netif_set_real_num_tx_queues(net_dev, num_queues);
3034         if (err) {
3035                 dev_err(dev, "netif_set_real_num_tx_queues() failed\n");
3036                 return err;
3037         }
3038         err = netif_set_real_num_rx_queues(net_dev, num_queues);
3039         if (err) {
3040                 dev_err(dev, "netif_set_real_num_rx_queues() failed\n");
3041                 return err;
3042         }
3043
3044         /* Capabilities listing */
3045         supported |= IFF_LIVE_ADDR_CHANGE;
3046
3047         if (options & DPNI_OPT_NO_MAC_FILTER)
3048                 not_supported |= IFF_UNICAST_FLT;
3049         else
3050                 supported |= IFF_UNICAST_FLT;
3051
3052         net_dev->priv_flags |= supported;
3053         net_dev->priv_flags &= ~not_supported;
3054
3055         /* Features */
3056         net_dev->features = NETIF_F_RXCSUM |
3057                             NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3058                             NETIF_F_SG | NETIF_F_HIGHDMA |
3059                             NETIF_F_LLTX;
3060         net_dev->hw_features = net_dev->features;
3061
3062         return 0;
3063 }
3064
3065 static int poll_link_state(void *arg)
3066 {
3067         struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg;
3068         int err;
3069
3070         while (!kthread_should_stop()) {
3071                 err = link_state_update(priv);
3072                 if (unlikely(err))
3073                         return err;
3074
3075                 msleep(DPAA2_ETH_LINK_STATE_REFRESH);
3076         }
3077
3078         return 0;
3079 }
3080
3081 static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg)
3082 {
3083         u32 status = ~0;
3084         struct device *dev = (struct device *)arg;
3085         struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev);
3086         struct net_device *net_dev = dev_get_drvdata(dev);
3087         int err;
3088
3089         err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle,
3090                                   DPNI_IRQ_INDEX, &status);
3091         if (unlikely(err)) {
3092                 netdev_err(net_dev, "Can't get irq status (err %d)\n", err);
3093                 return IRQ_HANDLED;
3094         }
3095
3096         if (status & DPNI_IRQ_EVENT_LINK_CHANGED)
3097                 link_state_update(netdev_priv(net_dev));
3098
3099         return IRQ_HANDLED;
3100 }
3101
3102 static int setup_irqs(struct fsl_mc_device *ls_dev)
3103 {
3104         int err = 0;
3105         struct fsl_mc_device_irq *irq;
3106
3107         err = fsl_mc_allocate_irqs(ls_dev);
3108         if (err) {
3109                 dev_err(&ls_dev->dev, "MC irqs allocation failed\n");
3110                 return err;
3111         }
3112
3113         irq = ls_dev->irqs[0];
3114         err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq,
3115                                         NULL, dpni_irq0_handler_thread,
3116                                         IRQF_NO_SUSPEND | IRQF_ONESHOT,
3117                                         dev_name(&ls_dev->dev), &ls_dev->dev);
3118         if (err < 0) {
3119                 dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err);
3120                 goto free_mc_irq;
3121         }
3122
3123         err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle,
3124                                 DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED);
3125         if (err < 0) {
3126                 dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err);
3127                 goto free_irq;
3128         }
3129
3130         err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle,
3131                                   DPNI_IRQ_INDEX, 1);
3132         if (err < 0) {
3133                 dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err);
3134                 goto free_irq;
3135         }
3136
3137         return 0;
3138
3139 free_irq:
3140         devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev);
3141 free_mc_irq:
3142         fsl_mc_free_irqs(ls_dev);
3143
3144         return err;
3145 }
3146
3147 static void add_ch_napi(struct dpaa2_eth_priv *priv)
3148 {
3149         int i;
3150         struct dpaa2_eth_channel *ch;
3151
3152         for (i = 0; i < priv->num_channels; i++) {
3153                 ch = priv->channel[i];
3154                 /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */
3155                 netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll,
3156                                NAPI_POLL_WEIGHT);
3157         }
3158 }
3159
3160 static void del_ch_napi(struct dpaa2_eth_priv *priv)
3161 {
3162         int i;
3163         struct dpaa2_eth_channel *ch;
3164
3165         for (i = 0; i < priv->num_channels; i++) {
3166                 ch = priv->channel[i];
3167                 netif_napi_del(&ch->napi);
3168         }
3169 }
3170
3171 static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev)
3172 {
3173         struct device *dev;
3174         struct net_device *net_dev = NULL;
3175         struct dpaa2_eth_priv *priv = NULL;
3176         int err = 0;
3177
3178         dev = &dpni_dev->dev;
3179
3180         /* Net device */
3181         net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_TX_QUEUES);
3182         if (!net_dev) {
3183                 dev_err(dev, "alloc_etherdev_mq() failed\n");
3184                 return -ENOMEM;
3185         }
3186
3187         SET_NETDEV_DEV(net_dev, dev);
3188         dev_set_drvdata(dev, net_dev);
3189
3190         priv = netdev_priv(net_dev);
3191         priv->net_dev = net_dev;
3192
3193         priv->iommu_domain = iommu_get_domain_for_dev(dev);
3194
3195         /* Obtain a MC portal */
3196         err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL,
3197                                      &priv->mc_io);
3198         if (err) {
3199                 if (err == -ENXIO)
3200                         err = -EPROBE_DEFER;
3201                 else
3202                         dev_err(dev, "MC portal allocation failed\n");
3203                 goto err_portal_alloc;
3204         }
3205
3206         /* MC objects initialization and configuration */
3207         err = setup_dpni(dpni_dev);
3208         if (err)
3209                 goto err_dpni_setup;
3210
3211         err = setup_dpio(priv);
3212         if (err)
3213                 goto err_dpio_setup;
3214
3215         setup_fqs(priv);
3216
3217         err = setup_dpbp(priv);
3218         if (err)
3219                 goto err_dpbp_setup;
3220
3221         err = bind_dpni(priv);
3222         if (err)
3223                 goto err_bind;
3224
3225         /* Add a NAPI context for each channel */
3226         add_ch_napi(priv);
3227
3228         /* Percpu statistics */
3229         priv->percpu_stats = alloc_percpu(*priv->percpu_stats);
3230         if (!priv->percpu_stats) {
3231                 dev_err(dev, "alloc_percpu(percpu_stats) failed\n");
3232                 err = -ENOMEM;
3233                 goto err_alloc_percpu_stats;
3234         }
3235         priv->percpu_extras = alloc_percpu(*priv->percpu_extras);
3236         if (!priv->percpu_extras) {
3237                 dev_err(dev, "alloc_percpu(percpu_extras) failed\n");
3238                 err = -ENOMEM;
3239                 goto err_alloc_percpu_extras;
3240         }
3241
3242         err = netdev_init(net_dev);
3243         if (err)
3244                 goto err_netdev_init;
3245
3246         /* Configure checksum offload based on current interface flags */
3247         err = set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM));
3248         if (err)
3249                 goto err_csum;
3250
3251         err = set_tx_csum(priv, !!(net_dev->features &
3252                                    (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)));
3253         if (err)
3254                 goto err_csum;
3255
3256         err = alloc_rings(priv);
3257         if (err)
3258                 goto err_alloc_rings;
3259
3260         err = setup_irqs(dpni_dev);
3261         if (err) {
3262                 netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n");
3263                 priv->poll_thread = kthread_run(poll_link_state, priv,
3264                                                 "%s_poll_link", net_dev->name);
3265                 if (IS_ERR(priv->poll_thread)) {
3266                         dev_err(dev, "Error starting polling thread\n");
3267                         goto err_poll_thread;
3268                 }
3269                 priv->do_link_poll = true;
3270         }
3271
3272         err = register_netdev(net_dev);
3273         if (err < 0) {
3274                 dev_err(dev, "register_netdev() failed\n");
3275                 goto err_netdev_reg;
3276         }
3277
3278 #ifdef CONFIG_DEBUG_FS
3279         dpaa2_dbg_add(priv);
3280 #endif
3281
3282         dev_info(dev, "Probed interface %s\n", net_dev->name);
3283         return 0;
3284
3285 err_netdev_reg:
3286         if (priv->do_link_poll)
3287                 kthread_stop(priv->poll_thread);
3288         else
3289                 fsl_mc_free_irqs(dpni_dev);
3290 err_poll_thread:
3291         free_rings(priv);
3292 err_alloc_rings:
3293 err_csum:
3294 err_netdev_init:
3295         free_percpu(priv->percpu_extras);
3296 err_alloc_percpu_extras:
3297         free_percpu(priv->percpu_stats);
3298 err_alloc_percpu_stats:
3299         del_ch_napi(priv);
3300 err_bind:
3301         free_dpbp(priv);
3302 err_dpbp_setup:
3303         free_dpio(priv);
3304 err_dpio_setup:
3305         free_dpni(priv);
3306 err_dpni_setup:
3307         fsl_mc_portal_free(priv->mc_io);
3308 err_portal_alloc:
3309         dev_set_drvdata(dev, NULL);
3310         free_netdev(net_dev);
3311
3312         return err;
3313 }
3314
3315 static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev)
3316 {
3317         struct device *dev;
3318         struct net_device *net_dev;
3319         struct dpaa2_eth_priv *priv;
3320
3321         dev = &ls_dev->dev;
3322         net_dev = dev_get_drvdata(dev);
3323         priv = netdev_priv(net_dev);
3324
3325 #ifdef CONFIG_DEBUG_FS
3326         dpaa2_dbg_remove(priv);
3327 #endif
3328         unregister_netdev(net_dev);
3329
3330         if (priv->do_link_poll)
3331                 kthread_stop(priv->poll_thread);
3332         else
3333                 fsl_mc_free_irqs(ls_dev);
3334
3335         free_rings(priv);
3336         free_percpu(priv->percpu_stats);
3337         free_percpu(priv->percpu_extras);
3338
3339         del_ch_napi(priv);
3340         free_dpbp(priv);
3341         free_dpio(priv);
3342         free_dpni(priv);
3343
3344         fsl_mc_portal_free(priv->mc_io);
3345
3346         free_netdev(net_dev);
3347
3348         dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name);
3349
3350         return 0;
3351 }
3352
3353 static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = {
3354         {
3355                 .vendor = FSL_MC_VENDOR_FREESCALE,
3356                 .obj_type = "dpni",
3357         },
3358         { .vendor = 0x0 }
3359 };
3360 MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table);
3361
3362 static struct fsl_mc_driver dpaa2_eth_driver = {
3363         .driver = {
3364                 .name = KBUILD_MODNAME,
3365                 .owner = THIS_MODULE,
3366         },
3367         .probe = dpaa2_eth_probe,
3368         .remove = dpaa2_eth_remove,
3369         .match_id_table = dpaa2_eth_match_id_table
3370 };
3371
3372 static int __init dpaa2_eth_driver_init(void)
3373 {
3374         int err;
3375
3376         dpaa2_eth_dbg_init();
3377         err = fsl_mc_driver_register(&dpaa2_eth_driver);
3378         if (err) {
3379                 dpaa2_eth_dbg_exit();
3380                 return err;
3381         }
3382
3383         return 0;
3384 }
3385
3386 static void __exit dpaa2_eth_driver_exit(void)
3387 {
3388         dpaa2_eth_dbg_exit();
3389         fsl_mc_driver_unregister(&dpaa2_eth_driver);
3390 }
3391
3392 module_init(dpaa2_eth_driver_init);
3393 module_exit(dpaa2_eth_driver_exit);