drivers/net/ethernet/pensando/ionic/ionic_lif.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ethtool.h>
   5 #include <linux/printk.h>
   6 #include <linux/dynamic_debug.h>
   7 #include <linux/netdevice.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/if_vlan.h>
  10 #include <linux/rtnetlink.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/pci.h>
  13 #include <linux/cpumask.h>
  14 #include <linux/crash_dump.h>
  15 #include <linux/vmalloc.h>
  16
  17 #include "ionic.h"
  18 #include "ionic_bus.h"
  19 #include "ionic_dev.h"
  20 #include "ionic_lif.h"
  21 #include "ionic_txrx.h"
  22 #include "ionic_ethtool.h"
  23 #include "ionic_debugfs.h"
  24
  25 /* queuetype support level */
  26 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
  27         [IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
  28         [IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
  29         [IONIC_QTYPE_RXQ]     = 0,   /* 0 = Base version with CQ+SG support */
  30         [IONIC_QTYPE_TXQ]     = 1,   /* 0 = Base version with CQ+SG support
  31                                       * 1 =   ... with Tx SG version 1
  32                                       */
  33 };
  34
  35 static void ionic_link_status_check(struct ionic_lif *lif);
  36 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
  37 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
  38 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
  39
  40 static void ionic_txrx_deinit(struct ionic_lif *lif);
  41 static int ionic_txrx_init(struct ionic_lif *lif);
  42 static int ionic_start_queues(struct ionic_lif *lif);
  43 static void ionic_stop_queues(struct ionic_lif *lif);
  44 static void ionic_lif_queue_identify(struct ionic_lif *lif);
  45
  46 static void ionic_dim_work(struct work_struct *work)
  47 {
  48         struct dim *dim = container_of(work, struct dim, work);
  49         struct dim_cq_moder cur_moder;
  50         struct ionic_qcq *qcq;
  51         u32 new_coal;
  52
  53         cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
  54         qcq = container_of(dim, struct ionic_qcq, dim);
  55         new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
  56         new_coal = new_coal ? new_coal : 1;
  57
  58         if (qcq->intr.dim_coal_hw != new_coal) {
  59                 unsigned int qi = qcq->cq.bound_q->index;
  60                 struct ionic_lif *lif = qcq->q.lif;
  61
  62                 qcq->intr.dim_coal_hw = new_coal;
  63
  64                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
  65                                      lif->rxqcqs[qi]->intr.index,
  66                                      qcq->intr.dim_coal_hw);
  67         }
  68
  69         dim->state = DIM_START_MEASURE;
  70 }
  71
  72 static void ionic_lif_deferred_work(struct work_struct *work)
  73 {
  74         struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
  75         struct ionic_deferred *def = &lif->deferred;
  76         struct ionic_deferred_work *w = NULL;
  77
  78         do {
  79                 spin_lock_bh(&def->lock);
  80                 if (!list_empty(&def->list)) {
  81                         w = list_first_entry(&def->list,
  82                                              struct ionic_deferred_work, list);
  83                         list_del(&w->list);
  84                 }
  85                 spin_unlock_bh(&def->lock);
  86
  87                 if (!w)
  88                         break;
  89
  90                 switch (w->type) {
  91                 case IONIC_DW_TYPE_RX_MODE:
  92                         ionic_lif_rx_mode(lif);
  93                         break;
  94                 case IONIC_DW_TYPE_LINK_STATUS:
  95                         ionic_link_status_check(lif);
  96                         break;
  97                 case IONIC_DW_TYPE_LIF_RESET:
  98                         if (w->fw_status) {
  99                                 ionic_lif_handle_fw_up(lif);
 100                         } else {
 101                                 ionic_lif_handle_fw_down(lif);
 102
 103                                 /* Fire off another watchdog to see
 104                                  * if the FW is already back rather than
 105                                  * waiting another whole cycle
 106                                  */
 107                                 mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
 108                         }
 109                         break;
 110                 default:
 111                         break;
 112                 }
 113                 kfree(w);
 114                 w = NULL;
 115         } while (true);
 116 }
 117
 118 void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
 119                                 struct ionic_deferred_work *work)
 120 {
 121         spin_lock_bh(&def->lock);
 122         list_add_tail(&work->list, &def->list);
 123         spin_unlock_bh(&def->lock);
 124         schedule_work(&def->work);
 125 }
 126
 127 static void ionic_link_status_check(struct ionic_lif *lif)
 128 {
 129         struct net_device *netdev = lif->netdev;
 130         u16 link_status;
 131         bool link_up;
 132
 133         if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 134                 return;
 135
 136         /* Don't put carrier back up if we're in a broken state */
 137         if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
 138                 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 139                 return;
 140         }
 141
 142         link_status = le16_to_cpu(lif->info->status.link_status);
 143         link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 144
 145         if (link_up) {
 146                 int err = 0;
 147
 148                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 149                         mutex_lock(&lif->queue_lock);
 150                         err = ionic_start_queues(lif);
 151                         if (err && err != -EBUSY) {
 152                                 netdev_err(lif->netdev,
 153                                            "Failed to start queues: %d\n", err);
 154                                 set_bit(IONIC_LIF_F_BROKEN, lif->state);
 155                                 netif_carrier_off(lif->netdev);
 156                         }
 157                         mutex_unlock(&lif->queue_lock);
 158                 }
 159
 160                 if (!err && !netif_carrier_ok(netdev)) {
 161                         ionic_port_identify(lif->ionic);
 162                         netdev_info(netdev, "Link up - %d Gbps\n",
 163                                     le32_to_cpu(lif->info->status.link_speed) / 1000);
 164                         netif_carrier_on(netdev);
 165                 }
 166         } else {
 167                 if (netif_carrier_ok(netdev)) {
 168                         netdev_info(netdev, "Link down\n");
 169                         netif_carrier_off(netdev);
 170                 }
 171
 172                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 173                         mutex_lock(&lif->queue_lock);
 174                         ionic_stop_queues(lif);
 175                         mutex_unlock(&lif->queue_lock);
 176                 }
 177         }
 178
 179         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 180 }
 181
 182 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
 183 {
 184         struct ionic_deferred_work *work;
 185
 186         /* we only need one request outstanding at a time */
 187         if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 188                 return;
 189
 190         if (!can_sleep) {
 191                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
 192                 if (!work) {
 193                         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 194                         return;
 195                 }
 196
 197                 work->type = IONIC_DW_TYPE_LINK_STATUS;
 198                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 199         } else {
 200                 ionic_link_status_check(lif);
 201         }
 202 }
 203
 204 static void ionic_napi_deadline(struct timer_list *timer)
 205 {
 206         struct ionic_qcq *qcq = container_of(timer, struct ionic_qcq, napi_deadline);
 207
 208         napi_schedule(&qcq->napi);
 209 }
 210
 211 static irqreturn_t ionic_isr(int irq, void *data)
 212 {
 213         struct napi_struct *napi = data;
 214
 215         napi_schedule_irqoff(napi);
 216
 217         return IRQ_HANDLED;
 218 }
 219
 220 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
 221 {
 222         struct ionic_intr_info *intr = &qcq->intr;
 223         struct device *dev = lif->ionic->dev;
 224         struct ionic_queue *q = &qcq->q;
 225         const char *name;
 226
 227         if (lif->registered)
 228                 name = lif->netdev->name;
 229         else
 230                 name = dev_name(dev);
 231
 232         snprintf(intr->name, sizeof(intr->name),
 233                  "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
 234
 235         return devm_request_irq(dev, intr->vector, ionic_isr,
 236                                 0, intr->name, &qcq->napi);
 237 }
 238
 239 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 240 {
 241         struct ionic *ionic = lif->ionic;
 242         int index;
 243
 244         index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
 245         if (index == ionic->nintrs) {
 246                 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
 247                             __func__, index, ionic->nintrs);
 248                 return -ENOSPC;
 249         }
 250
 251         set_bit(index, ionic->intrs);
 252         ionic_intr_init(&ionic->idev, intr, index);
 253
 254         return 0;
 255 }
 256
 257 static void ionic_intr_free(struct ionic *ionic, int index)
 258 {
 259         if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
 260                 clear_bit(index, ionic->intrs);
 261 }
 262
 263 static int ionic_qcq_enable(struct ionic_qcq *qcq)
 264 {
 265         struct ionic_queue *q = &qcq->q;
 266         struct ionic_lif *lif = q->lif;
 267         struct ionic_dev *idev;
 268         struct device *dev;
 269
 270         struct ionic_admin_ctx ctx = {
 271                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 272                 .cmd.q_control = {
 273                         .opcode = IONIC_CMD_Q_CONTROL,
 274                         .lif_index = cpu_to_le16(lif->index),
 275                         .type = q->type,
 276                         .index = cpu_to_le32(q->index),
 277                         .oper = IONIC_Q_ENABLE,
 278                 },
 279         };
 280         int ret;
 281
 282         idev = &lif->ionic->idev;
 283         dev = lif->ionic->dev;
 284
 285         dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
 286                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 287
 288         if (qcq->flags & IONIC_QCQ_F_INTR)
 289                 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
 290
 291         ret = ionic_adminq_post_wait(lif, &ctx);
 292         if (ret)
 293                 return ret;
 294
 295         if (qcq->napi.poll)
 296                 napi_enable(&qcq->napi);
 297
 298         if (qcq->flags & IONIC_QCQ_F_INTR) {
 299                 irq_set_affinity_hint(qcq->intr.vector,
 300                                       &qcq->intr.affinity_mask);
 301                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 302                                 IONIC_INTR_MASK_CLEAR);
 303         }
 304
 305         return 0;
 306 }
 307
 308 static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
 309 {
 310         struct ionic_queue *q;
 311
 312         struct ionic_admin_ctx ctx = {
 313                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 314                 .cmd.q_control = {
 315                         .opcode = IONIC_CMD_Q_CONTROL,
 316                         .oper = IONIC_Q_DISABLE,
 317                 },
 318         };
 319
 320         if (!qcq) {
 321                 netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
 322                 return -ENXIO;
 323         }
 324
 325         q = &qcq->q;
 326
 327         if (qcq->flags & IONIC_QCQ_F_INTR) {
 328                 struct ionic_dev *idev = &lif->ionic->idev;
 329
 330                 cancel_work_sync(&qcq->dim.work);
 331                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 332                                 IONIC_INTR_MASK_SET);
 333                 synchronize_irq(qcq->intr.vector);
 334                 irq_set_affinity_hint(qcq->intr.vector, NULL);
 335                 napi_disable(&qcq->napi);
 336                 del_timer_sync(&qcq->napi_deadline);
 337         }
 338
 339         /* If there was a previous fw communcation error, don't bother with
 340          * sending the adminq command and just return the same error value.
 341          */
 342         if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
 343                 return fw_err;
 344
 345         ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
 346         ctx.cmd.q_control.type = q->type;
 347         ctx.cmd.q_control.index = cpu_to_le32(q->index);
 348         dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
 349                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 350
 351         return ionic_adminq_post_wait(lif, &ctx);
 352 }
 353
 354 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 355 {
 356         struct ionic_dev *idev = &lif->ionic->idev;
 357
 358         if (!qcq)
 359                 return;
 360
 361         if (!(qcq->flags & IONIC_QCQ_F_INITED))
 362                 return;
 363
 364         if (qcq->flags & IONIC_QCQ_F_INTR) {
 365                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 366                                 IONIC_INTR_MASK_SET);
 367                 netif_napi_del(&qcq->napi);
 368         }
 369
 370         qcq->flags &= ~IONIC_QCQ_F_INITED;
 371 }
 372
 373 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 374 {
 375         if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
 376                 return;
 377
 378         irq_set_affinity_hint(qcq->intr.vector, NULL);
 379         devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
 380         qcq->intr.vector = 0;
 381         ionic_intr_free(lif->ionic, qcq->intr.index);
 382         qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 383 }
 384
 385 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 386 {
 387         struct device *dev = lif->ionic->dev;
 388
 389         if (!qcq)
 390                 return;
 391
 392         ionic_debugfs_del_qcq(qcq);
 393
 394         if (qcq->q_base) {
 395                 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
 396                 qcq->q_base = NULL;
 397                 qcq->q_base_pa = 0;
 398         }
 399
 400         if (qcq->cq_base) {
 401                 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
 402                 qcq->cq_base = NULL;
 403                 qcq->cq_base_pa = 0;
 404         }
 405
 406         if (qcq->sg_base) {
 407                 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
 408                 qcq->sg_base = NULL;
 409                 qcq->sg_base_pa = 0;
 410         }
 411
 412         ionic_qcq_intr_free(lif, qcq);
 413
 414         if (qcq->cq.info) {
 415                 vfree(qcq->cq.info);
 416                 qcq->cq.info = NULL;
 417         }
 418         if (qcq->q.info) {
 419                 vfree(qcq->q.info);
 420                 qcq->q.info = NULL;
 421         }
 422 }
 423
 424 static void ionic_qcqs_free(struct ionic_lif *lif)
 425 {
 426         struct device *dev = lif->ionic->dev;
 427         struct ionic_qcq *adminqcq;
 428         unsigned long irqflags;
 429
 430         if (lif->notifyqcq) {
 431                 ionic_qcq_free(lif, lif->notifyqcq);
 432                 devm_kfree(dev, lif->notifyqcq);
 433                 lif->notifyqcq = NULL;
 434         }
 435
 436         if (lif->adminqcq) {
 437                 spin_lock_irqsave(&lif->adminq_lock, irqflags);
 438                 adminqcq = READ_ONCE(lif->adminqcq);
 439                 lif->adminqcq = NULL;
 440                 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
 441                 if (adminqcq) {
 442                         ionic_qcq_free(lif, adminqcq);
 443                         devm_kfree(dev, adminqcq);
 444                 }
 445         }
 446
 447         if (lif->rxqcqs) {
 448                 devm_kfree(dev, lif->rxqstats);
 449                 lif->rxqstats = NULL;
 450                 devm_kfree(dev, lif->rxqcqs);
 451                 lif->rxqcqs = NULL;
 452         }
 453
 454         if (lif->txqcqs) {
 455                 devm_kfree(dev, lif->txqstats);
 456                 lif->txqstats = NULL;
 457                 devm_kfree(dev, lif->txqcqs);
 458                 lif->txqcqs = NULL;
 459         }
 460 }
 461
 462 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
 463                                       struct ionic_qcq *n_qcq)
 464 {
 465         if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
 466                 ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
 467                 n_qcq->flags &= ~IONIC_QCQ_F_INTR;
 468         }
 469
 470         n_qcq->intr.vector = src_qcq->intr.vector;
 471         n_qcq->intr.index = src_qcq->intr.index;
 472         n_qcq->napi_qcq = src_qcq->napi_qcq;
 473 }
 474
 475 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
 476 {
 477         int err;
 478
 479         if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
 480                 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 481                 return 0;
 482         }
 483
 484         err = ionic_intr_alloc(lif, &qcq->intr);
 485         if (err) {
 486                 netdev_warn(lif->netdev, "no intr for %s: %d\n",
 487                             qcq->q.name, err);
 488                 goto err_out;
 489         }
 490
 491         err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
 492         if (err < 0) {
 493                 netdev_warn(lif->netdev, "no vector for %s: %d\n",
 494                             qcq->q.name, err);
 495                 goto err_out_free_intr;
 496         }
 497         qcq->intr.vector = err;
 498         ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
 499                                IONIC_INTR_MASK_SET);
 500
 501         err = ionic_request_irq(lif, qcq);
 502         if (err) {
 503                 netdev_warn(lif->netdev, "irq request failed %d\n", err);
 504                 goto err_out_free_intr;
 505         }
 506
 507         /* try to get the irq on the local numa node first */
 508         qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
 509                                              dev_to_node(lif->ionic->dev));
 510         if (qcq->intr.cpu != -1)
 511                 cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
 512
 513         netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
 514         return 0;
 515
 516 err_out_free_intr:
 517         ionic_intr_free(lif->ionic, qcq->intr.index);
 518 err_out:
 519         return err;
 520 }
 521
 522 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
 523                            unsigned int index,
 524                            const char *name, unsigned int flags,
 525                            unsigned int num_descs, unsigned int desc_size,
 526                            unsigned int cq_desc_size,
 527                            unsigned int sg_desc_size,
 528                            unsigned int pid, struct ionic_qcq **qcq)
 529 {
 530         struct ionic_dev *idev = &lif->ionic->idev;
 531         struct device *dev = lif->ionic->dev;
 532         void *q_base, *cq_base, *sg_base;
 533         dma_addr_t cq_base_pa = 0;
 534         dma_addr_t sg_base_pa = 0;
 535         dma_addr_t q_base_pa = 0;
 536         struct ionic_qcq *new;
 537         int err;
 538
 539         *qcq = NULL;
 540
 541         new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
 542         if (!new) {
 543                 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
 544                 err = -ENOMEM;
 545                 goto err_out;
 546         }
 547
 548         new->q.dev = dev;
 549         new->flags = flags;
 550
 551         new->q.info = vzalloc(num_descs * sizeof(*new->q.info));
 552         if (!new->q.info) {
 553                 netdev_err(lif->netdev, "Cannot allocate queue info\n");
 554                 err = -ENOMEM;
 555                 goto err_out_free_qcq;
 556         }
 557
 558         new->q.type = type;
 559         new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
 560
 561         err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
 562                            desc_size, sg_desc_size, pid);
 563         if (err) {
 564                 netdev_err(lif->netdev, "Cannot initialize queue\n");
 565                 goto err_out_free_q_info;
 566         }
 567
 568         err = ionic_alloc_qcq_interrupt(lif, new);
 569         if (err)
 570                 goto err_out;
 571
 572         new->cq.info = vzalloc(num_descs * sizeof(*new->cq.info));
 573         if (!new->cq.info) {
 574                 netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
 575                 err = -ENOMEM;
 576                 goto err_out_free_irq;
 577         }
 578
 579         err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
 580         if (err) {
 581                 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
 582                 goto err_out_free_cq_info;
 583         }
 584
 585         if (flags & IONIC_QCQ_F_NOTIFYQ) {
 586                 int q_size;
 587
 588                 /* q & cq need to be contiguous in NotifyQ, so alloc it all in q
 589                  * and don't alloc qc.  We leave new->qc_size and new->qc_base
 590                  * as 0 to be sure we don't try to free it later.
 591                  */
 592                 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
 593                 new->q_size = PAGE_SIZE + q_size +
 594                               ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
 595                 new->q_base = dma_alloc_coherent(dev, new->q_size,
 596                                                  &new->q_base_pa, GFP_KERNEL);
 597                 if (!new->q_base) {
 598                         netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
 599                         err = -ENOMEM;
 600                         goto err_out_free_cq_info;
 601                 }
 602                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 603                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 604                 ionic_q_map(&new->q, q_base, q_base_pa);
 605
 606                 cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
 607                 cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
 608                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 609                 ionic_cq_bind(&new->cq, &new->q);
 610         } else {
 611                 new->q_size = PAGE_SIZE + (num_descs * desc_size);
 612                 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
 613                                                  GFP_KERNEL);
 614                 if (!new->q_base) {
 615                         netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
 616                         err = -ENOMEM;
 617                         goto err_out_free_cq_info;
 618                 }
 619                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 620                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 621                 ionic_q_map(&new->q, q_base, q_base_pa);
 622
 623                 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
 624                 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
 625                                                   GFP_KERNEL);
 626                 if (!new->cq_base) {
 627                         netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
 628                         err = -ENOMEM;
 629                         goto err_out_free_q;
 630                 }
 631                 cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
 632                 cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
 633                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 634                 ionic_cq_bind(&new->cq, &new->q);
 635         }
 636
 637         if (flags & IONIC_QCQ_F_SG) {
 638                 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
 639                 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
 640                                                   GFP_KERNEL);
 641                 if (!new->sg_base) {
 642                         netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
 643                         err = -ENOMEM;
 644                         goto err_out_free_cq;
 645                 }
 646                 sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
 647                 sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
 648                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 649         }
 650
 651         INIT_WORK(&new->dim.work, ionic_dim_work);
 652         new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 653
 654         *qcq = new;
 655
 656         return 0;
 657
 658 err_out_free_cq:
 659         dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
 660 err_out_free_q:
 661         dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
 662 err_out_free_cq_info:
 663         vfree(new->cq.info);
 664 err_out_free_irq:
 665         if (flags & IONIC_QCQ_F_INTR) {
 666                 devm_free_irq(dev, new->intr.vector, &new->napi);
 667                 ionic_intr_free(lif->ionic, new->intr.index);
 668         }
 669 err_out_free_q_info:
 670         vfree(new->q.info);
 671 err_out_free_qcq:
 672         devm_kfree(dev, new);
 673 err_out:
 674         dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
 675         return err;
 676 }
 677
 678 static int ionic_qcqs_alloc(struct ionic_lif *lif)
 679 {
 680         struct device *dev = lif->ionic->dev;
 681         unsigned int flags;
 682         int err;
 683
 684         flags = IONIC_QCQ_F_INTR;
 685         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 686                               IONIC_ADMINQ_LENGTH,
 687                               sizeof(struct ionic_admin_cmd),
 688                               sizeof(struct ionic_admin_comp),
 689                               0, lif->kern_pid, &lif->adminqcq);
 690         if (err)
 691                 return err;
 692         ionic_debugfs_add_qcq(lif, lif->adminqcq);
 693
 694         if (lif->ionic->nnqs_per_lif) {
 695                 flags = IONIC_QCQ_F_NOTIFYQ;
 696                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
 697                                       flags, IONIC_NOTIFYQ_LENGTH,
 698                                       sizeof(struct ionic_notifyq_cmd),
 699                                       sizeof(union ionic_notifyq_comp),
 700                                       0, lif->kern_pid, &lif->notifyqcq);
 701                 if (err)
 702                         goto err_out;
 703                 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
 704
 705                 /* Let the notifyq ride on the adminq interrupt */
 706                 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
 707         }
 708
 709         err = -ENOMEM;
 710         lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
 711                                    sizeof(*lif->txqcqs), GFP_KERNEL);
 712         if (!lif->txqcqs)
 713                 goto err_out;
 714         lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
 715                                    sizeof(*lif->rxqcqs), GFP_KERNEL);
 716         if (!lif->rxqcqs)
 717                 goto err_out;
 718
 719         lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
 720                                      sizeof(*lif->txqstats), GFP_KERNEL);
 721         if (!lif->txqstats)
 722                 goto err_out;
 723         lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
 724                                      sizeof(*lif->rxqstats), GFP_KERNEL);
 725         if (!lif->rxqstats)
 726                 goto err_out;
 727
 728         return 0;
 729
 730 err_out:
 731         ionic_qcqs_free(lif);
 732         return err;
 733 }
 734
 735 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
 736 {
 737         qcq->q.tail_idx = 0;
 738         qcq->q.head_idx = 0;
 739         qcq->cq.tail_idx = 0;
 740         qcq->cq.done_color = 1;
 741         memset(qcq->q_base, 0, qcq->q_size);
 742         memset(qcq->cq_base, 0, qcq->cq_size);
 743         memset(qcq->sg_base, 0, qcq->sg_size);
 744 }
 745
 746 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 747 {
 748         struct device *dev = lif->ionic->dev;
 749         struct ionic_queue *q = &qcq->q;
 750         struct ionic_cq *cq = &qcq->cq;
 751         struct ionic_admin_ctx ctx = {
 752                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 753                 .cmd.q_init = {
 754                         .opcode = IONIC_CMD_Q_INIT,
 755                         .lif_index = cpu_to_le16(lif->index),
 756                         .type = q->type,
 757                         .ver = lif->qtype_info[q->type].version,
 758                         .index = cpu_to_le32(q->index),
 759                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 760                                              IONIC_QINIT_F_SG),
 761                         .pid = cpu_to_le16(q->pid),
 762                         .ring_size = ilog2(q->num_descs),
 763                         .ring_base = cpu_to_le64(q->base_pa),
 764                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 765                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 766                         .features = cpu_to_le64(q->features),
 767                 },
 768         };
 769         unsigned int intr_index;
 770         int err;
 771
 772         intr_index = qcq->intr.index;
 773
 774         ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
 775
 776         dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
 777         dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
 778         dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 779         dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 780         dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 781         dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
 782         dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 783
 784         ionic_qcq_sanitize(qcq);
 785
 786         err = ionic_adminq_post_wait(lif, &ctx);
 787         if (err)
 788                 return err;
 789
 790         q->hw_type = ctx.comp.q_init.hw_type;
 791         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 792         q->dbval = IONIC_DBELL_QID(q->hw_index);
 793
 794         dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
 795         dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 796
 797         q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
 798         q->dbell_jiffies = jiffies;
 799
 800         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
 801                 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
 802                 qcq->napi_qcq = qcq;
 803                 timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
 804         }
 805
 806         qcq->flags |= IONIC_QCQ_F_INITED;
 807
 808         return 0;
 809 }
 810
 811 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 812 {
 813         struct device *dev = lif->ionic->dev;
 814         struct ionic_queue *q = &qcq->q;
 815         struct ionic_cq *cq = &qcq->cq;
 816         struct ionic_admin_ctx ctx = {
 817                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 818                 .cmd.q_init = {
 819                         .opcode = IONIC_CMD_Q_INIT,
 820                         .lif_index = cpu_to_le16(lif->index),
 821                         .type = q->type,
 822                         .ver = lif->qtype_info[q->type].version,
 823                         .index = cpu_to_le32(q->index),
 824                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 825                                              IONIC_QINIT_F_SG),
 826                         .intr_index = cpu_to_le16(cq->bound_intr->index),
 827                         .pid = cpu_to_le16(q->pid),
 828                         .ring_size = ilog2(q->num_descs),
 829                         .ring_base = cpu_to_le64(q->base_pa),
 830                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 831                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 832                         .features = cpu_to_le64(q->features),
 833                 },
 834         };
 835         int err;
 836
 837         dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
 838         dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
 839         dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 840         dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 841         dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 842         dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
 843         dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 844
 845         ionic_qcq_sanitize(qcq);
 846
 847         err = ionic_adminq_post_wait(lif, &ctx);
 848         if (err)
 849                 return err;
 850
 851         q->hw_type = ctx.comp.q_init.hw_type;
 852         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 853         q->dbval = IONIC_DBELL_QID(q->hw_index);
 854
 855         dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
 856         dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 857
 858         q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
 859         q->dbell_jiffies = jiffies;
 860
 861         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 862                 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
 863         else
 864                 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
 865
 866         qcq->napi_qcq = qcq;
 867         timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
 868
 869         qcq->flags |= IONIC_QCQ_F_INITED;
 870
 871         return 0;
 872 }
 873
 874 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
 875 {
 876         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 877         unsigned int txq_i, flags;
 878         struct ionic_qcq *txq;
 879         u64 features;
 880         int err;
 881
 882         if (lif->hwstamp_txq)
 883                 return 0;
 884
 885         features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
 886
 887         num_desc = IONIC_MIN_TXRX_DESC;
 888         desc_sz = sizeof(struct ionic_txq_desc);
 889         comp_sz = 2 * sizeof(struct ionic_txq_comp);
 890
 891         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
 892             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
 893                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
 894         else
 895                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
 896
 897         txq_i = lif->ionic->ntxqs_per_lif;
 898         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
 899
 900         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
 901                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 902                               lif->kern_pid, &txq);
 903         if (err)
 904                 goto err_qcq_alloc;
 905
 906         txq->q.features = features;
 907
 908         ionic_link_qcq_interrupts(lif->adminqcq, txq);
 909         ionic_debugfs_add_qcq(lif, txq);
 910
 911         lif->hwstamp_txq = txq;
 912
 913         if (netif_running(lif->netdev)) {
 914                 err = ionic_lif_txq_init(lif, txq);
 915                 if (err)
 916                         goto err_qcq_init;
 917
 918                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 919                         err = ionic_qcq_enable(txq);
 920                         if (err)
 921                                 goto err_qcq_enable;
 922                 }
 923         }
 924
 925         return 0;
 926
 927 err_qcq_enable:
 928         ionic_lif_qcq_deinit(lif, txq);
 929 err_qcq_init:
 930         lif->hwstamp_txq = NULL;
 931         ionic_debugfs_del_qcq(txq);
 932         ionic_qcq_free(lif, txq);
 933         devm_kfree(lif->ionic->dev, txq);
 934 err_qcq_alloc:
 935         return err;
 936 }
 937
 938 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
 939 {
 940         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 941         unsigned int rxq_i, flags;
 942         struct ionic_qcq *rxq;
 943         u64 features;
 944         int err;
 945
 946         if (lif->hwstamp_rxq)
 947                 return 0;
 948
 949         features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 950
 951         num_desc = IONIC_MIN_TXRX_DESC;
 952         desc_sz = sizeof(struct ionic_rxq_desc);
 953         comp_sz = 2 * sizeof(struct ionic_rxq_comp);
 954         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
 955
 956         rxq_i = lif->ionic->nrxqs_per_lif;
 957         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
 958
 959         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
 960                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 961                               lif->kern_pid, &rxq);
 962         if (err)
 963                 goto err_qcq_alloc;
 964
 965         rxq->q.features = features;
 966
 967         ionic_link_qcq_interrupts(lif->adminqcq, rxq);
 968         ionic_debugfs_add_qcq(lif, rxq);
 969
 970         lif->hwstamp_rxq = rxq;
 971
 972         if (netif_running(lif->netdev)) {
 973                 err = ionic_lif_rxq_init(lif, rxq);
 974                 if (err)
 975                         goto err_qcq_init;
 976
 977                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 978                         ionic_rx_fill(&rxq->q);
 979                         err = ionic_qcq_enable(rxq);
 980                         if (err)
 981                                 goto err_qcq_enable;
 982                 }
 983         }
 984
 985         return 0;
 986
 987 err_qcq_enable:
 988         ionic_lif_qcq_deinit(lif, rxq);
 989 err_qcq_init:
 990         lif->hwstamp_rxq = NULL;
 991         ionic_debugfs_del_qcq(rxq);
 992         ionic_qcq_free(lif, rxq);
 993         devm_kfree(lif->ionic->dev, rxq);
 994 err_qcq_alloc:
 995         return err;
 996 }
 997
 998 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
 999 {
1000         struct ionic_queue_params qparam;
1001
1002         ionic_init_queue_params(lif, &qparam);
1003
1004         if (rx_all)
1005                 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1006         else
1007                 qparam.rxq_features = 0;
1008
1009         /* if we're not running, just set the values and return */
1010         if (!netif_running(lif->netdev)) {
1011                 lif->rxq_features = qparam.rxq_features;
1012                 return 0;
1013         }
1014
1015         return ionic_reconfigure_queues(lif, &qparam);
1016 }
1017
1018 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1019 {
1020         struct ionic_admin_ctx ctx = {
1021                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1022                 .cmd.lif_setattr = {
1023                         .opcode = IONIC_CMD_LIF_SETATTR,
1024                         .index = cpu_to_le16(lif->index),
1025                         .attr = IONIC_LIF_ATTR_TXSTAMP,
1026                         .txstamp_mode = cpu_to_le16(txstamp_mode),
1027                 },
1028         };
1029
1030         return ionic_adminq_post_wait(lif, &ctx);
1031 }
1032
1033 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1034 {
1035         struct ionic_admin_ctx ctx = {
1036                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1037                 .cmd.rx_filter_del = {
1038                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1039                         .lif_index = cpu_to_le16(lif->index),
1040                 },
1041         };
1042         struct ionic_rx_filter *f;
1043         u32 filter_id;
1044         int err;
1045
1046         spin_lock_bh(&lif->rx_filters.lock);
1047
1048         f = ionic_rx_filter_rxsteer(lif);
1049         if (!f) {
1050                 spin_unlock_bh(&lif->rx_filters.lock);
1051                 return;
1052         }
1053
1054         filter_id = f->filter_id;
1055         ionic_rx_filter_free(lif, f);
1056
1057         spin_unlock_bh(&lif->rx_filters.lock);
1058
1059         netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1060
1061         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1062
1063         err = ionic_adminq_post_wait(lif, &ctx);
1064         if (err && err != -EEXIST)
1065                 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1066 }
1067
1068 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1069 {
1070         struct ionic_admin_ctx ctx = {
1071                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1072                 .cmd.rx_filter_add = {
1073                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1074                         .lif_index = cpu_to_le16(lif->index),
1075                         .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1076                         .pkt_class = cpu_to_le64(pkt_class),
1077                 },
1078         };
1079         u8 qtype;
1080         u32 qid;
1081         int err;
1082
1083         if (!lif->hwstamp_rxq)
1084                 return -EINVAL;
1085
1086         qtype = lif->hwstamp_rxq->q.type;
1087         ctx.cmd.rx_filter_add.qtype = qtype;
1088
1089         qid = lif->hwstamp_rxq->q.index;
1090         ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1091
1092         netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1093         err = ionic_adminq_post_wait(lif, &ctx);
1094         if (err && err != -EEXIST)
1095                 return err;
1096
1097         spin_lock_bh(&lif->rx_filters.lock);
1098         err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
1099         spin_unlock_bh(&lif->rx_filters.lock);
1100
1101         return err;
1102 }
1103
1104 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1105 {
1106         ionic_lif_del_hwstamp_rxfilt(lif);
1107
1108         if (!pkt_class)
1109                 return 0;
1110
1111         return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1112 }
1113
1114 static bool ionic_notifyq_service(struct ionic_cq *cq,
1115                                   struct ionic_cq_info *cq_info)
1116 {
1117         union ionic_notifyq_comp *comp = cq_info->cq_desc;
1118         struct ionic_deferred_work *work;
1119         struct net_device *netdev;
1120         struct ionic_queue *q;
1121         struct ionic_lif *lif;
1122         u64 eid;
1123
1124         q = cq->bound_q;
1125         lif = q->info[0].cb_arg;
1126         netdev = lif->netdev;
1127         eid = le64_to_cpu(comp->event.eid);
1128
1129         /* Have we run out of new completions to process? */
1130         if ((s64)(eid - lif->last_eid) <= 0)
1131                 return false;
1132
1133         lif->last_eid = eid;
1134
1135         dev_dbg(lif->ionic->dev, "notifyq event:\n");
1136         dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1137                          comp, sizeof(*comp), true);
1138
1139         switch (le16_to_cpu(comp->event.ecode)) {
1140         case IONIC_EVENT_LINK_CHANGE:
1141                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1142                 break;
1143         case IONIC_EVENT_RESET:
1144                 if (lif->ionic->idev.fw_status_ready &&
1145                     !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
1146                     !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
1147                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1148                         if (!work) {
1149                                 netdev_err(lif->netdev, "Reset event dropped\n");
1150                                 clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
1151                         } else {
1152                                 work->type = IONIC_DW_TYPE_LIF_RESET;
1153                                 ionic_lif_deferred_enqueue(&lif->deferred, work);
1154                         }
1155                 }
1156                 break;
1157         default:
1158                 netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1159                             comp->event.ecode, eid);
1160                 break;
1161         }
1162
1163         return true;
1164 }
1165
1166 static bool ionic_adminq_service(struct ionic_cq *cq,
1167                                  struct ionic_cq_info *cq_info)
1168 {
1169         struct ionic_admin_comp *comp = cq_info->cq_desc;
1170
1171         if (!color_match(comp->color, cq->done_color))
1172                 return false;
1173
1174         ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1175
1176         return true;
1177 }
1178
1179 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1180 {
1181         struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1182         struct ionic_lif *lif = napi_to_cq(napi)->lif;
1183         struct ionic_dev *idev = &lif->ionic->idev;
1184         unsigned long irqflags;
1185         unsigned int flags = 0;
1186         bool resched = false;
1187         int rx_work = 0;
1188         int tx_work = 0;
1189         int n_work = 0;
1190         int a_work = 0;
1191         int work_done;
1192         int credits;
1193
1194         if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1195                 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1196                                           ionic_notifyq_service, NULL, NULL);
1197
1198         spin_lock_irqsave(&lif->adminq_lock, irqflags);
1199         if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1200                 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1201                                           ionic_adminq_service, NULL, NULL);
1202         spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1203
1204         if (lif->hwstamp_rxq)
1205                 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1206                                            ionic_rx_service, NULL, NULL);
1207
1208         if (lif->hwstamp_txq)
1209                 tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1210                                            ionic_tx_service, NULL, NULL);
1211
1212         work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1213         if (work_done < budget && napi_complete_done(napi, work_done)) {
1214                 flags |= IONIC_INTR_CRED_UNMASK;
1215                 intr->rearm_count++;
1216         }
1217
1218         if (work_done || flags) {
1219                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1220                 credits = n_work + a_work + rx_work + tx_work;
1221                 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1222         }
1223
1224         if (!a_work && ionic_adminq_poke_doorbell(&lif->adminqcq->q))
1225                 resched = true;
1226         if (lif->hwstamp_rxq && !rx_work && ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q))
1227                 resched = true;
1228         if (lif->hwstamp_txq && !tx_work && ionic_txq_poke_doorbell(&lif->hwstamp_txq->q))
1229                 resched = true;
1230         if (resched)
1231                 mod_timer(&lif->adminqcq->napi_deadline,
1232                           jiffies + IONIC_NAPI_DEADLINE);
1233
1234         return work_done;
1235 }
1236
1237 void ionic_get_stats64(struct net_device *netdev,
1238                        struct rtnl_link_stats64 *ns)
1239 {
1240         struct ionic_lif *lif = netdev_priv(netdev);
1241         struct ionic_lif_stats *ls;
1242
1243         memset(ns, 0, sizeof(*ns));
1244         ls = &lif->info->stats;
1245
1246         ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1247                          le64_to_cpu(ls->rx_mcast_packets) +
1248                          le64_to_cpu(ls->rx_bcast_packets);
1249
1250         ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1251                          le64_to_cpu(ls->tx_mcast_packets) +
1252                          le64_to_cpu(ls->tx_bcast_packets);
1253
1254         ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1255                        le64_to_cpu(ls->rx_mcast_bytes) +
1256                        le64_to_cpu(ls->rx_bcast_bytes);
1257
1258         ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1259                        le64_to_cpu(ls->tx_mcast_bytes) +
1260                        le64_to_cpu(ls->tx_bcast_bytes);
1261
1262         ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1263                          le64_to_cpu(ls->rx_mcast_drop_packets) +
1264                          le64_to_cpu(ls->rx_bcast_drop_packets);
1265
1266         ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1267                          le64_to_cpu(ls->tx_mcast_drop_packets) +
1268                          le64_to_cpu(ls->tx_bcast_drop_packets);
1269
1270         ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1271
1272         ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1273
1274         ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1275                                le64_to_cpu(ls->rx_queue_disabled) +
1276                                le64_to_cpu(ls->rx_desc_fetch_error) +
1277                                le64_to_cpu(ls->rx_desc_data_error);
1278
1279         ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1280                                 le64_to_cpu(ls->tx_queue_disabled) +
1281                                 le64_to_cpu(ls->tx_desc_fetch_error) +
1282                                 le64_to_cpu(ls->tx_desc_data_error);
1283
1284         ns->rx_errors = ns->rx_over_errors +
1285                         ns->rx_missed_errors;
1286
1287         ns->tx_errors = ns->tx_aborted_errors;
1288 }
1289
1290 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1291 {
1292         return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
1293 }
1294
1295 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1296 {
1297         /* Don't delete our own address from the uc list */
1298         if (ether_addr_equal(addr, netdev->dev_addr))
1299                 return 0;
1300
1301         return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
1302 }
1303
1304 void ionic_lif_rx_mode(struct ionic_lif *lif)
1305 {
1306         struct net_device *netdev = lif->netdev;
1307         unsigned int nfilters;
1308         unsigned int nd_flags;
1309         char buf[128];
1310         u16 rx_mode;
1311         int i;
1312 #define REMAIN(__x) (sizeof(buf) - (__x))
1313
1314         mutex_lock(&lif->config_lock);
1315
1316         /* grab the flags once for local use */
1317         nd_flags = netdev->flags;
1318
1319         rx_mode = IONIC_RX_MODE_F_UNICAST;
1320         rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1321         rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1322         rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1323         rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1324
1325         /* sync the filters */
1326         ionic_rx_filter_sync(lif);
1327
1328         /* check for overflow state
1329          *    if so, we track that we overflowed and enable NIC PROMISC
1330          *    else if the overflow is set and not needed
1331          *       we remove our overflow flag and check the netdev flags
1332          *       to see if we can disable NIC PROMISC
1333          */
1334         nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1335
1336         if (((lif->nucast + lif->nmcast) >= nfilters) ||
1337             (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
1338                 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1339                 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1340         } else {
1341                 if (!(nd_flags & IFF_PROMISC))
1342                         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1343                 if (!(nd_flags & IFF_ALLMULTI))
1344                         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1345         }
1346
1347         i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1348                       lif->rx_mode, rx_mode);
1349         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1350                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1351         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1352                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1353         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1354                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1355         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1356                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1357         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1358                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1359         if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1360                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1361         netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1362
1363         if (lif->rx_mode != rx_mode) {
1364                 struct ionic_admin_ctx ctx = {
1365                         .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1366                         .cmd.rx_mode_set = {
1367                                 .opcode = IONIC_CMD_RX_MODE_SET,
1368                                 .lif_index = cpu_to_le16(lif->index),
1369                         },
1370                 };
1371                 int err;
1372
1373                 ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1374                 err = ionic_adminq_post_wait(lif, &ctx);
1375                 if (err)
1376                         netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1377                                     rx_mode, err);
1378                 else
1379                         lif->rx_mode = rx_mode;
1380         }
1381
1382         mutex_unlock(&lif->config_lock);
1383 }
1384
1385 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1386 {
1387         struct ionic_lif *lif = netdev_priv(netdev);
1388         struct ionic_deferred_work *work;
1389
1390         /* Sync the kernel filter list with the driver filter list */
1391         __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1392         __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1393
1394         /* Shove off the rest of the rxmode work to the work task
1395          * which will include syncing the filters to the firmware.
1396          */
1397         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1398         if (!work) {
1399                 netdev_err(lif->netdev, "rxmode change dropped\n");
1400                 return;
1401         }
1402         work->type = IONIC_DW_TYPE_RX_MODE;
1403         netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1404         ionic_lif_deferred_enqueue(&lif->deferred, work);
1405 }
1406
1407 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1408 {
1409         u64 wanted = 0;
1410
1411         if (features & NETIF_F_HW_VLAN_CTAG_TX)
1412                 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1413         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1414                 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1415         if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1416                 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1417         if (features & NETIF_F_RXHASH)
1418                 wanted |= IONIC_ETH_HW_RX_HASH;
1419         if (features & NETIF_F_RXCSUM)
1420                 wanted |= IONIC_ETH_HW_RX_CSUM;
1421         if (features & NETIF_F_SG)
1422                 wanted |= IONIC_ETH_HW_TX_SG;
1423         if (features & NETIF_F_HW_CSUM)
1424                 wanted |= IONIC_ETH_HW_TX_CSUM;
1425         if (features & NETIF_F_TSO)
1426                 wanted |= IONIC_ETH_HW_TSO;
1427         if (features & NETIF_F_TSO6)
1428                 wanted |= IONIC_ETH_HW_TSO_IPV6;
1429         if (features & NETIF_F_TSO_ECN)
1430                 wanted |= IONIC_ETH_HW_TSO_ECN;
1431         if (features & NETIF_F_GSO_GRE)
1432                 wanted |= IONIC_ETH_HW_TSO_GRE;
1433         if (features & NETIF_F_GSO_GRE_CSUM)
1434                 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1435         if (features & NETIF_F_GSO_IPXIP4)
1436                 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1437         if (features & NETIF_F_GSO_IPXIP6)
1438                 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1439         if (features & NETIF_F_GSO_UDP_TUNNEL)
1440                 wanted |= IONIC_ETH_HW_TSO_UDP;
1441         if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1442                 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1443
1444         return cpu_to_le64(wanted);
1445 }
1446
1447 static int ionic_set_nic_features(struct ionic_lif *lif,
1448                                   netdev_features_t features)
1449 {
1450         struct device *dev = lif->ionic->dev;
1451         struct ionic_admin_ctx ctx = {
1452                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1453                 .cmd.lif_setattr = {
1454                         .opcode = IONIC_CMD_LIF_SETATTR,
1455                         .index = cpu_to_le16(lif->index),
1456                         .attr = IONIC_LIF_ATTR_FEATURES,
1457                 },
1458         };
1459         u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1460                          IONIC_ETH_HW_VLAN_RX_STRIP |
1461                          IONIC_ETH_HW_VLAN_RX_FILTER;
1462         u64 old_hw_features;
1463         int err;
1464
1465         ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1466
1467         if (lif->phc)
1468                 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1469
1470         err = ionic_adminq_post_wait(lif, &ctx);
1471         if (err)
1472                 return err;
1473
1474         old_hw_features = lif->hw_features;
1475         lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1476                                        ctx.comp.lif_setattr.features);
1477
1478         if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1479                 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1480
1481         if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&
1482             !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1483                 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1484
1485         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1486                 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1487         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1488                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1489         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1490                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1491         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1492                 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1493         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1494                 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1495         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1496                 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1497         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1498                 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1499         if (lif->hw_features & IONIC_ETH_HW_TSO)
1500                 dev_dbg(dev, "feature ETH_HW_TSO\n");
1501         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1502                 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1503         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1504                 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1505         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1506                 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1507         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1508                 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1509         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1510                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1511         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1512                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1513         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1514                 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1515         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1516                 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1517         if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1518                 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1519
1520         return 0;
1521 }
1522
1523 static int ionic_init_nic_features(struct ionic_lif *lif)
1524 {
1525         struct net_device *netdev = lif->netdev;
1526         netdev_features_t features;
1527         int err;
1528
1529         /* set up what we expect to support by default */
1530         features = NETIF_F_HW_VLAN_CTAG_TX |
1531                    NETIF_F_HW_VLAN_CTAG_RX |
1532                    NETIF_F_HW_VLAN_CTAG_FILTER |
1533                    NETIF_F_SG |
1534                    NETIF_F_HW_CSUM |
1535                    NETIF_F_RXCSUM |
1536                    NETIF_F_TSO |
1537                    NETIF_F_TSO6 |
1538                    NETIF_F_TSO_ECN |
1539                    NETIF_F_GSO_GRE |
1540                    NETIF_F_GSO_GRE_CSUM |
1541                    NETIF_F_GSO_IPXIP4 |
1542                    NETIF_F_GSO_IPXIP6 |
1543                    NETIF_F_GSO_UDP_TUNNEL |
1544                    NETIF_F_GSO_UDP_TUNNEL_CSUM;
1545
1546         if (lif->nxqs > 1)
1547                 features |= NETIF_F_RXHASH;
1548
1549         err = ionic_set_nic_features(lif, features);
1550         if (err)
1551                 return err;
1552
1553         /* tell the netdev what we actually can support */
1554         netdev->features |= NETIF_F_HIGHDMA;
1555
1556         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1557                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1558         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1559                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1560         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1561                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1562         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1563                 netdev->hw_features |= NETIF_F_RXHASH;
1564         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1565                 netdev->hw_features |= NETIF_F_SG;
1566
1567         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1568                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1569         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1570                 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1571         if (lif->hw_features & IONIC_ETH_HW_TSO)
1572                 netdev->hw_enc_features |= NETIF_F_TSO;
1573         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1574                 netdev->hw_enc_features |= NETIF_F_TSO6;
1575         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1576                 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1577         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1578                 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1579         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1580                 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1581         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1582                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1583         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1584                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1585         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1586                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1587         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1588                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1589
1590         netdev->hw_features |= netdev->hw_enc_features;
1591         netdev->features |= netdev->hw_features;
1592         netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1593
1594         netdev->priv_flags |= IFF_UNICAST_FLT |
1595                               IFF_LIVE_ADDR_CHANGE;
1596
1597         return 0;
1598 }
1599
1600 static int ionic_set_features(struct net_device *netdev,
1601                               netdev_features_t features)
1602 {
1603         struct ionic_lif *lif = netdev_priv(netdev);
1604         int err;
1605
1606         netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1607                    __func__, (u64)lif->netdev->features, (u64)features);
1608
1609         err = ionic_set_nic_features(lif, features);
1610
1611         return err;
1612 }
1613
1614 static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)
1615 {
1616         struct ionic_admin_ctx ctx = {
1617                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1618                 .cmd.lif_setattr = {
1619                         .opcode = IONIC_CMD_LIF_SETATTR,
1620                         .index = cpu_to_le16(lif->index),
1621                         .attr = IONIC_LIF_ATTR_MAC,
1622                 },
1623         };
1624
1625         ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);
1626         return ionic_adminq_post_wait(lif, &ctx);
1627 }
1628
1629 static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)
1630 {
1631         struct ionic_admin_ctx ctx = {
1632                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1633                 .cmd.lif_getattr = {
1634                         .opcode = IONIC_CMD_LIF_GETATTR,
1635                         .index = cpu_to_le16(lif->index),
1636                         .attr = IONIC_LIF_ATTR_MAC,
1637                 },
1638         };
1639         int err;
1640
1641         err = ionic_adminq_post_wait(lif, &ctx);
1642         if (err)
1643                 return err;
1644
1645         ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);
1646         return 0;
1647 }
1648
1649 static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)
1650 {
1651         u8  get_mac[ETH_ALEN];
1652         int err;
1653
1654         err = ionic_set_attr_mac(lif, mac);
1655         if (err)
1656                 return err;
1657
1658         err = ionic_get_attr_mac(lif, get_mac);
1659         if (err)
1660                 return err;
1661
1662         /* To deal with older firmware that silently ignores the set attr mac:
1663          * doesn't actually change the mac and doesn't return an error, so we
1664          * do the get attr to verify whether or not the set actually happened
1665          */
1666         if (!ether_addr_equal(get_mac, mac))
1667                 return 1;
1668
1669         return 0;
1670 }
1671
1672 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1673 {
1674         struct ionic_lif *lif = netdev_priv(netdev);
1675         struct sockaddr *addr = sa;
1676         u8 *mac;
1677         int err;
1678
1679         mac = (u8 *)addr->sa_data;
1680         if (ether_addr_equal(netdev->dev_addr, mac))
1681                 return 0;
1682
1683         err = ionic_program_mac(lif, mac);
1684         if (err < 0)
1685                 return err;
1686
1687         if (err > 0)
1688                 netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
1689                            __func__);
1690
1691         err = eth_prepare_mac_addr_change(netdev, addr);
1692         if (err)
1693                 return err;
1694
1695         if (!is_zero_ether_addr(netdev->dev_addr)) {
1696                 netdev_info(netdev, "deleting mac addr %pM\n",
1697                             netdev->dev_addr);
1698                 ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
1699         }
1700
1701         eth_commit_mac_addr_change(netdev, addr);
1702         netdev_info(netdev, "updating mac addr %pM\n", mac);
1703
1704         return ionic_lif_addr_add(netdev_priv(netdev), mac);
1705 }
1706
1707 static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1708 {
1709         /* Stop and clean the queues before reconfiguration */
1710         netif_device_detach(lif->netdev);
1711         ionic_stop_queues(lif);
1712         ionic_txrx_deinit(lif);
1713 }
1714
1715 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1716 {
1717         int err;
1718
1719         /* Re-init the queues after reconfiguration */
1720
1721         /* The only way txrx_init can fail here is if communication
1722          * with FW is suddenly broken.  There's not much we can do
1723          * at this point - error messages have already been printed,
1724          * so we can continue on and the user can eventually do a
1725          * DOWN and UP to try to reset and clear the issue.
1726          */
1727         err = ionic_txrx_init(lif);
1728         ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1729         netif_device_attach(lif->netdev);
1730
1731         return err;
1732 }
1733
1734 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1735 {
1736         struct ionic_lif *lif = netdev_priv(netdev);
1737         struct ionic_admin_ctx ctx = {
1738                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1739                 .cmd.lif_setattr = {
1740                         .opcode = IONIC_CMD_LIF_SETATTR,
1741                         .index = cpu_to_le16(lif->index),
1742                         .attr = IONIC_LIF_ATTR_MTU,
1743                         .mtu = cpu_to_le32(new_mtu),
1744                 },
1745         };
1746         int err;
1747
1748         err = ionic_adminq_post_wait(lif, &ctx);
1749         if (err)
1750                 return err;
1751
1752         /* if we're not running, nothing more to do */
1753         if (!netif_running(netdev)) {
1754                 netdev->mtu = new_mtu;
1755                 return 0;
1756         }
1757
1758         mutex_lock(&lif->queue_lock);
1759         ionic_stop_queues_reconfig(lif);
1760         netdev->mtu = new_mtu;
1761         err = ionic_start_queues_reconfig(lif);
1762         mutex_unlock(&lif->queue_lock);
1763
1764         return err;
1765 }
1766
1767 static void ionic_tx_timeout_work(struct work_struct *ws)
1768 {
1769         struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1770
1771         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1772                 return;
1773
1774         /* if we were stopped before this scheduled job was launched,
1775          * don't bother the queues as they are already stopped.
1776          */
1777         if (!netif_running(lif->netdev))
1778                 return;
1779
1780         mutex_lock(&lif->queue_lock);
1781         ionic_stop_queues_reconfig(lif);
1782         ionic_start_queues_reconfig(lif);
1783         mutex_unlock(&lif->queue_lock);
1784 }
1785
1786 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1787 {
1788         struct ionic_lif *lif = netdev_priv(netdev);
1789
1790         netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1791         schedule_work(&lif->tx_timeout_work);
1792 }
1793
1794 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1795                                  u16 vid)
1796 {
1797         struct ionic_lif *lif = netdev_priv(netdev);
1798         int err;
1799
1800         err = ionic_lif_vlan_add(lif, vid);
1801         if (err)
1802                 return err;
1803
1804         ionic_lif_rx_mode(lif);
1805
1806         return 0;
1807 }
1808
1809 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1810                                   u16 vid)
1811 {
1812         struct ionic_lif *lif = netdev_priv(netdev);
1813         int err;
1814
1815         err = ionic_lif_vlan_del(lif, vid);
1816         if (err)
1817                 return err;
1818
1819         ionic_lif_rx_mode(lif);
1820
1821         return 0;
1822 }
1823
1824 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1825                          const u8 *key, const u32 *indir)
1826 {
1827         struct ionic_admin_ctx ctx = {
1828                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1829                 .cmd.lif_setattr = {
1830                         .opcode = IONIC_CMD_LIF_SETATTR,
1831                         .attr = IONIC_LIF_ATTR_RSS,
1832                         .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1833                 },
1834         };
1835         unsigned int i, tbl_sz;
1836
1837         if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1838                 lif->rss_types = types;
1839                 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1840         }
1841
1842         if (key)
1843                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1844
1845         if (indir) {
1846                 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1847                 for (i = 0; i < tbl_sz; i++)
1848                         lif->rss_ind_tbl[i] = indir[i];
1849         }
1850
1851         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1852                IONIC_RSS_HASH_KEY_SIZE);
1853
1854         return ionic_adminq_post_wait(lif, &ctx);
1855 }
1856
1857 static int ionic_lif_rss_init(struct ionic_lif *lif)
1858 {
1859         unsigned int tbl_sz;
1860         unsigned int i;
1861
1862         lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1863                          IONIC_RSS_TYPE_IPV4_TCP |
1864                          IONIC_RSS_TYPE_IPV4_UDP |
1865                          IONIC_RSS_TYPE_IPV6     |
1866                          IONIC_RSS_TYPE_IPV6_TCP |
1867                          IONIC_RSS_TYPE_IPV6_UDP;
1868
1869         /* Fill indirection table with 'default' values */
1870         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1871         for (i = 0; i < tbl_sz; i++)
1872                 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1873
1874         return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1875 }
1876
1877 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1878 {
1879         int tbl_sz;
1880
1881         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1882         memset(lif->rss_ind_tbl, 0, tbl_sz);
1883         memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1884
1885         ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1886 }
1887
1888 static void ionic_lif_quiesce(struct ionic_lif *lif)
1889 {
1890         struct ionic_admin_ctx ctx = {
1891                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1892                 .cmd.lif_setattr = {
1893                         .opcode = IONIC_CMD_LIF_SETATTR,
1894                         .index = cpu_to_le16(lif->index),
1895                         .attr = IONIC_LIF_ATTR_STATE,
1896                         .state = IONIC_LIF_QUIESCE,
1897                 },
1898         };
1899         int err;
1900
1901         err = ionic_adminq_post_wait(lif, &ctx);
1902         if (err)
1903                 netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
1904 }
1905
1906 static void ionic_txrx_disable(struct ionic_lif *lif)
1907 {
1908         unsigned int i;
1909         int err = 0;
1910
1911         if (lif->txqcqs) {
1912                 for (i = 0; i < lif->nxqs; i++)
1913                         err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
1914         }
1915
1916         if (lif->hwstamp_txq)
1917                 err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
1918
1919         if (lif->rxqcqs) {
1920                 for (i = 0; i < lif->nxqs; i++)
1921                         err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
1922         }
1923
1924         if (lif->hwstamp_rxq)
1925                 err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
1926
1927         ionic_lif_quiesce(lif);
1928 }
1929
1930 static void ionic_txrx_deinit(struct ionic_lif *lif)
1931 {
1932         unsigned int i;
1933
1934         if (lif->txqcqs) {
1935                 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1936                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1937                         ionic_tx_flush(&lif->txqcqs[i]->cq);
1938                         ionic_tx_empty(&lif->txqcqs[i]->q);
1939                 }
1940         }
1941
1942         if (lif->rxqcqs) {
1943                 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1944                         ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1945                         ionic_rx_empty(&lif->rxqcqs[i]->q);
1946                 }
1947         }
1948         lif->rx_mode = 0;
1949
1950         if (lif->hwstamp_txq) {
1951                 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1952                 ionic_tx_flush(&lif->hwstamp_txq->cq);
1953                 ionic_tx_empty(&lif->hwstamp_txq->q);
1954         }
1955
1956         if (lif->hwstamp_rxq) {
1957                 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1958                 ionic_rx_empty(&lif->hwstamp_rxq->q);
1959         }
1960 }
1961
1962 static void ionic_txrx_free(struct ionic_lif *lif)
1963 {
1964         unsigned int i;
1965
1966         if (lif->txqcqs) {
1967                 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
1968                         ionic_qcq_free(lif, lif->txqcqs[i]);
1969                         devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
1970                         lif->txqcqs[i] = NULL;
1971                 }
1972         }
1973
1974         if (lif->rxqcqs) {
1975                 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
1976                         ionic_qcq_free(lif, lif->rxqcqs[i]);
1977                         devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
1978                         lif->rxqcqs[i] = NULL;
1979                 }
1980         }
1981
1982         if (lif->hwstamp_txq) {
1983                 ionic_qcq_free(lif, lif->hwstamp_txq);
1984                 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
1985                 lif->hwstamp_txq = NULL;
1986         }
1987
1988         if (lif->hwstamp_rxq) {
1989                 ionic_qcq_free(lif, lif->hwstamp_rxq);
1990                 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
1991                 lif->hwstamp_rxq = NULL;
1992         }
1993 }
1994
1995 static int ionic_txrx_alloc(struct ionic_lif *lif)
1996 {
1997         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
1998         unsigned int flags, i;
1999         int err = 0;
2000
2001         num_desc = lif->ntxq_descs;
2002         desc_sz = sizeof(struct ionic_txq_desc);
2003         comp_sz = sizeof(struct ionic_txq_comp);
2004
2005         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2006             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2007                                           sizeof(struct ionic_txq_sg_desc_v1))
2008                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2009         else
2010                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2011
2012         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2013         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2014                 flags |= IONIC_QCQ_F_INTR;
2015         for (i = 0; i < lif->nxqs; i++) {
2016                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2017                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2018                                       lif->kern_pid, &lif->txqcqs[i]);
2019                 if (err)
2020                         goto err_out;
2021
2022                 if (flags & IONIC_QCQ_F_INTR) {
2023                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2024                                              lif->txqcqs[i]->intr.index,
2025                                              lif->tx_coalesce_hw);
2026                         if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2027                                 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2028                 }
2029
2030                 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2031         }
2032
2033         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2034
2035         num_desc = lif->nrxq_descs;
2036         desc_sz = sizeof(struct ionic_rxq_desc);
2037         comp_sz = sizeof(struct ionic_rxq_comp);
2038         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2039
2040         if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2041                 comp_sz *= 2;
2042
2043         for (i = 0; i < lif->nxqs; i++) {
2044                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2045                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2046                                       lif->kern_pid, &lif->rxqcqs[i]);
2047                 if (err)
2048                         goto err_out;
2049
2050                 lif->rxqcqs[i]->q.features = lif->rxq_features;
2051
2052                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2053                                      lif->rxqcqs[i]->intr.index,
2054                                      lif->rx_coalesce_hw);
2055                 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2056                         lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2057
2058                 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2059                         ionic_link_qcq_interrupts(lif->rxqcqs[i],
2060                                                   lif->txqcqs[i]);
2061
2062                 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2063         }
2064
2065         return 0;
2066
2067 err_out:
2068         ionic_txrx_free(lif);
2069
2070         return err;
2071 }
2072
2073 static int ionic_txrx_init(struct ionic_lif *lif)
2074 {
2075         unsigned int i;
2076         int err;
2077
2078         for (i = 0; i < lif->nxqs; i++) {
2079                 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2080                 if (err)
2081                         goto err_out;
2082
2083                 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2084                 if (err) {
2085                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2086                         goto err_out;
2087                 }
2088         }
2089
2090         if (lif->netdev->features & NETIF_F_RXHASH)
2091                 ionic_lif_rss_init(lif);
2092
2093         ionic_lif_rx_mode(lif);
2094
2095         return 0;
2096
2097 err_out:
2098         while (i--) {
2099                 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2100                 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2101         }
2102
2103         return err;
2104 }
2105
2106 static int ionic_txrx_enable(struct ionic_lif *lif)
2107 {
2108         int derr = 0;
2109         int i, err;
2110
2111         for (i = 0; i < lif->nxqs; i++) {
2112                 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2113                         dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2114                         err = -ENXIO;
2115                         goto err_out;
2116                 }
2117
2118                 ionic_rx_fill(&lif->rxqcqs[i]->q);
2119                 err = ionic_qcq_enable(lif->rxqcqs[i]);
2120                 if (err)
2121                         goto err_out;
2122
2123                 err = ionic_qcq_enable(lif->txqcqs[i]);
2124                 if (err) {
2125                         derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
2126                         goto err_out;
2127                 }
2128         }
2129
2130         if (lif->hwstamp_rxq) {
2131                 ionic_rx_fill(&lif->hwstamp_rxq->q);
2132                 err = ionic_qcq_enable(lif->hwstamp_rxq);
2133                 if (err)
2134                         goto err_out_hwstamp_rx;
2135         }
2136
2137         if (lif->hwstamp_txq) {
2138                 err = ionic_qcq_enable(lif->hwstamp_txq);
2139                 if (err)
2140                         goto err_out_hwstamp_tx;
2141         }
2142
2143         return 0;
2144
2145 err_out_hwstamp_tx:
2146         if (lif->hwstamp_rxq)
2147                 derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
2148 err_out_hwstamp_rx:
2149         i = lif->nxqs;
2150 err_out:
2151         while (i--) {
2152                 derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
2153                 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
2154         }
2155
2156         return err;
2157 }
2158
2159 static int ionic_start_queues(struct ionic_lif *lif)
2160 {
2161         int err;
2162
2163         if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2164                 return -EIO;
2165
2166         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2167                 return -EBUSY;
2168
2169         if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2170                 return 0;
2171
2172         err = ionic_txrx_enable(lif);
2173         if (err) {
2174                 clear_bit(IONIC_LIF_F_UP, lif->state);
2175                 return err;
2176         }
2177         netif_tx_wake_all_queues(lif->netdev);
2178
2179         return 0;
2180 }
2181
2182 static int ionic_open(struct net_device *netdev)
2183 {
2184         struct ionic_lif *lif = netdev_priv(netdev);
2185         int err;
2186
2187         /* If recovering from a broken state, clear the bit and we'll try again */
2188         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2189                 netdev_info(netdev, "clearing broken state\n");
2190
2191         mutex_lock(&lif->queue_lock);
2192
2193         err = ionic_txrx_alloc(lif);
2194         if (err)
2195                 goto err_unlock;
2196
2197         err = ionic_txrx_init(lif);
2198         if (err)
2199                 goto err_txrx_free;
2200
2201         err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2202         if (err)
2203                 goto err_txrx_deinit;
2204
2205         err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2206         if (err)
2207                 goto err_txrx_deinit;
2208
2209         /* don't start the queues until we have link */
2210         if (netif_carrier_ok(netdev)) {
2211                 err = ionic_start_queues(lif);
2212                 if (err)
2213                         goto err_txrx_deinit;
2214         }
2215
2216         /* If hardware timestamping is enabled, but the queues were freed by
2217          * ionic_stop, those need to be reallocated and initialized, too.
2218          */
2219         ionic_lif_hwstamp_recreate_queues(lif);
2220
2221         mutex_unlock(&lif->queue_lock);
2222
2223         return 0;
2224
2225 err_txrx_deinit:
2226         ionic_txrx_deinit(lif);
2227 err_txrx_free:
2228         ionic_txrx_free(lif);
2229 err_unlock:
2230         mutex_unlock(&lif->queue_lock);
2231         return err;
2232 }
2233
2234 static void ionic_stop_queues(struct ionic_lif *lif)
2235 {
2236         if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2237                 return;
2238
2239         netif_tx_disable(lif->netdev);
2240         ionic_txrx_disable(lif);
2241 }
2242
2243 static int ionic_stop(struct net_device *netdev)
2244 {
2245         struct ionic_lif *lif = netdev_priv(netdev);
2246
2247         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2248                 return 0;
2249
2250         mutex_lock(&lif->queue_lock);
2251         ionic_stop_queues(lif);
2252         ionic_txrx_deinit(lif);
2253         ionic_txrx_free(lif);
2254         mutex_unlock(&lif->queue_lock);
2255
2256         return 0;
2257 }
2258
2259 static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2260 {
2261         struct ionic_lif *lif = netdev_priv(netdev);
2262
2263         switch (cmd) {
2264         case SIOCSHWTSTAMP:
2265                 return ionic_lif_hwstamp_set(lif, ifr);
2266         case SIOCGHWTSTAMP:
2267                 return ionic_lif_hwstamp_get(lif, ifr);
2268         default:
2269                 return -EOPNOTSUPP;
2270         }
2271 }
2272
2273 static int ionic_get_fw_vf_config(struct ionic *ionic, int vf, struct ionic_vf *vfdata)
2274 {
2275         struct ionic_vf_getattr_comp comp = { 0 };
2276         int err;
2277         u8 attr;
2278
2279         attr = IONIC_VF_ATTR_VLAN;
2280         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2281         if (err && comp.status != IONIC_RC_ENOSUPP)
2282                 goto err_out;
2283         if (!err)
2284                 vfdata->vlanid = comp.vlanid;
2285
2286         attr = IONIC_VF_ATTR_SPOOFCHK;
2287         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2288         if (err && comp.status != IONIC_RC_ENOSUPP)
2289                 goto err_out;
2290         if (!err)
2291                 vfdata->spoofchk = comp.spoofchk;
2292
2293         attr = IONIC_VF_ATTR_LINKSTATE;
2294         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2295         if (err && comp.status != IONIC_RC_ENOSUPP)
2296                 goto err_out;
2297         if (!err) {
2298                 switch (comp.linkstate) {
2299                 case IONIC_VF_LINK_STATUS_UP:
2300                         vfdata->linkstate = IFLA_VF_LINK_STATE_ENABLE;
2301                         break;
2302                 case IONIC_VF_LINK_STATUS_DOWN:
2303                         vfdata->linkstate = IFLA_VF_LINK_STATE_DISABLE;
2304                         break;
2305                 case IONIC_VF_LINK_STATUS_AUTO:
2306                         vfdata->linkstate = IFLA_VF_LINK_STATE_AUTO;
2307                         break;
2308                 default:
2309                         dev_warn(ionic->dev, "Unexpected link state %u\n", comp.linkstate);
2310                         break;
2311                 }
2312         }
2313
2314         attr = IONIC_VF_ATTR_RATE;
2315         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2316         if (err && comp.status != IONIC_RC_ENOSUPP)
2317                 goto err_out;
2318         if (!err)
2319                 vfdata->maxrate = comp.maxrate;
2320
2321         attr = IONIC_VF_ATTR_TRUST;
2322         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2323         if (err && comp.status != IONIC_RC_ENOSUPP)
2324                 goto err_out;
2325         if (!err)
2326                 vfdata->trusted = comp.trust;
2327
2328         attr = IONIC_VF_ATTR_MAC;
2329         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2330         if (err && comp.status != IONIC_RC_ENOSUPP)
2331                 goto err_out;
2332         if (!err)
2333                 ether_addr_copy(vfdata->macaddr, comp.macaddr);
2334
2335 err_out:
2336         if (err)
2337                 dev_err(ionic->dev, "Failed to get %s for VF %d\n",
2338                         ionic_vf_attr_to_str(attr), vf);
2339
2340         return err;
2341 }
2342
2343 static int ionic_get_vf_config(struct net_device *netdev,
2344                                int vf, struct ifla_vf_info *ivf)
2345 {
2346         struct ionic_lif *lif = netdev_priv(netdev);
2347         struct ionic *ionic = lif->ionic;
2348         struct ionic_vf vfdata = { 0 };
2349         int ret = 0;
2350
2351         if (!netif_device_present(netdev))
2352                 return -EBUSY;
2353
2354         down_read(&ionic->vf_op_lock);
2355
2356         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2357                 ret = -EINVAL;
2358         } else {
2359                 ivf->vf = vf;
2360                 ivf->qos = 0;
2361
2362                 ret = ionic_get_fw_vf_config(ionic, vf, &vfdata);
2363                 if (!ret) {
2364                         ivf->vlan         = le16_to_cpu(vfdata.vlanid);
2365                         ivf->spoofchk     = vfdata.spoofchk;
2366                         ivf->linkstate    = vfdata.linkstate;
2367                         ivf->max_tx_rate  = le32_to_cpu(vfdata.maxrate);
2368                         ivf->trusted      = vfdata.trusted;
2369                         ether_addr_copy(ivf->mac, vfdata.macaddr);
2370                 }
2371         }
2372
2373         up_read(&ionic->vf_op_lock);
2374         return ret;
2375 }
2376
2377 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2378                               struct ifla_vf_stats *vf_stats)
2379 {
2380         struct ionic_lif *lif = netdev_priv(netdev);
2381         struct ionic *ionic = lif->ionic;
2382         struct ionic_lif_stats *vs;
2383         int ret = 0;
2384
2385         if (!netif_device_present(netdev))
2386                 return -EBUSY;
2387
2388         down_read(&ionic->vf_op_lock);
2389
2390         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2391                 ret = -EINVAL;
2392         } else {
2393                 memset(vf_stats, 0, sizeof(*vf_stats));
2394                 vs = &ionic->vfs[vf].stats;
2395
2396                 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2397                 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2398                 vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2399                 vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2400                 vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2401                 vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2402                 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2403                                        le64_to_cpu(vs->rx_mcast_drop_packets) +
2404                                        le64_to_cpu(vs->rx_bcast_drop_packets);
2405                 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2406                                        le64_to_cpu(vs->tx_mcast_drop_packets) +
2407                                        le64_to_cpu(vs->tx_bcast_drop_packets);
2408         }
2409
2410         up_read(&ionic->vf_op_lock);
2411         return ret;
2412 }
2413
2414 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2415 {
2416         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
2417         struct ionic_lif *lif = netdev_priv(netdev);
2418         struct ionic *ionic = lif->ionic;
2419         int ret;
2420
2421         if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2422                 return -EINVAL;
2423
2424         if (!netif_device_present(netdev))
2425                 return -EBUSY;
2426
2427         down_write(&ionic->vf_op_lock);
2428
2429         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2430                 ret = -EINVAL;
2431         } else {
2432                 ether_addr_copy(vfc.macaddr, mac);
2433                 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
2434                         __func__, vf, vfc.macaddr);
2435
2436                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2437                 if (!ret)
2438                         ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2439         }
2440
2441         up_write(&ionic->vf_op_lock);
2442         return ret;
2443 }
2444
2445 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2446                              u8 qos, __be16 proto)
2447 {
2448         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
2449         struct ionic_lif *lif = netdev_priv(netdev);
2450         struct ionic *ionic = lif->ionic;
2451         int ret;
2452
2453         /* until someday when we support qos */
2454         if (qos)
2455                 return -EINVAL;
2456
2457         if (vlan > 4095)
2458                 return -EINVAL;
2459
2460         if (proto != htons(ETH_P_8021Q))
2461                 return -EPROTONOSUPPORT;
2462
2463         if (!netif_device_present(netdev))
2464                 return -EBUSY;
2465
2466         down_write(&ionic->vf_op_lock);
2467
2468         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2469                 ret = -EINVAL;
2470         } else {
2471                 vfc.vlanid = cpu_to_le16(vlan);
2472                 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
2473                         __func__, vf, le16_to_cpu(vfc.vlanid));
2474
2475                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2476                 if (!ret)
2477                         ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2478         }
2479
2480         up_write(&ionic->vf_op_lock);
2481         return ret;
2482 }
2483
2484 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2485                              int tx_min, int tx_max)
2486 {
2487         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
2488         struct ionic_lif *lif = netdev_priv(netdev);
2489         struct ionic *ionic = lif->ionic;
2490         int ret;
2491
2492         /* setting the min just seems silly */
2493         if (tx_min)
2494                 return -EINVAL;
2495
2496         if (!netif_device_present(netdev))
2497                 return -EBUSY;
2498
2499         down_write(&ionic->vf_op_lock);
2500
2501         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2502                 ret = -EINVAL;
2503         } else {
2504                 vfc.maxrate = cpu_to_le32(tx_max);
2505                 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
2506                         __func__, vf, le32_to_cpu(vfc.maxrate));
2507
2508                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2509                 if (!ret)
2510                         lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2511         }
2512
2513         up_write(&ionic->vf_op_lock);
2514         return ret;
2515 }
2516
2517 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2518 {
2519         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
2520         struct ionic_lif *lif = netdev_priv(netdev);
2521         struct ionic *ionic = lif->ionic;
2522         int ret;
2523
2524         if (!netif_device_present(netdev))
2525                 return -EBUSY;
2526
2527         down_write(&ionic->vf_op_lock);
2528
2529         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2530                 ret = -EINVAL;
2531         } else {
2532                 vfc.spoofchk = set;
2533                 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
2534                         __func__, vf, vfc.spoofchk);
2535
2536                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2537                 if (!ret)
2538                         ionic->vfs[vf].spoofchk = set;
2539         }
2540
2541         up_write(&ionic->vf_op_lock);
2542         return ret;
2543 }
2544
2545 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2546 {
2547         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
2548         struct ionic_lif *lif = netdev_priv(netdev);
2549         struct ionic *ionic = lif->ionic;
2550         int ret;
2551
2552         if (!netif_device_present(netdev))
2553                 return -EBUSY;
2554
2555         down_write(&ionic->vf_op_lock);
2556
2557         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2558                 ret = -EINVAL;
2559         } else {
2560                 vfc.trust = set;
2561                 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
2562                         __func__, vf, vfc.trust);
2563
2564                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2565                 if (!ret)
2566                         ionic->vfs[vf].trusted = set;
2567         }
2568
2569         up_write(&ionic->vf_op_lock);
2570         return ret;
2571 }
2572
2573 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2574 {
2575         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
2576         struct ionic_lif *lif = netdev_priv(netdev);
2577         struct ionic *ionic = lif->ionic;
2578         u8 vfls;
2579         int ret;
2580
2581         switch (set) {
2582         case IFLA_VF_LINK_STATE_ENABLE:
2583                 vfls = IONIC_VF_LINK_STATUS_UP;
2584                 break;
2585         case IFLA_VF_LINK_STATE_DISABLE:
2586                 vfls = IONIC_VF_LINK_STATUS_DOWN;
2587                 break;
2588         case IFLA_VF_LINK_STATE_AUTO:
2589                 vfls = IONIC_VF_LINK_STATUS_AUTO;
2590                 break;
2591         default:
2592                 return -EINVAL;
2593         }
2594
2595         if (!netif_device_present(netdev))
2596                 return -EBUSY;
2597
2598         down_write(&ionic->vf_op_lock);
2599
2600         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2601                 ret = -EINVAL;
2602         } else {
2603                 vfc.linkstate = vfls;
2604                 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
2605                         __func__, vf, vfc.linkstate);
2606
2607                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2608                 if (!ret)
2609                         ionic->vfs[vf].linkstate = set;
2610         }
2611
2612         up_write(&ionic->vf_op_lock);
2613         return ret;
2614 }
2615
2616 static void ionic_vf_attr_replay(struct ionic_lif *lif)
2617 {
2618         struct ionic_vf_setattr_cmd vfc = { };
2619         struct ionic *ionic = lif->ionic;
2620         struct ionic_vf *v;
2621         int i;
2622
2623         if (!ionic->vfs)
2624                 return;
2625
2626         down_read(&ionic->vf_op_lock);
2627
2628         for (i = 0; i < ionic->num_vfs; i++) {
2629                 v = &ionic->vfs[i];
2630
2631                 if (v->stats_pa) {
2632                         vfc.attr = IONIC_VF_ATTR_STATSADDR;
2633                         vfc.stats_pa = cpu_to_le64(v->stats_pa);
2634                         ionic_set_vf_config(ionic, i, &vfc);
2635                         vfc.stats_pa = 0;
2636                 }
2637
2638                 if (!is_zero_ether_addr(v->macaddr)) {
2639                         vfc.attr = IONIC_VF_ATTR_MAC;
2640                         ether_addr_copy(vfc.macaddr, v->macaddr);
2641                         ionic_set_vf_config(ionic, i, &vfc);
2642                         eth_zero_addr(vfc.macaddr);
2643                 }
2644
2645                 if (v->vlanid) {
2646                         vfc.attr = IONIC_VF_ATTR_VLAN;
2647                         vfc.vlanid = v->vlanid;
2648                         ionic_set_vf_config(ionic, i, &vfc);
2649                         vfc.vlanid = 0;
2650                 }
2651
2652                 if (v->maxrate) {
2653                         vfc.attr = IONIC_VF_ATTR_RATE;
2654                         vfc.maxrate = v->maxrate;
2655                         ionic_set_vf_config(ionic, i, &vfc);
2656                         vfc.maxrate = 0;
2657                 }
2658
2659                 if (v->spoofchk) {
2660                         vfc.attr = IONIC_VF_ATTR_SPOOFCHK;
2661                         vfc.spoofchk = v->spoofchk;
2662                         ionic_set_vf_config(ionic, i, &vfc);
2663                         vfc.spoofchk = 0;
2664                 }
2665
2666                 if (v->trusted) {
2667                         vfc.attr = IONIC_VF_ATTR_TRUST;
2668                         vfc.trust = v->trusted;
2669                         ionic_set_vf_config(ionic, i, &vfc);
2670                         vfc.trust = 0;
2671                 }
2672
2673                 if (v->linkstate) {
2674                         vfc.attr = IONIC_VF_ATTR_LINKSTATE;
2675                         vfc.linkstate = v->linkstate;
2676                         ionic_set_vf_config(ionic, i, &vfc);
2677                         vfc.linkstate = 0;
2678                 }
2679         }
2680
2681         up_read(&ionic->vf_op_lock);
2682
2683         ionic_vf_start(ionic);
2684 }
2685
2686 static const struct net_device_ops ionic_netdev_ops = {
2687         .ndo_open               = ionic_open,
2688         .ndo_stop               = ionic_stop,
2689         .ndo_eth_ioctl          = ionic_eth_ioctl,
2690         .ndo_start_xmit         = ionic_start_xmit,
2691         .ndo_get_stats64        = ionic_get_stats64,
2692         .ndo_set_rx_mode        = ionic_ndo_set_rx_mode,
2693         .ndo_set_features       = ionic_set_features,
2694         .ndo_set_mac_address    = ionic_set_mac_address,
2695         .ndo_validate_addr      = eth_validate_addr,
2696         .ndo_tx_timeout         = ionic_tx_timeout,
2697         .ndo_change_mtu         = ionic_change_mtu,
2698         .ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2699         .ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2700         .ndo_set_vf_vlan        = ionic_set_vf_vlan,
2701         .ndo_set_vf_trust       = ionic_set_vf_trust,
2702         .ndo_set_vf_mac         = ionic_set_vf_mac,
2703         .ndo_set_vf_rate        = ionic_set_vf_rate,
2704         .ndo_set_vf_spoofchk    = ionic_set_vf_spoofchk,
2705         .ndo_get_vf_config      = ionic_get_vf_config,
2706         .ndo_set_vf_link_state  = ionic_set_vf_link_state,
2707         .ndo_get_vf_stats       = ionic_get_vf_stats,
2708 };
2709
2710 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2711 {
2712         /* only swapping the queues, not the napi, flags, or other stuff */
2713         swap(a->q.features,   b->q.features);
2714         swap(a->q.num_descs,  b->q.num_descs);
2715         swap(a->q.desc_size,  b->q.desc_size);
2716         swap(a->q.base,       b->q.base);
2717         swap(a->q.base_pa,    b->q.base_pa);
2718         swap(a->q.info,       b->q.info);
2719         swap(a->q_base,       b->q_base);
2720         swap(a->q_base_pa,    b->q_base_pa);
2721         swap(a->q_size,       b->q_size);
2722
2723         swap(a->q.sg_desc_size, b->q.sg_desc_size);
2724         swap(a->q.sg_base,    b->q.sg_base);
2725         swap(a->q.sg_base_pa, b->q.sg_base_pa);
2726         swap(a->sg_base,      b->sg_base);
2727         swap(a->sg_base_pa,   b->sg_base_pa);
2728         swap(a->sg_size,      b->sg_size);
2729
2730         swap(a->cq.num_descs, b->cq.num_descs);
2731         swap(a->cq.desc_size, b->cq.desc_size);
2732         swap(a->cq.base,      b->cq.base);
2733         swap(a->cq.base_pa,   b->cq.base_pa);
2734         swap(a->cq.info,      b->cq.info);
2735         swap(a->cq_base,      b->cq_base);
2736         swap(a->cq_base_pa,   b->cq_base_pa);
2737         swap(a->cq_size,      b->cq_size);
2738
2739         ionic_debugfs_del_qcq(a);
2740         ionic_debugfs_add_qcq(a->q.lif, a);
2741 }
2742
2743 int ionic_reconfigure_queues(struct ionic_lif *lif,
2744                              struct ionic_queue_params *qparam)
2745 {
2746         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2747         struct ionic_qcq **tx_qcqs = NULL;
2748         struct ionic_qcq **rx_qcqs = NULL;
2749         unsigned int flags, i;
2750         int err = 0;
2751
2752         /* allocate temporary qcq arrays to hold new queue structs */
2753         if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2754                 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2755                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2756                 if (!tx_qcqs) {
2757                         err = -ENOMEM;
2758                         goto err_out;
2759                 }
2760         }
2761         if (qparam->nxqs != lif->nxqs ||
2762             qparam->nrxq_descs != lif->nrxq_descs ||
2763             qparam->rxq_features != lif->rxq_features) {
2764                 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2765                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2766                 if (!rx_qcqs) {
2767                         err = -ENOMEM;
2768                         goto err_out;
2769                 }
2770         }
2771
2772         /* allocate new desc_info and rings, but leave the interrupt setup
2773          * until later so as to not mess with the still-running queues
2774          */
2775         if (tx_qcqs) {
2776                 num_desc = qparam->ntxq_descs;
2777                 desc_sz = sizeof(struct ionic_txq_desc);
2778                 comp_sz = sizeof(struct ionic_txq_comp);
2779
2780                 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2781                     lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2782                     sizeof(struct ionic_txq_sg_desc_v1))
2783                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2784                 else
2785                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2786
2787                 for (i = 0; i < qparam->nxqs; i++) {
2788                         flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2789                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2790                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2791                                               lif->kern_pid, &tx_qcqs[i]);
2792                         if (err)
2793                                 goto err_out;
2794                 }
2795         }
2796
2797         if (rx_qcqs) {
2798                 num_desc = qparam->nrxq_descs;
2799                 desc_sz = sizeof(struct ionic_rxq_desc);
2800                 comp_sz = sizeof(struct ionic_rxq_comp);
2801                 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2802
2803                 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2804                         comp_sz *= 2;
2805
2806                 for (i = 0; i < qparam->nxqs; i++) {
2807                         flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2808                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2809                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2810                                               lif->kern_pid, &rx_qcqs[i]);
2811                         if (err)
2812                                 goto err_out;
2813
2814                         rx_qcqs[i]->q.features = qparam->rxq_features;
2815                 }
2816         }
2817
2818         /* stop and clean the queues */
2819         ionic_stop_queues_reconfig(lif);
2820
2821         if (qparam->nxqs != lif->nxqs) {
2822                 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2823                 if (err)
2824                         goto err_out_reinit_unlock;
2825                 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2826                 if (err) {
2827                         netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2828                         goto err_out_reinit_unlock;
2829                 }
2830         }
2831
2832         /* swap new desc_info and rings, keeping existing interrupt config */
2833         if (tx_qcqs) {
2834                 lif->ntxq_descs = qparam->ntxq_descs;
2835                 for (i = 0; i < qparam->nxqs; i++)
2836                         ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2837         }
2838
2839         if (rx_qcqs) {
2840                 lif->nrxq_descs = qparam->nrxq_descs;
2841                 for (i = 0; i < qparam->nxqs; i++)
2842                         ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2843         }
2844
2845         /* if we need to change the interrupt layout, this is the time */
2846         if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2847             qparam->nxqs != lif->nxqs) {
2848                 if (qparam->intr_split) {
2849                         set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2850                 } else {
2851                         clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2852                         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2853                         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2854                 }
2855
2856                 /* clear existing interrupt assignments */
2857                 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2858                         ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2859                         ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2860                 }
2861
2862                 /* re-assign the interrupts */
2863                 for (i = 0; i < qparam->nxqs; i++) {
2864                         lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2865                         err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2866                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2867                                              lif->rxqcqs[i]->intr.index,
2868                                              lif->rx_coalesce_hw);
2869
2870                         if (qparam->intr_split) {
2871                                 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2872                                 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2873                                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2874                                                      lif->txqcqs[i]->intr.index,
2875                                                      lif->tx_coalesce_hw);
2876                                 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2877                                         lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2878                         } else {
2879                                 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2880                                 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2881                         }
2882                 }
2883         }
2884
2885         /* now we can rework the debugfs mappings */
2886         if (tx_qcqs) {
2887                 for (i = 0; i < qparam->nxqs; i++) {
2888                         ionic_debugfs_del_qcq(lif->txqcqs[i]);
2889                         ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2890                 }
2891         }
2892
2893         if (rx_qcqs) {
2894                 for (i = 0; i < qparam->nxqs; i++) {
2895                         ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2896                         ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2897                 }
2898         }
2899
2900         swap(lif->nxqs, qparam->nxqs);
2901         swap(lif->rxq_features, qparam->rxq_features);
2902
2903 err_out_reinit_unlock:
2904         /* re-init the queues, but don't lose an error code */
2905         if (err)
2906                 ionic_start_queues_reconfig(lif);
2907         else
2908                 err = ionic_start_queues_reconfig(lif);
2909
2910 err_out:
2911         /* free old allocs without cleaning intr */
2912         for (i = 0; i < qparam->nxqs; i++) {
2913                 if (tx_qcqs && tx_qcqs[i]) {
2914                         tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2915                         ionic_qcq_free(lif, tx_qcqs[i]);
2916                         devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2917                         tx_qcqs[i] = NULL;
2918                 }
2919                 if (rx_qcqs && rx_qcqs[i]) {
2920                         rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2921                         ionic_qcq_free(lif, rx_qcqs[i]);
2922                         devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2923                         rx_qcqs[i] = NULL;
2924                 }
2925         }
2926
2927         /* free q array */
2928         if (rx_qcqs) {
2929                 devm_kfree(lif->ionic->dev, rx_qcqs);
2930                 rx_qcqs = NULL;
2931         }
2932         if (tx_qcqs) {
2933                 devm_kfree(lif->ionic->dev, tx_qcqs);
2934                 tx_qcqs = NULL;
2935         }
2936
2937         /* clean the unused dma and info allocations when new set is smaller
2938          * than the full array, but leave the qcq shells in place
2939          */
2940         for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2941                 if (lif->txqcqs && lif->txqcqs[i]) {
2942                         lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2943                         ionic_qcq_free(lif, lif->txqcqs[i]);
2944                 }
2945
2946                 if (lif->rxqcqs && lif->rxqcqs[i]) {
2947                         lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2948                         ionic_qcq_free(lif, lif->rxqcqs[i]);
2949                 }
2950         }
2951
2952         if (err)
2953                 netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
2954
2955         return err;
2956 }
2957
2958 int ionic_lif_alloc(struct ionic *ionic)
2959 {
2960         struct device *dev = ionic->dev;
2961         union ionic_lif_identity *lid;
2962         struct net_device *netdev;
2963         struct ionic_lif *lif;
2964         int tbl_sz;
2965         int err;
2966
2967         lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2968         if (!lid)
2969                 return -ENOMEM;
2970
2971         netdev = alloc_etherdev_mqs(sizeof(*lif),
2972                                     ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2973         if (!netdev) {
2974                 dev_err(dev, "Cannot allocate netdev, aborting\n");
2975                 err = -ENOMEM;
2976                 goto err_out_free_lid;
2977         }
2978
2979         SET_NETDEV_DEV(netdev, dev);
2980
2981         lif = netdev_priv(netdev);
2982         lif->netdev = netdev;
2983         ionic->lif = lif;
2984         netdev->netdev_ops = &ionic_netdev_ops;
2985         ionic_ethtool_set_ops(netdev);
2986
2987         netdev->watchdog_timeo = 2 * HZ;
2988         netif_carrier_off(netdev);
2989
2990         lif->identity = lid;
2991         lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2992         err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2993         if (err) {
2994                 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2995                         lif->lif_type, err);
2996                 goto err_out_free_netdev;
2997         }
2998         lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2999                                      le32_to_cpu(lif->identity->eth.min_frame_size));
3000         lif->netdev->max_mtu =
3001                 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
3002
3003         lif->neqs = ionic->neqs_per_lif;
3004         lif->nxqs = ionic->ntxqs_per_lif;
3005
3006         lif->ionic = ionic;
3007         lif->index = 0;
3008
3009         if (is_kdump_kernel()) {
3010                 lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
3011                 lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
3012         } else {
3013                 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
3014                 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
3015         }
3016
3017         /* Convert the default coalesce value to actual hw resolution */
3018         lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
3019         lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
3020                                                     lif->rx_coalesce_usecs);
3021         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3022         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3023         set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
3024         set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
3025
3026         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
3027
3028         mutex_init(&lif->queue_lock);
3029         mutex_init(&lif->config_lock);
3030
3031         spin_lock_init(&lif->adminq_lock);
3032
3033         spin_lock_init(&lif->deferred.lock);
3034         INIT_LIST_HEAD(&lif->deferred.list);
3035         INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
3036
3037         /* allocate lif info */
3038         lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
3039         lif->info = dma_alloc_coherent(dev, lif->info_sz,
3040                                        &lif->info_pa, GFP_KERNEL);
3041         if (!lif->info) {
3042                 dev_err(dev, "Failed to allocate lif info, aborting\n");
3043                 err = -ENOMEM;
3044                 goto err_out_free_mutex;
3045         }
3046
3047         ionic_debugfs_add_lif(lif);
3048
3049         /* allocate control queues and txrx queue arrays */
3050         ionic_lif_queue_identify(lif);
3051         err = ionic_qcqs_alloc(lif);
3052         if (err)
3053                 goto err_out_free_lif_info;
3054
3055         /* allocate rss indirection table */
3056         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
3057         lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
3058         lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
3059                                               &lif->rss_ind_tbl_pa,
3060                                               GFP_KERNEL);
3061
3062         if (!lif->rss_ind_tbl) {
3063                 err = -ENOMEM;
3064                 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
3065                 goto err_out_free_qcqs;
3066         }
3067         netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
3068
3069         ionic_lif_alloc_phc(lif);
3070
3071         return 0;
3072
3073 err_out_free_qcqs:
3074         ionic_qcqs_free(lif);
3075 err_out_free_lif_info:
3076         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3077         lif->info = NULL;
3078         lif->info_pa = 0;
3079 err_out_free_mutex:
3080         mutex_destroy(&lif->config_lock);
3081         mutex_destroy(&lif->queue_lock);
3082 err_out_free_netdev:
3083         free_netdev(lif->netdev);
3084         lif = NULL;
3085 err_out_free_lid:
3086         kfree(lid);
3087
3088         return err;
3089 }
3090
3091 static void ionic_lif_reset(struct ionic_lif *lif)
3092 {
3093         struct ionic_dev *idev = &lif->ionic->idev;
3094
3095         mutex_lock(&lif->ionic->dev_cmd_lock);
3096         ionic_dev_cmd_lif_reset(idev, lif->index);
3097         ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3098         mutex_unlock(&lif->ionic->dev_cmd_lock);
3099 }
3100
3101 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
3102 {
3103         struct ionic *ionic = lif->ionic;
3104
3105         if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
3106                 return;
3107
3108         dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
3109
3110         netif_device_detach(lif->netdev);
3111
3112         mutex_lock(&lif->queue_lock);
3113         if (test_bit(IONIC_LIF_F_UP, lif->state)) {
3114                 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
3115                 ionic_stop_queues(lif);
3116         }
3117
3118         if (netif_running(lif->netdev)) {
3119                 ionic_txrx_deinit(lif);
3120                 ionic_txrx_free(lif);
3121         }
3122         ionic_lif_deinit(lif);
3123         ionic_reset(ionic);
3124         ionic_qcqs_free(lif);
3125
3126         mutex_unlock(&lif->queue_lock);
3127
3128         clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
3129         dev_info(ionic->dev, "FW Down: LIFs stopped\n");
3130 }
3131
3132 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
3133 {
3134         struct ionic *ionic = lif->ionic;
3135         int err;
3136
3137         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3138                 return;
3139
3140         dev_info(ionic->dev, "FW Up: restarting LIFs\n");
3141
3142         ionic_init_devinfo(ionic);
3143         err = ionic_identify(ionic);
3144         if (err)
3145                 goto err_out;
3146         err = ionic_port_identify(ionic);
3147         if (err)
3148                 goto err_out;
3149         err = ionic_port_init(ionic);
3150         if (err)
3151                 goto err_out;
3152
3153         mutex_lock(&lif->queue_lock);
3154
3155         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
3156                 dev_info(ionic->dev, "FW Up: clearing broken state\n");
3157
3158         err = ionic_qcqs_alloc(lif);
3159         if (err)
3160                 goto err_unlock;
3161
3162         err = ionic_lif_init(lif);
3163         if (err)
3164                 goto err_qcqs_free;
3165
3166         ionic_vf_attr_replay(lif);
3167
3168         if (lif->registered)
3169                 ionic_lif_set_netdev_info(lif);
3170
3171         ionic_rx_filter_replay(lif);
3172
3173         if (netif_running(lif->netdev)) {
3174                 err = ionic_txrx_alloc(lif);
3175                 if (err)
3176                         goto err_lifs_deinit;
3177
3178                 err = ionic_txrx_init(lif);
3179                 if (err)
3180                         goto err_txrx_free;
3181         }
3182
3183         mutex_unlock(&lif->queue_lock);
3184
3185         clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3186         ionic_link_status_check_request(lif, CAN_SLEEP);
3187         netif_device_attach(lif->netdev);
3188         dev_info(ionic->dev, "FW Up: LIFs restarted\n");
3189
3190         /* restore the hardware timestamping queues */
3191         ionic_lif_hwstamp_replay(lif);
3192
3193         return;
3194
3195 err_txrx_free:
3196         ionic_txrx_free(lif);
3197 err_lifs_deinit:
3198         ionic_lif_deinit(lif);
3199 err_qcqs_free:
3200         ionic_qcqs_free(lif);
3201 err_unlock:
3202         mutex_unlock(&lif->queue_lock);
3203 err_out:
3204         dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3205 }
3206
3207 void ionic_lif_free(struct ionic_lif *lif)
3208 {
3209         struct device *dev = lif->ionic->dev;
3210
3211         ionic_lif_free_phc(lif);
3212
3213         /* free rss indirection table */
3214         dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3215                           lif->rss_ind_tbl_pa);
3216         lif->rss_ind_tbl = NULL;
3217         lif->rss_ind_tbl_pa = 0;
3218
3219         /* free queues */
3220         ionic_qcqs_free(lif);
3221         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3222                 ionic_lif_reset(lif);
3223
3224         /* free lif info */
3225         kfree(lif->identity);
3226         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3227         lif->info = NULL;
3228         lif->info_pa = 0;
3229
3230         /* unmap doorbell page */
3231         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3232         lif->kern_dbpage = NULL;
3233
3234         mutex_destroy(&lif->config_lock);
3235         mutex_destroy(&lif->queue_lock);
3236
3237         /* free netdev & lif */
3238         ionic_debugfs_del_lif(lif);
3239         free_netdev(lif->netdev);
3240 }
3241
3242 void ionic_lif_deinit(struct ionic_lif *lif)
3243 {
3244         if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3245                 return;
3246
3247         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3248                 cancel_work_sync(&lif->deferred.work);
3249                 cancel_work_sync(&lif->tx_timeout_work);
3250                 ionic_rx_filters_deinit(lif);
3251                 if (lif->netdev->features & NETIF_F_RXHASH)
3252                         ionic_lif_rss_deinit(lif);
3253         }
3254
3255         napi_disable(&lif->adminqcq->napi);
3256         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3257         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3258
3259         ionic_lif_reset(lif);
3260 }
3261
3262 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3263 {
3264         struct device *dev = lif->ionic->dev;
3265         struct ionic_q_init_comp comp;
3266         struct ionic_dev *idev;
3267         struct ionic_qcq *qcq;
3268         struct ionic_queue *q;
3269         int err;
3270
3271         idev = &lif->ionic->idev;
3272         qcq = lif->adminqcq;
3273         q = &qcq->q;
3274
3275         mutex_lock(&lif->ionic->dev_cmd_lock);
3276         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3277         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3278         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3279         mutex_unlock(&lif->ionic->dev_cmd_lock);
3280         if (err) {
3281                 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3282                 return err;
3283         }
3284
3285         q->hw_type = comp.hw_type;
3286         q->hw_index = le32_to_cpu(comp.hw_index);
3287         q->dbval = IONIC_DBELL_QID(q->hw_index);
3288
3289         dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3290         dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3291
3292         q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;
3293         q->dbell_jiffies = jiffies;
3294
3295         netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
3296
3297         qcq->napi_qcq = qcq;
3298         timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
3299
3300         napi_enable(&qcq->napi);
3301
3302         if (qcq->flags & IONIC_QCQ_F_INTR)
3303                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3304                                 IONIC_INTR_MASK_CLEAR);
3305
3306         qcq->flags |= IONIC_QCQ_F_INITED;
3307
3308         return 0;
3309 }
3310
3311 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3312 {
3313         struct ionic_qcq *qcq = lif->notifyqcq;
3314         struct device *dev = lif->ionic->dev;
3315         struct ionic_queue *q = &qcq->q;
3316         int err;
3317
3318         struct ionic_admin_ctx ctx = {
3319                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3320                 .cmd.q_init = {
3321                         .opcode = IONIC_CMD_Q_INIT,
3322                         .lif_index = cpu_to_le16(lif->index),
3323                         .type = q->type,
3324                         .ver = lif->qtype_info[q->type].version,
3325                         .index = cpu_to_le32(q->index),
3326                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3327                                              IONIC_QINIT_F_ENA),
3328                         .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3329                         .pid = cpu_to_le16(q->pid),
3330                         .ring_size = ilog2(q->num_descs),
3331                         .ring_base = cpu_to_le64(q->base_pa),
3332                 }
3333         };
3334
3335         dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3336         dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3337         dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3338         dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3339
3340         err = ionic_adminq_post_wait(lif, &ctx);
3341         if (err)
3342                 return err;
3343
3344         lif->last_eid = 0;
3345         q->hw_type = ctx.comp.q_init.hw_type;
3346         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3347         q->dbval = IONIC_DBELL_QID(q->hw_index);
3348
3349         dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3350         dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3351
3352         /* preset the callback info */
3353         q->info[0].cb_arg = lif;
3354
3355         qcq->flags |= IONIC_QCQ_F_INITED;
3356
3357         return 0;
3358 }
3359
3360 static int ionic_station_set(struct ionic_lif *lif)
3361 {
3362         struct net_device *netdev = lif->netdev;
3363         struct ionic_admin_ctx ctx = {
3364                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3365                 .cmd.lif_getattr = {
3366                         .opcode = IONIC_CMD_LIF_GETATTR,
3367                         .index = cpu_to_le16(lif->index),
3368                         .attr = IONIC_LIF_ATTR_MAC,
3369                 },
3370         };
3371         u8 mac_address[ETH_ALEN];
3372         struct sockaddr addr;
3373         int err;
3374
3375         err = ionic_adminq_post_wait(lif, &ctx);
3376         if (err)
3377                 return err;
3378         netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3379                    ctx.comp.lif_getattr.mac);
3380         ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);
3381
3382         if (is_zero_ether_addr(mac_address)) {
3383                 eth_hw_addr_random(netdev);
3384                 netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);
3385                 ether_addr_copy(mac_address, netdev->dev_addr);
3386
3387                 err = ionic_program_mac(lif, mac_address);
3388                 if (err < 0)
3389                         return err;
3390
3391                 if (err > 0) {
3392                         netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",
3393                                    __func__);
3394                         return 0;
3395                 }
3396         }
3397
3398         if (!is_zero_ether_addr(netdev->dev_addr)) {
3399                 /* If the netdev mac is non-zero and doesn't match the default
3400                  * device address, it was set by something earlier and we're
3401                  * likely here again after a fw-upgrade reset.  We need to be
3402                  * sure the netdev mac is in our filter list.
3403                  */
3404                 if (!ether_addr_equal(mac_address, netdev->dev_addr))
3405                         ionic_lif_addr_add(lif, netdev->dev_addr);
3406         } else {
3407                 /* Update the netdev mac with the device's mac */
3408                 ether_addr_copy(addr.sa_data, mac_address);
3409                 addr.sa_family = AF_INET;
3410                 err = eth_prepare_mac_addr_change(netdev, &addr);
3411                 if (err) {
3412                         netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3413                                     addr.sa_data, err);
3414                         return 0;
3415                 }
3416
3417                 eth_commit_mac_addr_change(netdev, &addr);
3418         }
3419
3420         netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3421                    netdev->dev_addr);
3422         ionic_lif_addr_add(lif, netdev->dev_addr);
3423
3424         return 0;
3425 }
3426
3427 int ionic_lif_init(struct ionic_lif *lif)
3428 {
3429         struct ionic_dev *idev = &lif->ionic->idev;
3430         struct device *dev = lif->ionic->dev;
3431         struct ionic_lif_init_comp comp;
3432         int dbpage_num;
3433         int err;
3434
3435         mutex_lock(&lif->ionic->dev_cmd_lock);
3436         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3437         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3438         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3439         mutex_unlock(&lif->ionic->dev_cmd_lock);
3440         if (err)
3441                 return err;
3442
3443         lif->hw_index = le16_to_cpu(comp.hw_index);
3444
3445         /* now that we have the hw_index we can figure out our doorbell page */
3446         lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3447         if (!lif->dbid_count) {
3448                 dev_err(dev, "No doorbell pages, aborting\n");
3449                 return -EINVAL;
3450         }
3451
3452         lif->kern_pid = 0;
3453         dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3454         lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3455         if (!lif->kern_dbpage) {
3456                 dev_err(dev, "Cannot map dbpage, aborting\n");
3457                 return -ENOMEM;
3458         }
3459
3460         err = ionic_lif_adminq_init(lif);
3461         if (err)
3462                 goto err_out_adminq_deinit;
3463
3464         if (lif->ionic->nnqs_per_lif) {
3465                 err = ionic_lif_notifyq_init(lif);
3466                 if (err)
3467                         goto err_out_notifyq_deinit;
3468         }
3469
3470         err = ionic_init_nic_features(lif);
3471         if (err)
3472                 goto err_out_notifyq_deinit;
3473
3474         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3475                 err = ionic_rx_filters_init(lif);
3476                 if (err)
3477                         goto err_out_notifyq_deinit;
3478         }
3479
3480         err = ionic_station_set(lif);
3481         if (err)
3482                 goto err_out_notifyq_deinit;
3483
3484         lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3485
3486         set_bit(IONIC_LIF_F_INITED, lif->state);
3487
3488         INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3489
3490         return 0;
3491
3492 err_out_notifyq_deinit:
3493         napi_disable(&lif->adminqcq->napi);
3494         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3495 err_out_adminq_deinit:
3496         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3497         ionic_lif_reset(lif);
3498         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3499         lif->kern_dbpage = NULL;
3500
3501         return err;
3502 }
3503
3504 static void ionic_lif_notify_work(struct work_struct *ws)
3505 {
3506 }
3507
3508 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3509 {
3510         struct ionic_admin_ctx ctx = {
3511                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3512                 .cmd.lif_setattr = {
3513                         .opcode = IONIC_CMD_LIF_SETATTR,
3514                         .index = cpu_to_le16(lif->index),
3515                         .attr = IONIC_LIF_ATTR_NAME,
3516                 },
3517         };
3518
3519         strscpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3520                 sizeof(ctx.cmd.lif_setattr.name));
3521
3522         ionic_adminq_post_wait(lif, &ctx);
3523 }
3524
3525 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3526 {
3527         if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3528                 return NULL;
3529
3530         return netdev_priv(netdev);
3531 }
3532
3533 static int ionic_lif_notify(struct notifier_block *nb,
3534                             unsigned long event, void *info)
3535 {
3536         struct net_device *ndev = netdev_notifier_info_to_dev(info);
3537         struct ionic *ionic = container_of(nb, struct ionic, nb);
3538         struct ionic_lif *lif = ionic_netdev_lif(ndev);
3539
3540         if (!lif || lif->ionic != ionic)
3541                 return NOTIFY_DONE;
3542
3543         switch (event) {
3544         case NETDEV_CHANGENAME:
3545                 ionic_lif_set_netdev_info(lif);
3546                 break;
3547         }
3548
3549         return NOTIFY_DONE;
3550 }
3551
3552 int ionic_lif_register(struct ionic_lif *lif)
3553 {
3554         int err;
3555
3556         ionic_lif_register_phc(lif);
3557
3558         INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3559
3560         lif->ionic->nb.notifier_call = ionic_lif_notify;
3561
3562         err = register_netdevice_notifier(&lif->ionic->nb);
3563         if (err)
3564                 lif->ionic->nb.notifier_call = NULL;
3565
3566         /* only register LIF0 for now */
3567         err = register_netdev(lif->netdev);
3568         if (err) {
3569                 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3570                 ionic_lif_unregister_phc(lif);
3571                 return err;
3572         }
3573
3574         ionic_link_status_check_request(lif, CAN_SLEEP);
3575         lif->registered = true;
3576         ionic_lif_set_netdev_info(lif);
3577
3578         return 0;
3579 }
3580
3581 void ionic_lif_unregister(struct ionic_lif *lif)
3582 {
3583         if (lif->ionic->nb.notifier_call) {
3584                 unregister_netdevice_notifier(&lif->ionic->nb);
3585                 cancel_work_sync(&lif->ionic->nb_work);
3586                 lif->ionic->nb.notifier_call = NULL;
3587         }
3588
3589         if (lif->netdev->reg_state == NETREG_REGISTERED)
3590                 unregister_netdev(lif->netdev);
3591
3592         ionic_lif_unregister_phc(lif);
3593
3594         lif->registered = false;
3595 }
3596
3597 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3598 {
3599         union ionic_q_identity __iomem *q_ident;
3600         struct ionic *ionic = lif->ionic;
3601         struct ionic_dev *idev;
3602         int qtype;
3603         int err;
3604
3605         idev = &lif->ionic->idev;
3606         q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3607
3608         for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3609                 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3610
3611                 /* filter out the ones we know about */
3612                 switch (qtype) {
3613                 case IONIC_QTYPE_ADMINQ:
3614                 case IONIC_QTYPE_NOTIFYQ:
3615                 case IONIC_QTYPE_RXQ:
3616                 case IONIC_QTYPE_TXQ:
3617                         break;
3618                 default:
3619                         continue;
3620                 }
3621
3622                 memset(qti, 0, sizeof(*qti));
3623
3624                 mutex_lock(&ionic->dev_cmd_lock);
3625                 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3626                                              ionic_qtype_versions[qtype]);
3627                 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3628                 if (!err) {
3629                         qti->version   = readb(&q_ident->version);
3630                         qti->supported = readb(&q_ident->supported);
3631                         qti->features  = readq(&q_ident->features);
3632                         qti->desc_sz   = readw(&q_ident->desc_sz);
3633                         qti->comp_sz   = readw(&q_ident->comp_sz);
3634                         qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3635                         qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3636                         qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3637                 }
3638                 mutex_unlock(&ionic->dev_cmd_lock);
3639
3640                 if (err == -EINVAL) {
3641                         dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3642                         continue;
3643                 } else if (err == -EIO) {
3644                         dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3645                         return;
3646                 } else if (err) {
3647                         dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3648                                 qtype, err);
3649                         return;
3650                 }
3651
3652                 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3653                         qtype, qti->version);
3654                 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3655                         qtype, qti->supported);
3656                 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3657                         qtype, qti->features);
3658                 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3659                         qtype, qti->desc_sz);
3660                 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3661                         qtype, qti->comp_sz);
3662                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3663                         qtype, qti->sg_desc_sz);
3664                 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3665                         qtype, qti->max_sg_elems);
3666                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3667                         qtype, qti->sg_desc_stride);
3668         }
3669 }
3670
3671 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3672                        union ionic_lif_identity *lid)
3673 {
3674         struct ionic_dev *idev = &ionic->idev;
3675         size_t sz;
3676         int err;
3677
3678         sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3679
3680         mutex_lock(&ionic->dev_cmd_lock);
3681         ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3682         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3683         memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3684         mutex_unlock(&ionic->dev_cmd_lock);
3685         if (err)
3686                 return (err);
3687
3688         dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3689                 le64_to_cpu(lid->capabilities));
3690
3691         dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3692                 le32_to_cpu(lid->eth.max_ucast_filters));
3693         dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3694                 le32_to_cpu(lid->eth.max_mcast_filters));
3695         dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3696                 le64_to_cpu(lid->eth.config.features));
3697         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3698                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3699         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3700                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3701         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3702                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3703         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3704                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3705         dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3706         dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3707         dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3708                 le32_to_cpu(lid->eth.config.mtu));
3709
3710         return 0;
3711 }
3712
3713 int ionic_lif_size(struct ionic *ionic)
3714 {
3715         struct ionic_identity *ident = &ionic->ident;
3716         unsigned int nintrs, dev_nintrs;
3717         union ionic_lif_config *lc;
3718         unsigned int ntxqs_per_lif;
3719         unsigned int nrxqs_per_lif;
3720         unsigned int neqs_per_lif;
3721         unsigned int nnqs_per_lif;
3722         unsigned int nxqs, neqs;
3723         unsigned int min_intrs;
3724         int err;
3725
3726         /* retrieve basic values from FW */
3727         lc = &ident->lif.eth.config;
3728         dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3729         neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3730         nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3731         ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3732         nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3733
3734         /* limit values to play nice with kdump */
3735         if (is_kdump_kernel()) {
3736                 dev_nintrs = 2;
3737                 neqs_per_lif = 0;
3738                 nnqs_per_lif = 0;
3739                 ntxqs_per_lif = 1;
3740                 nrxqs_per_lif = 1;
3741         }
3742
3743         /* reserve last queue id for hardware timestamping */
3744         if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3745                 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3746                         lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3747                 } else {
3748                         ntxqs_per_lif -= 1;
3749                         nrxqs_per_lif -= 1;
3750                 }
3751         }
3752
3753         nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3754         nxqs = min(nxqs, num_online_cpus());
3755         neqs = min(neqs_per_lif, num_online_cpus());
3756
3757 try_again:
3758         /* interrupt usage:
3759          *    1 for master lif adminq/notifyq
3760          *    1 for each CPU for master lif TxRx queue pairs
3761          *    whatever's left is for RDMA queues
3762          */
3763         nintrs = 1 + nxqs + neqs;
3764         min_intrs = 2;  /* adminq + 1 TxRx queue pair */
3765
3766         if (nintrs > dev_nintrs)
3767                 goto try_fewer;
3768
3769         err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3770         if (err < 0 && err != -ENOSPC) {
3771                 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3772                 return err;
3773         }
3774         if (err == -ENOSPC)
3775                 goto try_fewer;
3776
3777         if (err != nintrs) {
3778                 ionic_bus_free_irq_vectors(ionic);
3779                 goto try_fewer;
3780         }
3781
3782         ionic->nnqs_per_lif = nnqs_per_lif;
3783         ionic->neqs_per_lif = neqs;
3784         ionic->ntxqs_per_lif = nxqs;
3785         ionic->nrxqs_per_lif = nxqs;
3786         ionic->nintrs = nintrs;
3787
3788         ionic_debugfs_add_sizes(ionic);
3789
3790         return 0;
3791
3792 try_fewer:
3793         if (nnqs_per_lif > 1) {
3794                 nnqs_per_lif >>= 1;
3795                 goto try_again;
3796         }
3797         if (neqs > 1) {
3798                 neqs >>= 1;
3799                 goto try_again;
3800         }
3801         if (nxqs > 1) {
3802                 nxqs >>= 1;
3803                 goto try_again;
3804         }
3805         dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3806         return -ENOSPC;
3807 }