drivers/net/wireless/mediatek/mt76/dma.c

   1 /*
   2  * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
   3  *
   4  * Permission to use, copy, modify, and/or distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include <linux/dma-mapping.h>
  18 #include "mt76.h"
  19 #include "dma.h"
  20
  21 #define DMA_DUMMY_TXWI  ((void *) ~0)
  22
  23 static int
  24 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q)
  25 {
  26         int size;
  27         int i;
  28
  29         spin_lock_init(&q->lock);
  30         INIT_LIST_HEAD(&q->swq);
  31
  32         size = q->ndesc * sizeof(struct mt76_desc);
  33         q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
  34         if (!q->desc)
  35                 return -ENOMEM;
  36
  37         size = q->ndesc * sizeof(*q->entry);
  38         q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
  39         if (!q->entry)
  40                 return -ENOMEM;
  41
  42         /* clear descriptors */
  43         for (i = 0; i < q->ndesc; i++)
  44                 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  45
  46         iowrite32(q->desc_dma, &q->regs->desc_base);
  47         iowrite32(0, &q->regs->cpu_idx);
  48         iowrite32(0, &q->regs->dma_idx);
  49         iowrite32(q->ndesc, &q->regs->ring_size);
  50
  51         return 0;
  52 }
  53
  54 static int
  55 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
  56                  struct mt76_queue_buf *buf, int nbufs, u32 info,
  57                  struct sk_buff *skb, void *txwi)
  58 {
  59         struct mt76_desc *desc;
  60         u32 ctrl;
  61         int i, idx = -1;
  62
  63         if (txwi)
  64                 q->entry[q->head].txwi = DMA_DUMMY_TXWI;
  65
  66         for (i = 0; i < nbufs; i += 2, buf += 2) {
  67                 u32 buf0 = buf[0].addr, buf1 = 0;
  68
  69                 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
  70                 if (i < nbufs - 1) {
  71                         buf1 = buf[1].addr;
  72                         ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
  73                 }
  74
  75                 if (i == nbufs - 1)
  76                         ctrl |= MT_DMA_CTL_LAST_SEC0;
  77                 else if (i == nbufs - 2)
  78                         ctrl |= MT_DMA_CTL_LAST_SEC1;
  79
  80                 idx = q->head;
  81                 q->head = (q->head + 1) % q->ndesc;
  82
  83                 desc = &q->desc[idx];
  84
  85                 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
  86                 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
  87                 WRITE_ONCE(desc->info, cpu_to_le32(info));
  88                 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
  89
  90                 q->queued++;
  91         }
  92
  93         q->entry[idx].txwi = txwi;
  94         q->entry[idx].skb = skb;
  95
  96         return idx;
  97 }
  98
  99 static void
 100 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 101                         struct mt76_queue_entry *prev_e)
 102 {
 103         struct mt76_queue_entry *e = &q->entry[idx];
 104         __le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
 105         u32 ctrl = le32_to_cpu(__ctrl);
 106
 107         if (!e->txwi || !e->skb) {
 108                 __le32 addr = READ_ONCE(q->desc[idx].buf0);
 109                 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
 110
 111                 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
 112                                  DMA_TO_DEVICE);
 113         }
 114
 115         if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
 116                 __le32 addr = READ_ONCE(q->desc[idx].buf1);
 117                 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);
 118
 119                 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
 120                                  DMA_TO_DEVICE);
 121         }
 122
 123         if (e->txwi == DMA_DUMMY_TXWI)
 124                 e->txwi = NULL;
 125
 126         *prev_e = *e;
 127         memset(e, 0, sizeof(*e));
 128 }
 129
 130 static void
 131 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
 132 {
 133         q->head = ioread32(&q->regs->dma_idx);
 134         q->tail = q->head;
 135         iowrite32(q->head, &q->regs->cpu_idx);
 136 }
 137
 138 static void
 139 mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
 140 {
 141         struct mt76_queue *q = &dev->q_tx[qid];
 142         struct mt76_queue_entry entry;
 143         bool wake = false;
 144         int last;
 145
 146         if (!q->ndesc)
 147                 return;
 148
 149         spin_lock_bh(&q->lock);
 150         if (flush)
 151                 last = -1;
 152         else
 153                 last = ioread32(&q->regs->dma_idx);
 154
 155         while (q->queued && q->tail != last) {
 156                 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
 157                 if (entry.schedule)
 158                         q->swq_queued--;
 159
 160                 q->tail = (q->tail + 1) % q->ndesc;
 161                 q->queued--;
 162
 163                 if (entry.skb) {
 164                         spin_unlock_bh(&q->lock);
 165                         dev->drv->tx_complete_skb(dev, q, &entry, flush);
 166                         spin_lock_bh(&q->lock);
 167                 }
 168
 169                 if (entry.txwi) {
 170                         mt76_put_txwi(dev, entry.txwi);
 171                         wake = !flush;
 172                 }
 173
 174                 if (!flush && q->tail == last)
 175                         last = ioread32(&q->regs->dma_idx);
 176         }
 177
 178         if (!flush)
 179                 mt76_txq_schedule(dev, q);
 180         else
 181                 mt76_dma_sync_idx(dev, q);
 182
 183         wake = wake && qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
 184
 185         if (!q->queued)
 186                 wake_up(&dev->tx_wait);
 187
 188         spin_unlock_bh(&q->lock);
 189
 190         if (wake)
 191                 ieee80211_wake_queue(dev->hw, qid);
 192 }
 193
 194 static void *
 195 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 196                  int *len, u32 *info, bool *more)
 197 {
 198         struct mt76_queue_entry *e = &q->entry[idx];
 199         struct mt76_desc *desc = &q->desc[idx];
 200         dma_addr_t buf_addr;
 201         void *buf = e->buf;
 202         int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
 203
 204         buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
 205         if (len) {
 206                 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
 207                 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
 208                 *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
 209         }
 210
 211         if (info)
 212                 *info = le32_to_cpu(desc->info);
 213
 214         dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
 215         e->buf = NULL;
 216
 217         return buf;
 218 }
 219
 220 static void *
 221 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
 222                  int *len, u32 *info, bool *more)
 223 {
 224         int idx = q->tail;
 225
 226         *more = false;
 227         if (!q->queued)
 228                 return NULL;
 229
 230         if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
 231                 return NULL;
 232
 233         q->tail = (q->tail + 1) % q->ndesc;
 234         q->queued--;
 235
 236         return mt76_dma_get_buf(dev, q, idx, len, info, more);
 237 }
 238
 239 static void
 240 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
 241 {
 242         iowrite32(q->head, &q->regs->cpu_idx);
 243 }
 244
 245 static int
 246 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, enum mt76_txq_id qid,
 247                           struct sk_buff *skb, u32 tx_info)
 248 {
 249         struct mt76_queue *q = &dev->q_tx[qid];
 250         struct mt76_queue_buf buf;
 251         dma_addr_t addr;
 252
 253         addr = dma_map_single(dev->dev, skb->data, skb->len,
 254                               DMA_TO_DEVICE);
 255         if (dma_mapping_error(dev->dev, addr))
 256                 return -ENOMEM;
 257
 258         buf.addr = addr;
 259         buf.len = skb->len;
 260
 261         spin_lock_bh(&q->lock);
 262         mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
 263         mt76_dma_kick_queue(dev, q);
 264         spin_unlock_bh(&q->lock);
 265
 266         return 0;
 267 }
 268
 269 int mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
 270                           struct sk_buff *skb, struct mt76_wcid *wcid,
 271                           struct ieee80211_sta *sta)
 272 {
 273         struct mt76_queue_entry e;
 274         struct mt76_txwi_cache *t;
 275         struct mt76_queue_buf buf[32];
 276         struct sk_buff *iter;
 277         dma_addr_t addr;
 278         int len;
 279         u32 tx_info = 0;
 280         int n, ret;
 281
 282         t = mt76_get_txwi(dev);
 283         if (!t) {
 284                 ieee80211_free_txskb(dev->hw, skb);
 285                 return -ENOMEM;
 286         }
 287
 288         skb->prev = skb->next = NULL;
 289         dma_sync_single_for_cpu(dev->dev, t->dma_addr, sizeof(t->txwi),
 290                                 DMA_TO_DEVICE);
 291         ret = dev->drv->tx_prepare_skb(dev, &t->txwi, skb, q, wcid, sta,
 292                                        &tx_info);
 293         dma_sync_single_for_device(dev->dev, t->dma_addr, sizeof(t->txwi),
 294                                    DMA_TO_DEVICE);
 295         if (ret < 0)
 296                 goto free;
 297
 298         len = skb->len - skb->data_len;
 299         addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
 300         if (dma_mapping_error(dev->dev, addr)) {
 301                 ret = -ENOMEM;
 302                 goto free;
 303         }
 304
 305         n = 0;
 306         buf[n].addr = t->dma_addr;
 307         buf[n++].len = dev->drv->txwi_size;
 308         buf[n].addr = addr;
 309         buf[n++].len = len;
 310
 311         skb_walk_frags(skb, iter) {
 312                 if (n == ARRAY_SIZE(buf))
 313                         goto unmap;
 314
 315                 addr = dma_map_single(dev->dev, iter->data, iter->len,
 316                                       DMA_TO_DEVICE);
 317                 if (dma_mapping_error(dev->dev, addr))
 318                         goto unmap;
 319
 320                 buf[n].addr = addr;
 321                 buf[n++].len = iter->len;
 322         }
 323
 324         if (q->queued + (n + 1) / 2 >= q->ndesc - 1)
 325                 goto unmap;
 326
 327         return mt76_dma_add_buf(dev, q, buf, n, tx_info, skb, t);
 328
 329 unmap:
 330         ret = -ENOMEM;
 331         for (n--; n > 0; n--)
 332                 dma_unmap_single(dev->dev, buf[n].addr, buf[n].len,
 333                                  DMA_TO_DEVICE);
 334
 335 free:
 336         e.skb = skb;
 337         e.txwi = t;
 338         dev->drv->tx_complete_skb(dev, q, &e, true);
 339         mt76_put_txwi(dev, t);
 340         return ret;
 341 }
 342 EXPORT_SYMBOL_GPL(mt76_dma_tx_queue_skb);
 343
 344 static int
 345 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
 346 {
 347         dma_addr_t addr;
 348         void *buf;
 349         int frames = 0;
 350         int len = SKB_WITH_OVERHEAD(q->buf_size);
 351         int offset = q->buf_offset;
 352         int idx;
 353
 354         spin_lock_bh(&q->lock);
 355
 356         while (q->queued < q->ndesc - 1) {
 357                 struct mt76_queue_buf qbuf;
 358
 359                 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
 360                 if (!buf)
 361                         break;
 362
 363                 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
 364                 if (dma_mapping_error(dev->dev, addr)) {
 365                         skb_free_frag(buf);
 366                         break;
 367                 }
 368
 369                 qbuf.addr = addr + offset;
 370                 qbuf.len = len - offset;
 371                 idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
 372                 frames++;
 373         }
 374
 375         if (frames)
 376                 mt76_dma_kick_queue(dev, q);
 377
 378         spin_unlock_bh(&q->lock);
 379
 380         return frames;
 381 }
 382
 383 static void
 384 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
 385 {
 386         struct page *page;
 387         void *buf;
 388         bool more;
 389
 390         spin_lock_bh(&q->lock);
 391         do {
 392                 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
 393                 if (!buf)
 394                         break;
 395
 396                 skb_free_frag(buf);
 397         } while (1);
 398         spin_unlock_bh(&q->lock);
 399
 400         if (!q->rx_page.va)
 401                 return;
 402
 403         page = virt_to_page(q->rx_page.va);
 404         __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
 405         memset(&q->rx_page, 0, sizeof(q->rx_page));
 406 }
 407
 408 static void
 409 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
 410 {
 411         struct mt76_queue *q = &dev->q_rx[qid];
 412         int i;
 413
 414         for (i = 0; i < q->ndesc; i++)
 415                 q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);
 416
 417         mt76_dma_rx_cleanup(dev, q);
 418         mt76_dma_sync_idx(dev, q);
 419         mt76_dma_rx_fill(dev, q);
 420 }
 421
 422 static void
 423 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
 424                   int len, bool more)
 425 {
 426         struct page *page = virt_to_head_page(data);
 427         int offset = data - page_address(page);
 428         struct sk_buff *skb = q->rx_head;
 429
 430         offset += q->buf_offset;
 431         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len,
 432                         q->buf_size);
 433
 434         if (more)
 435                 return;
 436
 437         q->rx_head = NULL;
 438         dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 439 }
 440
 441 static int
 442 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
 443 {
 444         int len, data_len, done = 0;
 445         struct sk_buff *skb;
 446         unsigned char *data;
 447         bool more;
 448
 449         while (done < budget) {
 450                 u32 info;
 451
 452                 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
 453                 if (!data)
 454                         break;
 455
 456                 if (q->rx_head)
 457                         data_len = q->buf_size;
 458                 else
 459                         data_len = SKB_WITH_OVERHEAD(q->buf_size);
 460
 461                 if (data_len < len + q->buf_offset) {
 462                         dev_kfree_skb(q->rx_head);
 463                         q->rx_head = NULL;
 464
 465                         skb_free_frag(data);
 466                         continue;
 467                 }
 468
 469                 if (q->rx_head) {
 470                         mt76_add_fragment(dev, q, data, len, more);
 471                         continue;
 472                 }
 473
 474                 skb = build_skb(data, q->buf_size);
 475                 if (!skb) {
 476                         skb_free_frag(data);
 477                         continue;
 478                 }
 479                 skb_reserve(skb, q->buf_offset);
 480
 481                 if (q == &dev->q_rx[MT_RXQ_MCU]) {
 482                         u32 *rxfce = (u32 *) skb->cb;
 483                         *rxfce = info;
 484                 }
 485
 486                 __skb_put(skb, len);
 487                 done++;
 488
 489                 if (more) {
 490                         q->rx_head = skb;
 491                         continue;
 492                 }
 493
 494                 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 495         }
 496
 497         mt76_dma_rx_fill(dev, q);
 498         return done;
 499 }
 500
 501 static int
 502 mt76_dma_rx_poll(struct napi_struct *napi, int budget)
 503 {
 504         struct mt76_dev *dev;
 505         int qid, done = 0, cur;
 506
 507         dev = container_of(napi->dev, struct mt76_dev, napi_dev);
 508         qid = napi - dev->napi;
 509
 510         rcu_read_lock();
 511
 512         do {
 513                 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
 514                 mt76_rx_poll_complete(dev, qid, napi);
 515                 done += cur;
 516         } while (cur && done < budget);
 517
 518         rcu_read_unlock();
 519
 520         if (done < budget) {
 521                 napi_complete(napi);
 522                 dev->drv->rx_poll_complete(dev, qid);
 523         }
 524
 525         return done;
 526 }
 527
 528 static int
 529 mt76_dma_init(struct mt76_dev *dev)
 530 {
 531         int i;
 532
 533         init_dummy_netdev(&dev->napi_dev);
 534
 535         for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
 536                 netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
 537                                64);
 538                 mt76_dma_rx_fill(dev, &dev->q_rx[i]);
 539                 skb_queue_head_init(&dev->rx_skb[i]);
 540                 napi_enable(&dev->napi[i]);
 541         }
 542
 543         return 0;
 544 }
 545
 546 static const struct mt76_queue_ops mt76_dma_ops = {
 547         .init = mt76_dma_init,
 548         .alloc = mt76_dma_alloc_queue,
 549         .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
 550         .tx_queue_skb = mt76_dma_tx_queue_skb,
 551         .tx_cleanup = mt76_dma_tx_cleanup,
 552         .rx_reset = mt76_dma_rx_reset,
 553         .kick = mt76_dma_kick_queue,
 554 };
 555
 556 void mt76_dma_attach(struct mt76_dev *dev)
 557 {
 558         dev->queue_ops = &mt76_dma_ops;
 559 }
 560 EXPORT_SYMBOL_GPL(mt76_dma_attach);
 561
 562 void mt76_dma_cleanup(struct mt76_dev *dev)
 563 {
 564         int i;
 565
 566         for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
 567                 mt76_dma_tx_cleanup(dev, i, true);
 568
 569         for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
 570                 netif_napi_del(&dev->napi[i]);
 571                 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
 572         }
 573 }
 574 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);