drivers/mailbox/bcm-flexrm-mailbox.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright (C) 2017 Broadcom
   3
   4 /*
   5  * Broadcom FlexRM Mailbox Driver
   6  *
   7  * Each Broadcom FlexSparx4 offload engine is implemented as an
   8  * extension to Broadcom FlexRM ring manager. The FlexRM ring
   9  * manager provides a set of rings which can be used to submit
  10  * work to a FlexSparx4 offload engine.
  11  *
  12  * This driver creates a mailbox controller using a set of FlexRM
  13  * rings where each mailbox channel represents a separate FlexRM ring.
  14  */
  15
  16 #include <asm/barrier.h>
  17 #include <asm/byteorder.h>
  18 #include <linux/atomic.h>
  19 #include <linux/bitmap.h>
  20 #include <linux/debugfs.h>
  21 #include <linux/delay.h>
  22 #include <linux/device.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/dmapool.h>
  25 #include <linux/err.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/kernel.h>
  28 #include <linux/mailbox_controller.h>
  29 #include <linux/mailbox_client.h>
  30 #include <linux/mailbox/brcm-message.h>
  31 #include <linux/module.h>
  32 #include <linux/msi.h>
  33 #include <linux/of_address.h>
  34 #include <linux/of_irq.h>
  35 #include <linux/platform_device.h>
  36 #include <linux/spinlock.h>
  37
  38 /* ====== FlexRM register defines ===== */
  39
  40 /* FlexRM configuration */
  41 #define RING_REGS_SIZE                                  0x10000
  42 #define RING_DESC_SIZE                                  8
  43 #define RING_DESC_INDEX(offset)                         \
  44                         ((offset) / RING_DESC_SIZE)
  45 #define RING_DESC_OFFSET(index)                         \
  46                         ((index) * RING_DESC_SIZE)
  47 #define RING_MAX_REQ_COUNT                              1024
  48 #define RING_BD_ALIGN_ORDER                             12
  49 #define RING_BD_ALIGN_CHECK(addr)                       \
  50                         (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
  51 #define RING_BD_TOGGLE_INVALID(offset)                  \
  52                         (((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
  53 #define RING_BD_TOGGLE_VALID(offset)                    \
  54                         (!RING_BD_TOGGLE_INVALID(offset))
  55 #define RING_BD_DESC_PER_REQ                            32
  56 #define RING_BD_DESC_COUNT                              \
  57                         (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
  58 #define RING_BD_SIZE                                    \
  59                         (RING_BD_DESC_COUNT * RING_DESC_SIZE)
  60 #define RING_CMPL_ALIGN_ORDER                           13
  61 #define RING_CMPL_DESC_COUNT                            RING_MAX_REQ_COUNT
  62 #define RING_CMPL_SIZE                                  \
  63                         (RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
  64 #define RING_VER_MAGIC                                  0x76303031
  65
  66 /* Per-Ring register offsets */
  67 #define RING_VER                                        0x000
  68 #define RING_BD_START_ADDR                              0x004
  69 #define RING_BD_READ_PTR                                0x008
  70 #define RING_BD_WRITE_PTR                               0x00c
  71 #define RING_BD_READ_PTR_DDR_LS                         0x010
  72 #define RING_BD_READ_PTR_DDR_MS                         0x014
  73 #define RING_CMPL_START_ADDR                            0x018
  74 #define RING_CMPL_WRITE_PTR                             0x01c
  75 #define RING_NUM_REQ_RECV_LS                            0x020
  76 #define RING_NUM_REQ_RECV_MS                            0x024
  77 #define RING_NUM_REQ_TRANS_LS                           0x028
  78 #define RING_NUM_REQ_TRANS_MS                           0x02c
  79 #define RING_NUM_REQ_OUTSTAND                           0x030
  80 #define RING_CONTROL                                    0x034
  81 #define RING_FLUSH_DONE                                 0x038
  82 #define RING_MSI_ADDR_LS                                0x03c
  83 #define RING_MSI_ADDR_MS                                0x040
  84 #define RING_MSI_CONTROL                                0x048
  85 #define RING_BD_READ_PTR_DDR_CONTROL                    0x04c
  86 #define RING_MSI_DATA_VALUE                             0x064
  87
  88 /* Register RING_BD_START_ADDR fields */
  89 #define BD_LAST_UPDATE_HW_SHIFT                         28
  90 #define BD_LAST_UPDATE_HW_MASK                          0x1
  91 #define BD_START_ADDR_VALUE(pa)                         \
  92         ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
  93 #define BD_START_ADDR_DECODE(val)                       \
  94         ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
  95
  96 /* Register RING_CMPL_START_ADDR fields */
  97 #define CMPL_START_ADDR_VALUE(pa)                       \
  98         ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
  99
 100 /* Register RING_CONTROL fields */
 101 #define CONTROL_MASK_DISABLE_CONTROL                    12
 102 #define CONTROL_FLUSH_SHIFT                             5
 103 #define CONTROL_ACTIVE_SHIFT                            4
 104 #define CONTROL_RATE_ADAPT_MASK                         0xf
 105 #define CONTROL_RATE_DYNAMIC                            0x0
 106 #define CONTROL_RATE_FAST                               0x8
 107 #define CONTROL_RATE_MEDIUM                             0x9
 108 #define CONTROL_RATE_SLOW                               0xa
 109 #define CONTROL_RATE_IDLE                               0xb
 110
 111 /* Register RING_FLUSH_DONE fields */
 112 #define FLUSH_DONE_MASK                                 0x1
 113
 114 /* Register RING_MSI_CONTROL fields */
 115 #define MSI_TIMER_VAL_SHIFT                             16
 116 #define MSI_TIMER_VAL_MASK                              0xffff
 117 #define MSI_ENABLE_SHIFT                                15
 118 #define MSI_ENABLE_MASK                                 0x1
 119 #define MSI_COUNT_SHIFT                                 0
 120 #define MSI_COUNT_MASK                                  0x3ff
 121
 122 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
 123 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT                 16
 124 #define BD_READ_PTR_DDR_TIMER_VAL_MASK                  0xffff
 125 #define BD_READ_PTR_DDR_ENABLE_SHIFT                    15
 126 #define BD_READ_PTR_DDR_ENABLE_MASK                     0x1
 127
 128 /* ====== FlexRM ring descriptor defines ===== */
 129
 130 /* Completion descriptor format */
 131 #define CMPL_OPAQUE_SHIFT                       0
 132 #define CMPL_OPAQUE_MASK                        0xffff
 133 #define CMPL_ENGINE_STATUS_SHIFT                16
 134 #define CMPL_ENGINE_STATUS_MASK                 0xffff
 135 #define CMPL_DME_STATUS_SHIFT                   32
 136 #define CMPL_DME_STATUS_MASK                    0xffff
 137 #define CMPL_RM_STATUS_SHIFT                    48
 138 #define CMPL_RM_STATUS_MASK                     0xffff
 139
 140 /* Completion DME status code */
 141 #define DME_STATUS_MEM_COR_ERR                  BIT(0)
 142 #define DME_STATUS_MEM_UCOR_ERR                 BIT(1)
 143 #define DME_STATUS_FIFO_UNDERFLOW               BIT(2)
 144 #define DME_STATUS_FIFO_OVERFLOW                BIT(3)
 145 #define DME_STATUS_RRESP_ERR                    BIT(4)
 146 #define DME_STATUS_BRESP_ERR                    BIT(5)
 147 #define DME_STATUS_ERROR_MASK                   (DME_STATUS_MEM_COR_ERR | \
 148                                                  DME_STATUS_MEM_UCOR_ERR | \
 149                                                  DME_STATUS_FIFO_UNDERFLOW | \
 150                                                  DME_STATUS_FIFO_OVERFLOW | \
 151                                                  DME_STATUS_RRESP_ERR | \
 152                                                  DME_STATUS_BRESP_ERR)
 153
 154 /* Completion RM status code */
 155 #define RM_STATUS_CODE_SHIFT                    0
 156 #define RM_STATUS_CODE_MASK                     0x3ff
 157 #define RM_STATUS_CODE_GOOD                     0x0
 158 #define RM_STATUS_CODE_AE_TIMEOUT               0x3ff
 159
 160 /* General descriptor format */
 161 #define DESC_TYPE_SHIFT                         60
 162 #define DESC_TYPE_MASK                          0xf
 163 #define DESC_PAYLOAD_SHIFT                      0
 164 #define DESC_PAYLOAD_MASK                       0x0fffffffffffffff
 165
 166 /* Null descriptor format  */
 167 #define NULL_TYPE                               0
 168 #define NULL_TOGGLE_SHIFT                       58
 169 #define NULL_TOGGLE_MASK                        0x1
 170
 171 /* Header descriptor format */
 172 #define HEADER_TYPE                             1
 173 #define HEADER_TOGGLE_SHIFT                     58
 174 #define HEADER_TOGGLE_MASK                      0x1
 175 #define HEADER_ENDPKT_SHIFT                     57
 176 #define HEADER_ENDPKT_MASK                      0x1
 177 #define HEADER_STARTPKT_SHIFT                   56
 178 #define HEADER_STARTPKT_MASK                    0x1
 179 #define HEADER_BDCOUNT_SHIFT                    36
 180 #define HEADER_BDCOUNT_MASK                     0x1f
 181 #define HEADER_BDCOUNT_MAX                      HEADER_BDCOUNT_MASK
 182 #define HEADER_FLAGS_SHIFT                      16
 183 #define HEADER_FLAGS_MASK                       0xffff
 184 #define HEADER_OPAQUE_SHIFT                     0
 185 #define HEADER_OPAQUE_MASK                      0xffff
 186
 187 /* Source (SRC) descriptor format */
 188 #define SRC_TYPE                                2
 189 #define SRC_LENGTH_SHIFT                        44
 190 #define SRC_LENGTH_MASK                         0xffff
 191 #define SRC_ADDR_SHIFT                          0
 192 #define SRC_ADDR_MASK                           0x00000fffffffffff
 193
 194 /* Destination (DST) descriptor format */
 195 #define DST_TYPE                                3
 196 #define DST_LENGTH_SHIFT                        44
 197 #define DST_LENGTH_MASK                         0xffff
 198 #define DST_ADDR_SHIFT                          0
 199 #define DST_ADDR_MASK                           0x00000fffffffffff
 200
 201 /* Immediate (IMM) descriptor format */
 202 #define IMM_TYPE                                4
 203 #define IMM_DATA_SHIFT                          0
 204 #define IMM_DATA_MASK                           0x0fffffffffffffff
 205
 206 /* Next pointer (NPTR) descriptor format */
 207 #define NPTR_TYPE                               5
 208 #define NPTR_TOGGLE_SHIFT                       58
 209 #define NPTR_TOGGLE_MASK                        0x1
 210 #define NPTR_ADDR_SHIFT                         0
 211 #define NPTR_ADDR_MASK                          0x00000fffffffffff
 212
 213 /* Mega source (MSRC) descriptor format */
 214 #define MSRC_TYPE                               6
 215 #define MSRC_LENGTH_SHIFT                       44
 216 #define MSRC_LENGTH_MASK                        0xffff
 217 #define MSRC_ADDR_SHIFT                         0
 218 #define MSRC_ADDR_MASK                          0x00000fffffffffff
 219
 220 /* Mega destination (MDST) descriptor format */
 221 #define MDST_TYPE                               7
 222 #define MDST_LENGTH_SHIFT                       44
 223 #define MDST_LENGTH_MASK                        0xffff
 224 #define MDST_ADDR_SHIFT                         0
 225 #define MDST_ADDR_MASK                          0x00000fffffffffff
 226
 227 /* Source with tlast (SRCT) descriptor format */
 228 #define SRCT_TYPE                               8
 229 #define SRCT_LENGTH_SHIFT                       44
 230 #define SRCT_LENGTH_MASK                        0xffff
 231 #define SRCT_ADDR_SHIFT                         0
 232 #define SRCT_ADDR_MASK                          0x00000fffffffffff
 233
 234 /* Destination with tlast (DSTT) descriptor format */
 235 #define DSTT_TYPE                               9
 236 #define DSTT_LENGTH_SHIFT                       44
 237 #define DSTT_LENGTH_MASK                        0xffff
 238 #define DSTT_ADDR_SHIFT                         0
 239 #define DSTT_ADDR_MASK                          0x00000fffffffffff
 240
 241 /* Immediate with tlast (IMMT) descriptor format */
 242 #define IMMT_TYPE                               10
 243 #define IMMT_DATA_SHIFT                         0
 244 #define IMMT_DATA_MASK                          0x0fffffffffffffff
 245
 246 /* Descriptor helper macros */
 247 #define DESC_DEC(_d, _s, _m)                    (((_d) >> (_s)) & (_m))
 248 #define DESC_ENC(_d, _v, _s, _m)                \
 249                         do { \
 250                                 (_d) &= ~((u64)(_m) << (_s)); \
 251                                 (_d) |= (((u64)(_v) & (_m)) << (_s)); \
 252                         } while (0)
 253
 254 /* ====== FlexRM data structures ===== */
 255
 256 struct flexrm_ring {
 257         /* Unprotected members */
 258         int num;
 259         struct flexrm_mbox *mbox;
 260         void __iomem *regs;
 261         bool irq_requested;
 262         unsigned int irq;
 263         cpumask_t irq_aff_hint;
 264         unsigned int msi_timer_val;
 265         unsigned int msi_count_threshold;
 266         struct brcm_message *requests[RING_MAX_REQ_COUNT];
 267         void *bd_base;
 268         dma_addr_t bd_dma_base;
 269         u32 bd_write_offset;
 270         void *cmpl_base;
 271         dma_addr_t cmpl_dma_base;
 272         /* Atomic stats */
 273         atomic_t msg_send_count;
 274         atomic_t msg_cmpl_count;
 275         /* Protected members */
 276         spinlock_t lock;
 277         DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
 278         u32 cmpl_read_offset;
 279 };
 280
 281 struct flexrm_mbox {
 282         struct device *dev;
 283         void __iomem *regs;
 284         u32 num_rings;
 285         struct flexrm_ring *rings;
 286         struct dma_pool *bd_pool;
 287         struct dma_pool *cmpl_pool;
 288         struct dentry *root;
 289         struct mbox_controller controller;
 290 };
 291
 292 /* ====== FlexRM ring descriptor helper routines ===== */
 293
 294 static u64 flexrm_read_desc(void *desc_ptr)
 295 {
 296         return le64_to_cpu(*((u64 *)desc_ptr));
 297 }
 298
 299 static void flexrm_write_desc(void *desc_ptr, u64 desc)
 300 {
 301         *((u64 *)desc_ptr) = cpu_to_le64(desc);
 302 }
 303
 304 static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
 305 {
 306         return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
 307 }
 308
 309 static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
 310 {
 311         u32 status;
 312
 313         status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
 314                           CMPL_DME_STATUS_MASK);
 315         if (status & DME_STATUS_ERROR_MASK)
 316                 return -EIO;
 317
 318         status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
 319                           CMPL_RM_STATUS_MASK);
 320         status &= RM_STATUS_CODE_MASK;
 321         if (status == RM_STATUS_CODE_AE_TIMEOUT)
 322                 return -ETIMEDOUT;
 323
 324         return 0;
 325 }
 326
 327 static bool flexrm_is_next_table_desc(void *desc_ptr)
 328 {
 329         u64 desc = flexrm_read_desc(desc_ptr);
 330         u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 331
 332         return (type == NPTR_TYPE) ? true : false;
 333 }
 334
 335 static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
 336 {
 337         u64 desc = 0;
 338
 339         DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 340         DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
 341         DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
 342
 343         return desc;
 344 }
 345
 346 static u64 flexrm_null_desc(u32 toggle)
 347 {
 348         u64 desc = 0;
 349
 350         DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 351         DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
 352
 353         return desc;
 354 }
 355
 356 static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
 357 {
 358         u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
 359
 360         if (!(nhcnt % HEADER_BDCOUNT_MAX))
 361                 hcnt += 1;
 362
 363         return hcnt;
 364 }
 365
 366 static void flexrm_flip_header_toggle(void *desc_ptr)
 367 {
 368         u64 desc = flexrm_read_desc(desc_ptr);
 369
 370         if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
 371                 desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
 372         else
 373                 desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
 374
 375         flexrm_write_desc(desc_ptr, desc);
 376 }
 377
 378 static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
 379                                u32 bdcount, u32 flags, u32 opaque)
 380 {
 381         u64 desc = 0;
 382
 383         DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 384         DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
 385         DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
 386         DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
 387         DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
 388         DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
 389         DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
 390
 391         return desc;
 392 }
 393
 394 static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
 395                                  u64 desc, void **desc_ptr, u32 *toggle,
 396                                  void *start_desc, void *end_desc)
 397 {
 398         u64 d;
 399         u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
 400
 401         /* Sanity check */
 402         if (nhcnt <= nhpos)
 403                 return;
 404
 405         /*
 406          * Each request or packet start with a HEADER descriptor followed
 407          * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
 408          * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
 409          * following a HEADER descriptor is represented by BDCOUNT field
 410          * of HEADER descriptor. The max value of BDCOUNT field is 31 which
 411          * means we can only have 31 non-HEADER descriptors following one
 412          * HEADER descriptor.
 413          *
 414          * In general use, number of non-HEADER descriptors can easily go
 415          * beyond 31. To tackle this situation, we have packet (or request)
 416          * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
 417          *
 418          * To use packet extension, the first HEADER descriptor of request
 419          * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
 420          * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
 421          * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
 422          * TOGGLE bit of the first HEADER will be set to invalid state to
 423          * ensure that FlexRM does not start fetching descriptors till all
 424          * descriptors are enqueued. The user of this function will flip
 425          * the TOGGLE bit of first HEADER after all descriptors are
 426          * enqueued.
 427          */
 428
 429         if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
 430                 /* Prepare the header descriptor */
 431                 nhavail = (nhcnt - nhpos);
 432                 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
 433                 _startpkt = (nhpos == 0) ? 0x1 : 0x0;
 434                 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
 435                 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
 436                                 nhavail : HEADER_BDCOUNT_MAX;
 437                 if (nhavail <= HEADER_BDCOUNT_MAX)
 438                         _bdcount = nhavail;
 439                 else
 440                         _bdcount = HEADER_BDCOUNT_MAX;
 441                 d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
 442                                         _bdcount, 0x0, reqid);
 443
 444                 /* Write header descriptor */
 445                 flexrm_write_desc(*desc_ptr, d);
 446
 447                 /* Point to next descriptor */
 448                 *desc_ptr += sizeof(desc);
 449                 if (*desc_ptr == end_desc)
 450                         *desc_ptr = start_desc;
 451
 452                 /* Skip next pointer descriptors */
 453                 while (flexrm_is_next_table_desc(*desc_ptr)) {
 454                         *toggle = (*toggle) ? 0 : 1;
 455                         *desc_ptr += sizeof(desc);
 456                         if (*desc_ptr == end_desc)
 457                                 *desc_ptr = start_desc;
 458                 }
 459         }
 460
 461         /* Write desired descriptor */
 462         flexrm_write_desc(*desc_ptr, desc);
 463
 464         /* Point to next descriptor */
 465         *desc_ptr += sizeof(desc);
 466         if (*desc_ptr == end_desc)
 467                 *desc_ptr = start_desc;
 468
 469         /* Skip next pointer descriptors */
 470         while (flexrm_is_next_table_desc(*desc_ptr)) {
 471                 *toggle = (*toggle) ? 0 : 1;
 472                 *desc_ptr += sizeof(desc);
 473                 if (*desc_ptr == end_desc)
 474                         *desc_ptr = start_desc;
 475         }
 476 }
 477
 478 static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
 479 {
 480         u64 desc = 0;
 481
 482         DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 483         DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
 484         DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
 485
 486         return desc;
 487 }
 488
 489 static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
 490 {
 491         u64 desc = 0;
 492
 493         DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 494         DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
 495         DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
 496
 497         return desc;
 498 }
 499
 500 static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
 501 {
 502         u64 desc = 0;
 503
 504         DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 505         DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
 506         DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
 507
 508         return desc;
 509 }
 510
 511 static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
 512 {
 513         u64 desc = 0;
 514
 515         DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 516         DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
 517         DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
 518
 519         return desc;
 520 }
 521
 522 static u64 flexrm_imm_desc(u64 data)
 523 {
 524         u64 desc = 0;
 525
 526         DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 527         DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
 528
 529         return desc;
 530 }
 531
 532 static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
 533 {
 534         u64 desc = 0;
 535
 536         DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 537         DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
 538         DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
 539
 540         return desc;
 541 }
 542
 543 static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
 544 {
 545         u64 desc = 0;
 546
 547         DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 548         DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
 549         DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
 550
 551         return desc;
 552 }
 553
 554 static u64 flexrm_immt_desc(u64 data)
 555 {
 556         u64 desc = 0;
 557
 558         DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 559         DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
 560
 561         return desc;
 562 }
 563
 564 static bool flexrm_spu_sanity_check(struct brcm_message *msg)
 565 {
 566         struct scatterlist *sg;
 567
 568         if (!msg->spu.src || !msg->spu.dst)
 569                 return false;
 570         for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
 571                 if (sg->length & 0xf) {
 572                         if (sg->length > SRC_LENGTH_MASK)
 573                                 return false;
 574                 } else {
 575                         if (sg->length > (MSRC_LENGTH_MASK * 16))
 576                                 return false;
 577                 }
 578         }
 579         for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
 580                 if (sg->length & 0xf) {
 581                         if (sg->length > DST_LENGTH_MASK)
 582                                 return false;
 583                 } else {
 584                         if (sg->length > (MDST_LENGTH_MASK * 16))
 585                                 return false;
 586                 }
 587         }
 588
 589         return true;
 590 }
 591
 592 static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
 593 {
 594         u32 cnt = 0;
 595         unsigned int dst_target = 0;
 596         struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
 597
 598         while (src_sg || dst_sg) {
 599                 if (src_sg) {
 600                         cnt++;
 601                         dst_target = src_sg->length;
 602                         src_sg = sg_next(src_sg);
 603                 } else
 604                         dst_target = UINT_MAX;
 605
 606                 while (dst_target && dst_sg) {
 607                         cnt++;
 608                         if (dst_sg->length < dst_target)
 609                                 dst_target -= dst_sg->length;
 610                         else
 611                                 dst_target = 0;
 612                         dst_sg = sg_next(dst_sg);
 613                 }
 614         }
 615
 616         return cnt;
 617 }
 618
 619 static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
 620 {
 621         int rc;
 622
 623         rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
 624                         DMA_TO_DEVICE);
 625         if (!rc)
 626                 return -EIO;
 627
 628         rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
 629                         DMA_FROM_DEVICE);
 630         if (!rc) {
 631                 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
 632                              DMA_TO_DEVICE);
 633                 return -EIO;
 634         }
 635
 636         return 0;
 637 }
 638
 639 static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
 640 {
 641         dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
 642                      DMA_FROM_DEVICE);
 643         dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
 644                      DMA_TO_DEVICE);
 645 }
 646
 647 static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
 648                                      u32 reqid, void *desc_ptr, u32 toggle,
 649                                      void *start_desc, void *end_desc)
 650 {
 651         u64 d;
 652         u32 nhpos = 0;
 653         void *orig_desc_ptr = desc_ptr;
 654         unsigned int dst_target = 0;
 655         struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
 656
 657         while (src_sg || dst_sg) {
 658                 if (src_sg) {
 659                         if (sg_dma_len(src_sg) & 0xf)
 660                                 d = flexrm_src_desc(sg_dma_address(src_sg),
 661                                                      sg_dma_len(src_sg));
 662                         else
 663                                 d = flexrm_msrc_desc(sg_dma_address(src_sg),
 664                                                       sg_dma_len(src_sg)/16);
 665                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 666                                              d, &desc_ptr, &toggle,
 667                                              start_desc, end_desc);
 668                         nhpos++;
 669                         dst_target = sg_dma_len(src_sg);
 670                         src_sg = sg_next(src_sg);
 671                 } else
 672                         dst_target = UINT_MAX;
 673
 674                 while (dst_target && dst_sg) {
 675                         if (sg_dma_len(dst_sg) & 0xf)
 676                                 d = flexrm_dst_desc(sg_dma_address(dst_sg),
 677                                                      sg_dma_len(dst_sg));
 678                         else
 679                                 d = flexrm_mdst_desc(sg_dma_address(dst_sg),
 680                                                       sg_dma_len(dst_sg)/16);
 681                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 682                                              d, &desc_ptr, &toggle,
 683                                              start_desc, end_desc);
 684                         nhpos++;
 685                         if (sg_dma_len(dst_sg) < dst_target)
 686                                 dst_target -= sg_dma_len(dst_sg);
 687                         else
 688                                 dst_target = 0;
 689                         dst_sg = sg_next(dst_sg);
 690                 }
 691         }
 692
 693         /* Null descriptor with invalid toggle bit */
 694         flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
 695
 696         /* Ensure that descriptors have been written to memory */
 697         wmb();
 698
 699         /* Flip toggle bit in header */
 700         flexrm_flip_header_toggle(orig_desc_ptr);
 701
 702         return desc_ptr;
 703 }
 704
 705 static bool flexrm_sba_sanity_check(struct brcm_message *msg)
 706 {
 707         u32 i;
 708
 709         if (!msg->sba.cmds || !msg->sba.cmds_count)
 710                 return false;
 711
 712         for (i = 0; i < msg->sba.cmds_count; i++) {
 713                 if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
 714                      (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
 715                     (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
 716                         return false;
 717                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
 718                     (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
 719                         return false;
 720                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
 721                     (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
 722                         return false;
 723                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
 724                     (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
 725                         return false;
 726                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
 727                     (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
 728                         return false;
 729         }
 730
 731         return true;
 732 }
 733
 734 static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
 735 {
 736         u32 i, cnt;
 737
 738         cnt = 0;
 739         for (i = 0; i < msg->sba.cmds_count; i++) {
 740                 cnt++;
 741
 742                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
 743                     (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
 744                         cnt++;
 745
 746                 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
 747                         cnt++;
 748
 749                 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
 750                         cnt++;
 751         }
 752
 753         return cnt;
 754 }
 755
 756 static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
 757                                      u32 reqid, void *desc_ptr, u32 toggle,
 758                                      void *start_desc, void *end_desc)
 759 {
 760         u64 d;
 761         u32 i, nhpos = 0;
 762         struct brcm_sba_command *c;
 763         void *orig_desc_ptr = desc_ptr;
 764
 765         /* Convert SBA commands into descriptors */
 766         for (i = 0; i < msg->sba.cmds_count; i++) {
 767                 c = &msg->sba.cmds[i];
 768
 769                 if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
 770                     (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
 771                         /* Destination response descriptor */
 772                         d = flexrm_dst_desc(c->resp, c->resp_len);
 773                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 774                                              d, &desc_ptr, &toggle,
 775                                              start_desc, end_desc);
 776                         nhpos++;
 777                 } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
 778                         /* Destination response with tlast descriptor */
 779                         d = flexrm_dstt_desc(c->resp, c->resp_len);
 780                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 781                                              d, &desc_ptr, &toggle,
 782                                              start_desc, end_desc);
 783                         nhpos++;
 784                 }
 785
 786                 if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
 787                         /* Destination with tlast descriptor */
 788                         d = flexrm_dstt_desc(c->data, c->data_len);
 789                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 790                                              d, &desc_ptr, &toggle,
 791                                              start_desc, end_desc);
 792                         nhpos++;
 793                 }
 794
 795                 if (c->flags & BRCM_SBA_CMD_TYPE_B) {
 796                         /* Command as immediate descriptor */
 797                         d = flexrm_imm_desc(c->cmd);
 798                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 799                                              d, &desc_ptr, &toggle,
 800                                              start_desc, end_desc);
 801                         nhpos++;
 802                 } else {
 803                         /* Command as immediate descriptor with tlast */
 804                         d = flexrm_immt_desc(c->cmd);
 805                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 806                                              d, &desc_ptr, &toggle,
 807                                              start_desc, end_desc);
 808                         nhpos++;
 809                 }
 810
 811                 if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
 812                     (c->flags & BRCM_SBA_CMD_TYPE_C)) {
 813                         /* Source with tlast descriptor */
 814                         d = flexrm_srct_desc(c->data, c->data_len);
 815                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 816                                              d, &desc_ptr, &toggle,
 817                                              start_desc, end_desc);
 818                         nhpos++;
 819                 }
 820         }
 821
 822         /* Null descriptor with invalid toggle bit */
 823         flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
 824
 825         /* Ensure that descriptors have been written to memory */
 826         wmb();
 827
 828         /* Flip toggle bit in header */
 829         flexrm_flip_header_toggle(orig_desc_ptr);
 830
 831         return desc_ptr;
 832 }
 833
 834 static bool flexrm_sanity_check(struct brcm_message *msg)
 835 {
 836         if (!msg)
 837                 return false;
 838
 839         switch (msg->type) {
 840         case BRCM_MESSAGE_SPU:
 841                 return flexrm_spu_sanity_check(msg);
 842         case BRCM_MESSAGE_SBA:
 843                 return flexrm_sba_sanity_check(msg);
 844         default:
 845                 return false;
 846         };
 847 }
 848
 849 static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
 850 {
 851         if (!msg)
 852                 return 0;
 853
 854         switch (msg->type) {
 855         case BRCM_MESSAGE_SPU:
 856                 return flexrm_spu_estimate_nonheader_desc_count(msg);
 857         case BRCM_MESSAGE_SBA:
 858                 return flexrm_sba_estimate_nonheader_desc_count(msg);
 859         default:
 860                 return 0;
 861         };
 862 }
 863
 864 static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
 865 {
 866         if (!dev || !msg)
 867                 return -EINVAL;
 868
 869         switch (msg->type) {
 870         case BRCM_MESSAGE_SPU:
 871                 return flexrm_spu_dma_map(dev, msg);
 872         default:
 873                 break;
 874         }
 875
 876         return 0;
 877 }
 878
 879 static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
 880 {
 881         if (!dev || !msg)
 882                 return;
 883
 884         switch (msg->type) {
 885         case BRCM_MESSAGE_SPU:
 886                 flexrm_spu_dma_unmap(dev, msg);
 887                 break;
 888         default:
 889                 break;
 890         }
 891 }
 892
 893 static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
 894                                 u32 reqid, void *desc_ptr, u32 toggle,
 895                                 void *start_desc, void *end_desc)
 896 {
 897         if (!msg || !desc_ptr || !start_desc || !end_desc)
 898                 return ERR_PTR(-ENOTSUPP);
 899
 900         if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
 901                 return ERR_PTR(-ERANGE);
 902
 903         switch (msg->type) {
 904         case BRCM_MESSAGE_SPU:
 905                 return flexrm_spu_write_descs(msg, nhcnt, reqid,
 906                                                desc_ptr, toggle,
 907                                                start_desc, end_desc);
 908         case BRCM_MESSAGE_SBA:
 909                 return flexrm_sba_write_descs(msg, nhcnt, reqid,
 910                                                desc_ptr, toggle,
 911                                                start_desc, end_desc);
 912         default:
 913                 return ERR_PTR(-ENOTSUPP);
 914         };
 915 }
 916
 917 /* ====== FlexRM driver helper routines ===== */
 918
 919 static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
 920                                            struct seq_file *file)
 921 {
 922         int i;
 923         const char *state;
 924         struct flexrm_ring *ring;
 925
 926         seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
 927                    "Ring#", "State", "BD_Addr", "BD_Size",
 928                    "Cmpl_Addr", "Cmpl_Size");
 929
 930         for (i = 0; i < mbox->num_rings; i++) {
 931                 ring = &mbox->rings[i];
 932                 if (readl(ring->regs + RING_CONTROL) &
 933                     BIT(CONTROL_ACTIVE_SHIFT))
 934                         state = "active";
 935                 else
 936                         state = "inactive";
 937                 seq_printf(file,
 938                            "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
 939                            ring->num, state,
 940                            (unsigned long long)ring->bd_dma_base,
 941                            (u32)RING_BD_SIZE,
 942                            (unsigned long long)ring->cmpl_dma_base,
 943                            (u32)RING_CMPL_SIZE);
 944         }
 945 }
 946
 947 static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
 948                                           struct seq_file *file)
 949 {
 950         int i;
 951         u32 val, bd_read_offset;
 952         struct flexrm_ring *ring;
 953
 954         seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
 955                    "Ring#", "BD_Read", "BD_Write",
 956                    "Cmpl_Read", "Submitted", "Completed");
 957
 958         for (i = 0; i < mbox->num_rings; i++) {
 959                 ring = &mbox->rings[i];
 960                 bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
 961                 val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
 962                 bd_read_offset *= RING_DESC_SIZE;
 963                 bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
 964                                         ring->bd_dma_base);
 965                 seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
 966                            ring->num,
 967                            (u32)bd_read_offset,
 968                            (u32)ring->bd_write_offset,
 969                            (u32)ring->cmpl_read_offset,
 970                            (u32)atomic_read(&ring->msg_send_count),
 971                            (u32)atomic_read(&ring->msg_cmpl_count));
 972         }
 973 }
 974
 975 static int flexrm_new_request(struct flexrm_ring *ring,
 976                                 struct brcm_message *batch_msg,
 977                                 struct brcm_message *msg)
 978 {
 979         void *next;
 980         unsigned long flags;
 981         u32 val, count, nhcnt;
 982         u32 read_offset, write_offset;
 983         bool exit_cleanup = false;
 984         int ret = 0, reqid;
 985
 986         /* Do sanity check on message */
 987         if (!flexrm_sanity_check(msg))
 988                 return -EIO;
 989         msg->error = 0;
 990
 991         /* If no requests possible then save data pointer and goto done. */
 992         spin_lock_irqsave(&ring->lock, flags);
 993         reqid = bitmap_find_free_region(ring->requests_bmap,
 994                                         RING_MAX_REQ_COUNT, 0);
 995         spin_unlock_irqrestore(&ring->lock, flags);
 996         if (reqid < 0)
 997                 return -ENOSPC;
 998         ring->requests[reqid] = msg;
 999
1000         /* Do DMA mappings for the message */
1001         ret = flexrm_dma_map(ring->mbox->dev, msg);
1002         if (ret < 0) {
1003                 ring->requests[reqid] = NULL;
1004                 spin_lock_irqsave(&ring->lock, flags);
1005                 bitmap_release_region(ring->requests_bmap, reqid, 0);
1006                 spin_unlock_irqrestore(&ring->lock, flags);
1007                 return ret;
1008         }
1009
1010         /* Determine current HW BD read offset */
1011         read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
1012         val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
1013         read_offset *= RING_DESC_SIZE;
1014         read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
1015
1016         /*
1017          * Number required descriptors = number of non-header descriptors +
1018          *                               number of header descriptors +
1019          *                               1x null descriptor
1020          */
1021         nhcnt = flexrm_estimate_nonheader_desc_count(msg);
1022         count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
1023
1024         /* Check for available descriptor space. */
1025         write_offset = ring->bd_write_offset;
1026         while (count) {
1027                 if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
1028                         count--;
1029                 write_offset += RING_DESC_SIZE;
1030                 if (write_offset == RING_BD_SIZE)
1031                         write_offset = 0x0;
1032                 if (write_offset == read_offset)
1033                         break;
1034         }
1035         if (count) {
1036                 ret = -ENOSPC;
1037                 exit_cleanup = true;
1038                 goto exit;
1039         }
1040
1041         /* Write descriptors to ring */
1042         next = flexrm_write_descs(msg, nhcnt, reqid,
1043                         ring->bd_base + ring->bd_write_offset,
1044                         RING_BD_TOGGLE_VALID(ring->bd_write_offset),
1045                         ring->bd_base, ring->bd_base + RING_BD_SIZE);
1046         if (IS_ERR(next)) {
1047                 ret = PTR_ERR(next);
1048                 exit_cleanup = true;
1049                 goto exit;
1050         }
1051
1052         /* Save ring BD write offset */
1053         ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
1054
1055         /* Increment number of messages sent */
1056         atomic_inc_return(&ring->msg_send_count);
1057
1058 exit:
1059         /* Update error status in message */
1060         msg->error = ret;
1061
1062         /* Cleanup if we failed */
1063         if (exit_cleanup) {
1064                 flexrm_dma_unmap(ring->mbox->dev, msg);
1065                 ring->requests[reqid] = NULL;
1066                 spin_lock_irqsave(&ring->lock, flags);
1067                 bitmap_release_region(ring->requests_bmap, reqid, 0);
1068                 spin_unlock_irqrestore(&ring->lock, flags);
1069         }
1070
1071         return ret;
1072 }
1073
1074 static int flexrm_process_completions(struct flexrm_ring *ring)
1075 {
1076         u64 desc;
1077         int err, count = 0;
1078         unsigned long flags;
1079         struct brcm_message *msg = NULL;
1080         u32 reqid, cmpl_read_offset, cmpl_write_offset;
1081         struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
1082
1083         spin_lock_irqsave(&ring->lock, flags);
1084
1085         /*
1086          * Get current completion read and write offset
1087          *
1088          * Note: We should read completion write pointer at least once
1089          * after we get a MSI interrupt because HW maintains internal
1090          * MSI status which will allow next MSI interrupt only after
1091          * completion write pointer is read.
1092          */
1093         cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1094         cmpl_write_offset *= RING_DESC_SIZE;
1095         cmpl_read_offset = ring->cmpl_read_offset;
1096         ring->cmpl_read_offset = cmpl_write_offset;
1097
1098         spin_unlock_irqrestore(&ring->lock, flags);
1099
1100         /* For each completed request notify mailbox clients */
1101         reqid = 0;
1102         while (cmpl_read_offset != cmpl_write_offset) {
1103                 /* Dequeue next completion descriptor */
1104                 desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
1105
1106                 /* Next read offset */
1107                 cmpl_read_offset += RING_DESC_SIZE;
1108                 if (cmpl_read_offset == RING_CMPL_SIZE)
1109                         cmpl_read_offset = 0;
1110
1111                 /* Decode error from completion descriptor */
1112                 err = flexrm_cmpl_desc_to_error(desc);
1113                 if (err < 0) {
1114                         dev_warn(ring->mbox->dev,
1115                         "ring%d got completion desc=0x%lx with error %d\n",
1116                         ring->num, (unsigned long)desc, err);
1117                 }
1118
1119                 /* Determine request id from completion descriptor */
1120                 reqid = flexrm_cmpl_desc_to_reqid(desc);
1121
1122                 /* Determine message pointer based on reqid */
1123                 msg = ring->requests[reqid];
1124                 if (!msg) {
1125                         dev_warn(ring->mbox->dev,
1126                         "ring%d null msg pointer for completion desc=0x%lx\n",
1127                         ring->num, (unsigned long)desc);
1128                         continue;
1129                 }
1130
1131                 /* Release reqid for recycling */
1132                 ring->requests[reqid] = NULL;
1133                 spin_lock_irqsave(&ring->lock, flags);
1134                 bitmap_release_region(ring->requests_bmap, reqid, 0);
1135                 spin_unlock_irqrestore(&ring->lock, flags);
1136
1137                 /* Unmap DMA mappings */
1138                 flexrm_dma_unmap(ring->mbox->dev, msg);
1139
1140                 /* Give-back message to mailbox client */
1141                 msg->error = err;
1142                 mbox_chan_received_data(chan, msg);
1143
1144                 /* Increment number of completions processed */
1145                 atomic_inc_return(&ring->msg_cmpl_count);
1146                 count++;
1147         }
1148
1149         return count;
1150 }
1151
1152 /* ====== FlexRM Debugfs callbacks ====== */
1153
1154 static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
1155 {
1156         struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1157
1158         /* Write config in file */
1159         flexrm_write_config_in_seqfile(mbox, file);
1160
1161         return 0;
1162 }
1163
1164 static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
1165 {
1166         struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1167
1168         /* Write stats in file */
1169         flexrm_write_stats_in_seqfile(mbox, file);
1170
1171         return 0;
1172 }
1173
1174 /* ====== FlexRM interrupt handler ===== */
1175
1176 static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
1177 {
1178         /* We only have MSI for completions so just wakeup IRQ thread */
1179         /* Ring related errors will be informed via completion descriptors */
1180
1181         return IRQ_WAKE_THREAD;
1182 }
1183
1184 static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
1185 {
1186         flexrm_process_completions(dev_id);
1187
1188         return IRQ_HANDLED;
1189 }
1190
1191 /* ====== FlexRM mailbox callbacks ===== */
1192
1193 static int flexrm_send_data(struct mbox_chan *chan, void *data)
1194 {
1195         int i, rc;
1196         struct flexrm_ring *ring = chan->con_priv;
1197         struct brcm_message *msg = data;
1198
1199         if (msg->type == BRCM_MESSAGE_BATCH) {
1200                 for (i = msg->batch.msgs_queued;
1201                      i < msg->batch.msgs_count; i++) {
1202                         rc = flexrm_new_request(ring, msg,
1203                                                  &msg->batch.msgs[i]);
1204                         if (rc) {
1205                                 msg->error = rc;
1206                                 return rc;
1207                         }
1208                         msg->batch.msgs_queued++;
1209                 }
1210                 return 0;
1211         }
1212
1213         return flexrm_new_request(ring, NULL, data);
1214 }
1215
1216 static bool flexrm_peek_data(struct mbox_chan *chan)
1217 {
1218         int cnt = flexrm_process_completions(chan->con_priv);
1219
1220         return (cnt > 0) ? true : false;
1221 }
1222
1223 static int flexrm_startup(struct mbox_chan *chan)
1224 {
1225         u64 d;
1226         u32 val, off;
1227         int ret = 0;
1228         dma_addr_t next_addr;
1229         struct flexrm_ring *ring = chan->con_priv;
1230
1231         /* Allocate BD memory */
1232         ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
1233                                        GFP_KERNEL, &ring->bd_dma_base);
1234         if (!ring->bd_base) {
1235                 dev_err(ring->mbox->dev,
1236                         "can't allocate BD memory for ring%d\n",
1237                         ring->num);
1238                 ret = -ENOMEM;
1239                 goto fail;
1240         }
1241
1242         /* Configure next table pointer entries in BD memory */
1243         for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
1244                 next_addr = off + RING_DESC_SIZE;
1245                 if (next_addr == RING_BD_SIZE)
1246                         next_addr = 0;
1247                 next_addr += ring->bd_dma_base;
1248                 if (RING_BD_ALIGN_CHECK(next_addr))
1249                         d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
1250                                                     next_addr);
1251                 else
1252                         d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
1253                 flexrm_write_desc(ring->bd_base + off, d);
1254         }
1255
1256         /* Allocate completion memory */
1257         ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,
1258                                          GFP_KERNEL, &ring->cmpl_dma_base);
1259         if (!ring->cmpl_base) {
1260                 dev_err(ring->mbox->dev,
1261                         "can't allocate completion memory for ring%d\n",
1262                         ring->num);
1263                 ret = -ENOMEM;
1264                 goto fail_free_bd_memory;
1265         }
1266
1267         /* Request IRQ */
1268         if (ring->irq == UINT_MAX) {
1269                 dev_err(ring->mbox->dev,
1270                         "ring%d IRQ not available\n", ring->num);
1271                 ret = -ENODEV;
1272                 goto fail_free_cmpl_memory;
1273         }
1274         ret = request_threaded_irq(ring->irq,
1275                                    flexrm_irq_event,
1276                                    flexrm_irq_thread,
1277                                    0, dev_name(ring->mbox->dev), ring);
1278         if (ret) {
1279                 dev_err(ring->mbox->dev,
1280                         "failed to request ring%d IRQ\n", ring->num);
1281                 goto fail_free_cmpl_memory;
1282         }
1283         ring->irq_requested = true;
1284
1285         /* Set IRQ affinity hint */
1286         ring->irq_aff_hint = CPU_MASK_NONE;
1287         val = ring->mbox->num_rings;
1288         val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
1289         cpumask_set_cpu((ring->num / val) % num_online_cpus(),
1290                         &ring->irq_aff_hint);
1291         ret = irq_update_affinity_hint(ring->irq, &ring->irq_aff_hint);
1292         if (ret) {
1293                 dev_err(ring->mbox->dev,
1294                         "failed to set IRQ affinity hint for ring%d\n",
1295                         ring->num);
1296                 goto fail_free_irq;
1297         }
1298
1299         /* Disable/inactivate ring */
1300         writel_relaxed(0x0, ring->regs + RING_CONTROL);
1301
1302         /* Program BD start address */
1303         val = BD_START_ADDR_VALUE(ring->bd_dma_base);
1304         writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
1305
1306         /* BD write pointer will be same as HW write pointer */
1307         ring->bd_write_offset =
1308                         readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
1309         ring->bd_write_offset *= RING_DESC_SIZE;
1310
1311         /* Program completion start address */
1312         val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
1313         writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
1314
1315         /* Completion read pointer will be same as HW write pointer */
1316         ring->cmpl_read_offset =
1317                         readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1318         ring->cmpl_read_offset *= RING_DESC_SIZE;
1319
1320         /* Read ring Tx, Rx, and Outstanding counts to clear */
1321         readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
1322         readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
1323         readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
1324         readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
1325         readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
1326
1327         /* Configure RING_MSI_CONTROL */
1328         val = 0;
1329         val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
1330         val |= BIT(MSI_ENABLE_SHIFT);
1331         val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
1332         writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
1333
1334         /* Enable/activate ring */
1335         val = BIT(CONTROL_ACTIVE_SHIFT);
1336         writel_relaxed(val, ring->regs + RING_CONTROL);
1337
1338         /* Reset stats to zero */
1339         atomic_set(&ring->msg_send_count, 0);
1340         atomic_set(&ring->msg_cmpl_count, 0);
1341
1342         return 0;
1343
1344 fail_free_irq:
1345         free_irq(ring->irq, ring);
1346         ring->irq_requested = false;
1347 fail_free_cmpl_memory:
1348         dma_pool_free(ring->mbox->cmpl_pool,
1349                       ring->cmpl_base, ring->cmpl_dma_base);
1350         ring->cmpl_base = NULL;
1351 fail_free_bd_memory:
1352         dma_pool_free(ring->mbox->bd_pool,
1353                       ring->bd_base, ring->bd_dma_base);
1354         ring->bd_base = NULL;
1355 fail:
1356         return ret;
1357 }
1358
1359 static void flexrm_shutdown(struct mbox_chan *chan)
1360 {
1361         u32 reqid;
1362         unsigned int timeout;
1363         struct brcm_message *msg;
1364         struct flexrm_ring *ring = chan->con_priv;
1365
1366         /* Disable/inactivate ring */
1367         writel_relaxed(0x0, ring->regs + RING_CONTROL);
1368
1369         /* Set ring flush state */
1370         timeout = 1000; /* timeout of 1s */
1371         writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
1372                         ring->regs + RING_CONTROL);
1373         do {
1374                 if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1375                     FLUSH_DONE_MASK)
1376                         break;
1377                 mdelay(1);
1378         } while (--timeout);
1379         if (!timeout)
1380                 dev_err(ring->mbox->dev,
1381                         "setting ring%d flush state timedout\n", ring->num);
1382
1383         /* Clear ring flush state */
1384         timeout = 1000; /* timeout of 1s */
1385         writel_relaxed(0x0, ring->regs + RING_CONTROL);
1386         do {
1387                 if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1388                       FLUSH_DONE_MASK))
1389                         break;
1390                 mdelay(1);
1391         } while (--timeout);
1392         if (!timeout)
1393                 dev_err(ring->mbox->dev,
1394                         "clearing ring%d flush state timedout\n", ring->num);
1395
1396         /* Abort all in-flight requests */
1397         for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
1398                 msg = ring->requests[reqid];
1399                 if (!msg)
1400                         continue;
1401
1402                 /* Release reqid for recycling */
1403                 ring->requests[reqid] = NULL;
1404
1405                 /* Unmap DMA mappings */
1406                 flexrm_dma_unmap(ring->mbox->dev, msg);
1407
1408                 /* Give-back message to mailbox client */
1409                 msg->error = -EIO;
1410                 mbox_chan_received_data(chan, msg);
1411         }
1412
1413         /* Clear requests bitmap */
1414         bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1415
1416         /* Release IRQ */
1417         if (ring->irq_requested) {
1418                 irq_update_affinity_hint(ring->irq, NULL);
1419                 free_irq(ring->irq, ring);
1420                 ring->irq_requested = false;
1421         }
1422
1423         /* Free-up completion descriptor ring */
1424         if (ring->cmpl_base) {
1425                 dma_pool_free(ring->mbox->cmpl_pool,
1426                               ring->cmpl_base, ring->cmpl_dma_base);
1427                 ring->cmpl_base = NULL;
1428         }
1429
1430         /* Free-up BD descriptor ring */
1431         if (ring->bd_base) {
1432                 dma_pool_free(ring->mbox->bd_pool,
1433                               ring->bd_base, ring->bd_dma_base);
1434                 ring->bd_base = NULL;
1435         }
1436 }
1437
1438 static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
1439         .send_data      = flexrm_send_data,
1440         .startup        = flexrm_startup,
1441         .shutdown       = flexrm_shutdown,
1442         .peek_data      = flexrm_peek_data,
1443 };
1444
1445 static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
1446                                         const struct of_phandle_args *pa)
1447 {
1448         struct mbox_chan *chan;
1449         struct flexrm_ring *ring;
1450
1451         if (pa->args_count < 3)
1452                 return ERR_PTR(-EINVAL);
1453
1454         if (pa->args[0] >= cntlr->num_chans)
1455                 return ERR_PTR(-ENOENT);
1456
1457         if (pa->args[1] > MSI_COUNT_MASK)
1458                 return ERR_PTR(-EINVAL);
1459
1460         if (pa->args[2] > MSI_TIMER_VAL_MASK)
1461                 return ERR_PTR(-EINVAL);
1462
1463         chan = &cntlr->chans[pa->args[0]];
1464         ring = chan->con_priv;
1465         ring->msi_count_threshold = pa->args[1];
1466         ring->msi_timer_val = pa->args[2];
1467
1468         return chan;
1469 }
1470
1471 /* ====== FlexRM platform driver ===== */
1472
1473 static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
1474 {
1475         struct device *dev = msi_desc_to_dev(desc);
1476         struct flexrm_mbox *mbox = dev_get_drvdata(dev);
1477         struct flexrm_ring *ring = &mbox->rings[desc->msi_index];
1478
1479         /* Configure per-Ring MSI registers */
1480         writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
1481         writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
1482         writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
1483 }
1484
1485 static int flexrm_mbox_probe(struct platform_device *pdev)
1486 {
1487         int index, ret = 0;
1488         void __iomem *regs;
1489         void __iomem *regs_end;
1490         struct resource *iomem;
1491         struct flexrm_ring *ring;
1492         struct flexrm_mbox *mbox;
1493         struct device *dev = &pdev->dev;
1494
1495         /* Allocate driver mailbox struct */
1496         mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
1497         if (!mbox) {
1498                 ret = -ENOMEM;
1499                 goto fail;
1500         }
1501         mbox->dev = dev;
1502         platform_set_drvdata(pdev, mbox);
1503
1504         /* Get resource for registers */
1505         iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1506         if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
1507                 ret = -ENODEV;
1508                 goto fail;
1509         }
1510
1511         /* Map registers of all rings */
1512         mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
1513         if (IS_ERR(mbox->regs)) {
1514                 ret = PTR_ERR(mbox->regs);
1515                 goto fail;
1516         }
1517         regs_end = mbox->regs + resource_size(iomem);
1518
1519         /* Scan and count available rings */
1520         mbox->num_rings = 0;
1521         for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
1522                 if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
1523                         mbox->num_rings++;
1524         }
1525         if (!mbox->num_rings) {
1526                 ret = -ENODEV;
1527                 goto fail;
1528         }
1529
1530         /* Allocate driver ring structs */
1531         ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
1532         if (!ring) {
1533                 ret = -ENOMEM;
1534                 goto fail;
1535         }
1536         mbox->rings = ring;
1537
1538         /* Initialize members of driver ring structs */
1539         regs = mbox->regs;
1540         for (index = 0; index < mbox->num_rings; index++) {
1541                 ring = &mbox->rings[index];
1542                 ring->num = index;
1543                 ring->mbox = mbox;
1544                 while ((regs < regs_end) &&
1545                        (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
1546                         regs += RING_REGS_SIZE;
1547                 if (regs_end <= regs) {
1548                         ret = -ENODEV;
1549                         goto fail;
1550                 }
1551                 ring->regs = regs;
1552                 regs += RING_REGS_SIZE;
1553                 ring->irq = UINT_MAX;
1554                 ring->irq_requested = false;
1555                 ring->msi_timer_val = MSI_TIMER_VAL_MASK;
1556                 ring->msi_count_threshold = 0x1;
1557                 memset(ring->requests, 0, sizeof(ring->requests));
1558                 ring->bd_base = NULL;
1559                 ring->bd_dma_base = 0;
1560                 ring->cmpl_base = NULL;
1561                 ring->cmpl_dma_base = 0;
1562                 atomic_set(&ring->msg_send_count, 0);
1563                 atomic_set(&ring->msg_cmpl_count, 0);
1564                 spin_lock_init(&ring->lock);
1565                 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1566                 ring->cmpl_read_offset = 0;
1567         }
1568
1569         /* FlexRM is capable of 40-bit physical addresses only */
1570         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
1571         if (ret) {
1572                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1573                 if (ret)
1574                         goto fail;
1575         }
1576
1577         /* Create DMA pool for ring BD memory */
1578         mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
1579                                         1 << RING_BD_ALIGN_ORDER, 0);
1580         if (!mbox->bd_pool) {
1581                 ret = -ENOMEM;
1582                 goto fail;
1583         }
1584
1585         /* Create DMA pool for ring completion memory */
1586         mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
1587                                           1 << RING_CMPL_ALIGN_ORDER, 0);
1588         if (!mbox->cmpl_pool) {
1589                 ret = -ENOMEM;
1590                 goto fail_destroy_bd_pool;
1591         }
1592
1593         /* Allocate platform MSIs for each ring */
1594         ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
1595                                                 flexrm_mbox_msi_write);
1596         if (ret)
1597                 goto fail_destroy_cmpl_pool;
1598
1599         /* Save alloced IRQ numbers for each ring */
1600         for (index = 0; index < mbox->num_rings; index++)
1601                 mbox->rings[index].irq = msi_get_virq(dev, index);
1602
1603         /* Check availability of debugfs */
1604         if (!debugfs_initialized())
1605                 goto skip_debugfs;
1606
1607         /* Create debugfs root entry */
1608         mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
1609
1610         /* Create debugfs config entry */
1611         debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root,
1612                                     flexrm_debugfs_conf_show);
1613
1614         /* Create debugfs stats entry */
1615         debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root,
1616                                     flexrm_debugfs_stats_show);
1617
1618 skip_debugfs:
1619
1620         /* Initialize mailbox controller */
1621         mbox->controller.txdone_irq = false;
1622         mbox->controller.txdone_poll = false;
1623         mbox->controller.ops = &flexrm_mbox_chan_ops;
1624         mbox->controller.dev = dev;
1625         mbox->controller.num_chans = mbox->num_rings;
1626         mbox->controller.of_xlate = flexrm_mbox_of_xlate;
1627         mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
1628                                 sizeof(*mbox->controller.chans), GFP_KERNEL);
1629         if (!mbox->controller.chans) {
1630                 ret = -ENOMEM;
1631                 goto fail_free_debugfs_root;
1632         }
1633         for (index = 0; index < mbox->num_rings; index++)
1634                 mbox->controller.chans[index].con_priv = &mbox->rings[index];
1635
1636         /* Register mailbox controller */
1637         ret = devm_mbox_controller_register(dev, &mbox->controller);
1638         if (ret)
1639                 goto fail_free_debugfs_root;
1640
1641         dev_info(dev, "registered flexrm mailbox with %d channels\n",
1642                         mbox->controller.num_chans);
1643
1644         return 0;
1645
1646 fail_free_debugfs_root:
1647         debugfs_remove_recursive(mbox->root);
1648         platform_msi_domain_free_irqs(dev);
1649 fail_destroy_cmpl_pool:
1650         dma_pool_destroy(mbox->cmpl_pool);
1651 fail_destroy_bd_pool:
1652         dma_pool_destroy(mbox->bd_pool);
1653 fail:
1654         return ret;
1655 }
1656
1657 static int flexrm_mbox_remove(struct platform_device *pdev)
1658 {
1659         struct device *dev = &pdev->dev;
1660         struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
1661
1662         debugfs_remove_recursive(mbox->root);
1663
1664         platform_msi_domain_free_irqs(dev);
1665
1666         dma_pool_destroy(mbox->cmpl_pool);
1667         dma_pool_destroy(mbox->bd_pool);
1668
1669         return 0;
1670 }
1671
1672 static const struct of_device_id flexrm_mbox_of_match[] = {
1673         { .compatible = "brcm,iproc-flexrm-mbox", },
1674         {},
1675 };
1676 MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
1677
1678 static struct platform_driver flexrm_mbox_driver = {
1679         .driver = {
1680                 .name = "brcm-flexrm-mbox",
1681                 .of_match_table = flexrm_mbox_of_match,
1682         },
1683         .probe          = flexrm_mbox_probe,
1684         .remove         = flexrm_mbox_remove,
1685 };
1686 module_platform_driver(flexrm_mbox_driver);
1687
1688 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1689 MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
1690 MODULE_LICENSE("GPL v2");