drivers/ntb/test/ntb_tool.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   8  *   Copyright (C) 2017 T-Platforms All Rights Reserved.
   9  *
  10  *   This program is free software; you can redistribute it and/or modify
  11  *   it under the terms of version 2 of the GNU General Public License as
  12  *   published by the Free Software Foundation.
  13  *
  14  *   This program is distributed in the hope that it will be useful, but
  15  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  *   General Public License for more details.
  18  *
  19  *   BSD LICENSE
  20  *
  21  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  22  *   Copyright (C) 2017 T-Platforms All Rights Reserved.
  23  *
  24  *   Redistribution and use in source and binary forms, with or without
  25  *   modification, are permitted provided that the following conditions
  26  *   are met:
  27  *
  28  *     * Redistributions of source code must retain the above copyright
  29  *       notice, this list of conditions and the following disclaimer.
  30  *     * Redistributions in binary form must reproduce the above copy
  31  *       notice, this list of conditions and the following disclaimer in
  32  *       the documentation and/or other materials provided with the
  33  *       distribution.
  34  *     * Neither the name of Intel Corporation nor the names of its
  35  *       contributors may be used to endorse or promote products derived
  36  *       from this software without specific prior written permission.
  37  *
  38  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  39  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  40  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  41  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  42  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  43  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  44  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  45  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  46  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  47  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  48  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  49  *
  50  * PCIe NTB Debugging Tool Linux driver
  51  */
  52
  53 /*
  54  * How to use this tool, by example.
  55  *
  56  * Assuming $DBG_DIR is something like:
  57  * '/sys/kernel/debug/ntb_tool/0000:00:03.0'
  58  * Suppose aside from local device there is at least one remote device
  59  * connected to NTB with index 0.
  60  *-----------------------------------------------------------------------------
  61  * Eg: check local/peer device information.
  62  *
  63  * # Get local device port number
  64  * root@self# cat $DBG_DIR/port
  65  *
  66  * # Check local device functionality
  67  * root@self# ls $DBG_DIR
  68  * db            msg1          msg_sts     peer4/        port
  69  * db_event      msg2          peer0/      peer5/        spad0
  70  * db_mask       msg3          peer1/      peer_db       spad1
  71  * link          msg_event     peer2/      peer_db_mask  spad2
  72  * msg0          msg_mask      peer3/      peer_spad     spad3
  73  * # As one can see it supports:
  74  * # 1) four inbound message registers
  75  * # 2) four inbound scratchpads
  76  * # 3) up to six peer devices
  77  *
  78  * # Check peer device port number
  79  * root@self# cat $DBG_DIR/peer0/port
  80  *
  81  * # Check peer device(s) functionality to be used
  82  * root@self# ls $DBG_DIR/peer0
  83  * link             mw_trans0       mw_trans6        port
  84  * link_event       mw_trans1       mw_trans7        spad0
  85  * msg0             mw_trans2       peer_mw_trans0   spad1
  86  * msg1             mw_trans3       peer_mw_trans1   spad2
  87  * msg2             mw_trans4       peer_mw_trans2   spad3
  88  * msg3             mw_trans5       peer_mw_trans3
  89  * # As one can see we got:
  90  * # 1) four outbound message registers
  91  * # 2) four outbound scratchpads
  92  * # 3) eight inbound memory windows
  93  * # 4) four outbound memory windows
  94  *-----------------------------------------------------------------------------
  95  * Eg: NTB link tests
  96  *
  97  * # Set local link up/down
  98  * root@self# echo Y > $DBG_DIR/link
  99  * root@self# echo N > $DBG_DIR/link
 100  *
 101  * # Check if link with peer device is up/down:
 102  * root@self# cat $DBG_DIR/peer0/link
 103  *
 104  * # Block until the link is up/down
 105  * root@self# echo Y > $DBG_DIR/peer0/link_event
 106  * root@self# echo N > $DBG_DIR/peer0/link_event
 107  *-----------------------------------------------------------------------------
 108  * Eg: Doorbell registers tests (some functionality might be absent)
 109  *
 110  * # Set/clear/get local doorbell
 111  * root@self# echo 's 1' > $DBG_DIR/db
 112  * root@self# echo 'c 1' > $DBG_DIR/db
 113  * root@self# cat  $DBG_DIR/db
 114  *
 115  * # Set/clear/get local doorbell mask
 116  * root@self# echo 's 1' > $DBG_DIR/db_mask
 117  * root@self# echo 'c 1' > $DBG_DIR/db_mask
 118  * root@self# cat $DBG_DIR/db_mask
 119  *
 120  * # Ring/clear/get peer doorbell
 121  * root@peer# echo 's 1' > $DBG_DIR/peer_db
 122  * root@peer# echo 'c 1' > $DBG_DIR/peer_db
 123  * root@peer# cat $DBG_DIR/peer_db
 124  *
 125  * # Set/clear/get peer doorbell mask
 126  * root@self# echo 's 1' > $DBG_DIR/peer_db_mask
 127  * root@self# echo 'c 1' > $DBG_DIR/peer_db_mask
 128  * root@self# cat $DBG_DIR/peer_db_mask
 129  *
 130  * # Block until local doorbell is set with specified value
 131  * root@self# echo 1 > $DBG_DIR/db_event
 132  *-----------------------------------------------------------------------------
 133  * Eg: Message registers tests (functionality might be absent)
 134  *
 135  * # Set/clear/get in/out message registers status
 136  * root@self# echo 's 1' > $DBG_DIR/msg_sts
 137  * root@self# echo 'c 1' > $DBG_DIR/msg_sts
 138  * root@self# cat $DBG_DIR/msg_sts
 139  *
 140  * # Set/clear in/out message registers mask
 141  * root@self# echo 's 1' > $DBG_DIR/msg_mask
 142  * root@self# echo 'c 1' > $DBG_DIR/msg_mask
 143  *
 144  * # Get inbound message register #0 value and source of port index
 145  * root@self# cat  $DBG_DIR/msg0
 146  *
 147  * # Send some data to peer over outbound message register #0
 148  * root@self# echo 0x01020304 > $DBG_DIR/peer0/msg0
 149  *-----------------------------------------------------------------------------
 150  * Eg: Scratchpad registers tests (functionality might be absent)
 151  *
 152  * # Write/read to/from local scratchpad register #0
 153  * root@peer# echo 0x01020304 > $DBG_DIR/spad0
 154  * root@peer# cat $DBG_DIR/spad0
 155  *
 156  * # Write/read to/from peer scratchpad register #0
 157  * root@peer# echo 0x01020304 > $DBG_DIR/peer0/spad0
 158  * root@peer# cat $DBG_DIR/peer0/spad0
 159  *-----------------------------------------------------------------------------
 160  * Eg: Memory windows tests
 161  *
 162  * # Create inbound memory window buffer of specified size/get its base address
 163  * root@peer# echo 16384 > $DBG_DIR/peer0/mw_trans0
 164  * root@peer# cat $DBG_DIR/peer0/mw_trans0
 165  *
 166  * # Write/read data to/from inbound memory window
 167  * root@peer# echo Hello > $DBG_DIR/peer0/mw0
 168  * root@peer# head -c 7 $DBG_DIR/peer0/mw0
 169  *
 170  * # Map outbound memory window/check it settings (on peer device)
 171  * root@peer# echo 0xADD0BA5E:16384 > $DBG_DIR/peer0/peer_mw_trans0
 172  * root@peer# cat $DBG_DIR/peer0/peer_mw_trans0
 173  *
 174  * # Write/read data to/from outbound memory window (on peer device)
 175  * root@peer# echo olleH > $DBG_DIR/peer0/peer_mw0
 176  * root@peer# head -c 7 $DBG_DIR/peer0/peer_mw0
 177  */
 178
 179 #include <linux/init.h>
 180 #include <linux/kernel.h>
 181 #include <linux/module.h>
 182
 183 #include <linux/debugfs.h>
 184 #include <linux/dma-mapping.h>
 185 #include <linux/pci.h>
 186 #include <linux/slab.h>
 187 #include <linux/uaccess.h>
 188
 189 #include <linux/ntb.h>
 190
 191 #define DRIVER_NAME             "ntb_tool"
 192 #define DRIVER_VERSION          "2.0"
 193
 194 MODULE_LICENSE("Dual BSD/GPL");
 195 MODULE_VERSION(DRIVER_VERSION);
 196 MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
 197 MODULE_DESCRIPTION("PCIe NTB Debugging Tool");
 198
 199 /*
 200  * Inbound and outbound memory windows descriptor. Union members selection
 201  * depends on the MW type the structure describes. mm_base/dma_base are the
 202  * virtual and DMA address of an inbound MW. io_base/tr_base are the MMIO
 203  * mapped virtual and xlat addresses of an outbound MW respectively.
 204  */
 205 struct tool_mw {
 206         int widx;
 207         int pidx;
 208         struct tool_ctx *tc;
 209         union {
 210                 u8 *mm_base;
 211                 u8 __iomem *io_base;
 212         };
 213         union {
 214                 dma_addr_t dma_base;
 215                 u64 tr_base;
 216         };
 217         resource_size_t size;
 218         struct dentry *dbgfs_file;
 219 };
 220
 221 /*
 222  * Wrapper structure is used to distinguish the outbound MW peers reference
 223  * within the corresponding DebugFS directory IO operation.
 224  */
 225 struct tool_mw_wrap {
 226         int pidx;
 227         struct tool_mw *mw;
 228 };
 229
 230 struct tool_msg {
 231         int midx;
 232         int pidx;
 233         struct tool_ctx *tc;
 234 };
 235
 236 struct tool_spad {
 237         int sidx;
 238         int pidx;
 239         struct tool_ctx *tc;
 240 };
 241
 242 struct tool_peer {
 243         int pidx;
 244         struct tool_ctx *tc;
 245         int inmw_cnt;
 246         struct tool_mw *inmws;
 247         int outmw_cnt;
 248         struct tool_mw_wrap *outmws;
 249         int outmsg_cnt;
 250         struct tool_msg *outmsgs;
 251         int outspad_cnt;
 252         struct tool_spad *outspads;
 253         struct dentry *dbgfs_dir;
 254 };
 255
 256 struct tool_ctx {
 257         struct ntb_dev *ntb;
 258         wait_queue_head_t link_wq;
 259         wait_queue_head_t db_wq;
 260         wait_queue_head_t msg_wq;
 261         int outmw_cnt;
 262         struct tool_mw *outmws;
 263         int peer_cnt;
 264         struct tool_peer *peers;
 265         int inmsg_cnt;
 266         struct tool_msg *inmsgs;
 267         int inspad_cnt;
 268         struct tool_spad *inspads;
 269         struct dentry *dbgfs_dir;
 270 };
 271
 272 #define TOOL_FOPS_RDWR(__name, __read, __write) \
 273         const struct file_operations __name = { \
 274                 .owner = THIS_MODULE,           \
 275                 .open = simple_open,            \
 276                 .read = __read,                 \
 277                 .write = __write,               \
 278         }
 279
 280 #define TOOL_BUF_LEN 32
 281
 282 static struct dentry *tool_dbgfs_topdir;
 283
 284 /*==============================================================================
 285  *                               NTB events handlers
 286  *==============================================================================
 287  */
 288
 289 static void tool_link_event(void *ctx)
 290 {
 291         struct tool_ctx *tc = ctx;
 292         enum ntb_speed speed;
 293         enum ntb_width width;
 294         int up;
 295
 296         up = ntb_link_is_up(tc->ntb, &speed, &width);
 297
 298         dev_dbg(&tc->ntb->dev, "link is %s speed %d width %d\n",
 299                 up ? "up" : "down", speed, width);
 300
 301         wake_up(&tc->link_wq);
 302 }
 303
 304 static void tool_db_event(void *ctx, int vec)
 305 {
 306         struct tool_ctx *tc = ctx;
 307         u64 db_bits, db_mask;
 308
 309         db_mask = ntb_db_vector_mask(tc->ntb, vec);
 310         db_bits = ntb_db_read(tc->ntb);
 311
 312         dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
 313                 vec, db_mask, db_bits);
 314
 315         wake_up(&tc->db_wq);
 316 }
 317
 318 static void tool_msg_event(void *ctx)
 319 {
 320         struct tool_ctx *tc = ctx;
 321         u64 msg_sts;
 322
 323         msg_sts = ntb_msg_read_sts(tc->ntb);
 324
 325         dev_dbg(&tc->ntb->dev, "message bits %#llx\n", msg_sts);
 326
 327         wake_up(&tc->msg_wq);
 328 }
 329
 330 static const struct ntb_ctx_ops tool_ops = {
 331         .link_event = tool_link_event,
 332         .db_event = tool_db_event,
 333         .msg_event = tool_msg_event
 334 };
 335
 336 /*==============================================================================
 337  *                        Common read/write methods
 338  *==============================================================================
 339  */
 340
 341 static ssize_t tool_fn_read(struct tool_ctx *tc, char __user *ubuf,
 342                             size_t size, loff_t *offp,
 343                             u64 (*fn_read)(struct ntb_dev *))
 344 {
 345         size_t buf_size;
 346         char buf[TOOL_BUF_LEN];
 347         ssize_t pos;
 348
 349         if (!fn_read)
 350                 return -EINVAL;
 351
 352         buf_size = min(size, sizeof(buf));
 353
 354         pos = scnprintf(buf, buf_size, "%#llx\n", fn_read(tc->ntb));
 355
 356         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 357 }
 358
 359 static ssize_t tool_fn_write(struct tool_ctx *tc,
 360                              const char __user *ubuf,
 361                              size_t size, loff_t *offp,
 362                              int (*fn_set)(struct ntb_dev *, u64),
 363                              int (*fn_clear)(struct ntb_dev *, u64))
 364 {
 365         char *buf, cmd;
 366         ssize_t ret;
 367         u64 bits;
 368         int n;
 369
 370         if (*offp)
 371                 return 0;
 372
 373         buf = kmalloc(size + 1, GFP_KERNEL);
 374         if (!buf)
 375                 return -ENOMEM;
 376
 377         if (copy_from_user(buf, ubuf, size)) {
 378                 kfree(buf);
 379                 return -EFAULT;
 380         }
 381
 382         buf[size] = 0;
 383
 384         n = sscanf(buf, "%c %lli", &cmd, &bits);
 385
 386         kfree(buf);
 387
 388         if (n != 2) {
 389                 ret = -EINVAL;
 390         } else if (cmd == 's') {
 391                 if (!fn_set)
 392                         ret = -EINVAL;
 393                 else
 394                         ret = fn_set(tc->ntb, bits);
 395         } else if (cmd == 'c') {
 396                 if (!fn_clear)
 397                         ret = -EINVAL;
 398                 else
 399                         ret = fn_clear(tc->ntb, bits);
 400         } else {
 401                 ret = -EINVAL;
 402         }
 403
 404         return ret ? : size;
 405 }
 406
 407 /*==============================================================================
 408  *                            Port read/write methods
 409  *==============================================================================
 410  */
 411
 412 static ssize_t tool_port_read(struct file *filep, char __user *ubuf,
 413                               size_t size, loff_t *offp)
 414 {
 415         struct tool_ctx *tc = filep->private_data;
 416         char buf[TOOL_BUF_LEN];
 417         int pos;
 418
 419         pos = scnprintf(buf, sizeof(buf), "%d\n", ntb_port_number(tc->ntb));
 420
 421         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 422 }
 423
 424 static TOOL_FOPS_RDWR(tool_port_fops,
 425                       tool_port_read,
 426                       NULL);
 427
 428 static ssize_t tool_peer_port_read(struct file *filep, char __user *ubuf,
 429                                    size_t size, loff_t *offp)
 430 {
 431         struct tool_peer *peer = filep->private_data;
 432         struct tool_ctx *tc = peer->tc;
 433         char buf[TOOL_BUF_LEN];
 434         int pos;
 435
 436         pos = scnprintf(buf, sizeof(buf), "%d\n",
 437                 ntb_peer_port_number(tc->ntb, peer->pidx));
 438
 439         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 440 }
 441
 442 static TOOL_FOPS_RDWR(tool_peer_port_fops,
 443                       tool_peer_port_read,
 444                       NULL);
 445
 446 static int tool_init_peers(struct tool_ctx *tc)
 447 {
 448         int pidx;
 449
 450         tc->peer_cnt = ntb_peer_port_count(tc->ntb);
 451         tc->peers = devm_kcalloc(&tc->ntb->dev, tc->peer_cnt,
 452                                  sizeof(*tc->peers), GFP_KERNEL);
 453         if (tc->peers == NULL)
 454                 return -ENOMEM;
 455
 456         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
 457                 tc->peers[pidx].pidx = pidx;
 458                 tc->peers[pidx].tc = tc;
 459         }
 460
 461         return 0;
 462 }
 463
 464 /*==============================================================================
 465  *                       Link state read/write methods
 466  *==============================================================================
 467  */
 468
 469 static ssize_t tool_link_write(struct file *filep, const char __user *ubuf,
 470                                size_t size, loff_t *offp)
 471 {
 472         struct tool_ctx *tc = filep->private_data;
 473         bool val;
 474         int ret;
 475
 476         ret = kstrtobool_from_user(ubuf, size, &val);
 477         if (ret)
 478                 return ret;
 479
 480         if (val)
 481                 ret = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 482         else
 483                 ret = ntb_link_disable(tc->ntb);
 484
 485         if (ret)
 486                 return ret;
 487
 488         return size;
 489 }
 490
 491 static TOOL_FOPS_RDWR(tool_link_fops,
 492                       NULL,
 493                       tool_link_write);
 494
 495 static ssize_t tool_peer_link_read(struct file *filep, char __user *ubuf,
 496                                    size_t size, loff_t *offp)
 497 {
 498         struct tool_peer *peer = filep->private_data;
 499         struct tool_ctx *tc = peer->tc;
 500         char buf[3];
 501
 502         if (ntb_link_is_up(tc->ntb, NULL, NULL) & BIT(peer->pidx))
 503                 buf[0] = 'Y';
 504         else
 505                 buf[0] = 'N';
 506         buf[1] = '\n';
 507         buf[2] = '\0';
 508
 509         return simple_read_from_buffer(ubuf, size, offp, buf, 2);
 510 }
 511
 512 static TOOL_FOPS_RDWR(tool_peer_link_fops,
 513                       tool_peer_link_read,
 514                       NULL);
 515
 516 static ssize_t tool_peer_link_event_write(struct file *filep,
 517                                           const char __user *ubuf,
 518                                           size_t size, loff_t *offp)
 519 {
 520         struct tool_peer *peer = filep->private_data;
 521         struct tool_ctx *tc = peer->tc;
 522         u64 link_msk;
 523         bool val;
 524         int ret;
 525
 526         ret = kstrtobool_from_user(ubuf, size, &val);
 527         if (ret)
 528                 return ret;
 529
 530         link_msk = BIT_ULL_MASK(peer->pidx);
 531
 532         if (wait_event_interruptible(tc->link_wq,
 533                 !!(ntb_link_is_up(tc->ntb, NULL, NULL) & link_msk) == val))
 534                 return -ERESTART;
 535
 536         return size;
 537 }
 538
 539 static TOOL_FOPS_RDWR(tool_peer_link_event_fops,
 540                       NULL,
 541                       tool_peer_link_event_write);
 542
 543 /*==============================================================================
 544  *                  Memory windows read/write/setting methods
 545  *==============================================================================
 546  */
 547
 548 static ssize_t tool_mw_read(struct file *filep, char __user *ubuf,
 549                             size_t size, loff_t *offp)
 550 {
 551         struct tool_mw *inmw = filep->private_data;
 552
 553         if (inmw->mm_base == NULL)
 554                 return -ENXIO;
 555
 556         return simple_read_from_buffer(ubuf, size, offp,
 557                                        inmw->mm_base, inmw->size);
 558 }
 559
 560 static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf,
 561                              size_t size, loff_t *offp)
 562 {
 563         struct tool_mw *inmw = filep->private_data;
 564
 565         if (inmw->mm_base == NULL)
 566                 return -ENXIO;
 567
 568         return simple_write_to_buffer(inmw->mm_base, inmw->size, offp,
 569                                       ubuf, size);
 570 }
 571
 572 static TOOL_FOPS_RDWR(tool_mw_fops,
 573                       tool_mw_read,
 574                       tool_mw_write);
 575
 576 static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
 577                          size_t req_size)
 578 {
 579         resource_size_t size, addr_align, size_align;
 580         struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
 581         char buf[TOOL_BUF_LEN];
 582         int ret;
 583
 584         if (inmw->mm_base != NULL)
 585                 return 0;
 586
 587         ret = ntb_mw_get_align(tc->ntb, pidx, widx, &addr_align,
 588                                 &size_align, &size);
 589         if (ret)
 590                 return ret;
 591
 592         inmw->size = min_t(resource_size_t, req_size, size);
 593         inmw->size = round_up(inmw->size, addr_align);
 594         inmw->size = round_up(inmw->size, size_align);
 595         inmw->mm_base = dma_alloc_coherent(&tc->ntb->pdev->dev, inmw->size,
 596                                            &inmw->dma_base, GFP_KERNEL);
 597         if (!inmw->mm_base)
 598                 return -ENOMEM;
 599
 600         if (!IS_ALIGNED(inmw->dma_base, addr_align)) {
 601                 ret = -ENOMEM;
 602                 goto err_free_dma;
 603         }
 604
 605         ret = ntb_mw_set_trans(tc->ntb, pidx, widx, inmw->dma_base, inmw->size);
 606         if (ret)
 607                 goto err_free_dma;
 608
 609         snprintf(buf, sizeof(buf), "mw%d", widx);
 610         inmw->dbgfs_file = debugfs_create_file(buf, 0600,
 611                                                tc->peers[pidx].dbgfs_dir, inmw,
 612                                                &tool_mw_fops);
 613
 614         return 0;
 615
 616 err_free_dma:
 617         dma_free_coherent(&tc->ntb->pdev->dev, inmw->size, inmw->mm_base,
 618                           inmw->dma_base);
 619         inmw->mm_base = NULL;
 620         inmw->dma_base = 0;
 621         inmw->size = 0;
 622
 623         return ret;
 624 }
 625
 626 static void tool_free_mw(struct tool_ctx *tc, int pidx, int widx)
 627 {
 628         struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
 629
 630         debugfs_remove(inmw->dbgfs_file);
 631
 632         if (inmw->mm_base != NULL) {
 633                 ntb_mw_clear_trans(tc->ntb, pidx, widx);
 634                 dma_free_coherent(&tc->ntb->pdev->dev, inmw->size,
 635                                   inmw->mm_base, inmw->dma_base);
 636         }
 637
 638         inmw->mm_base = NULL;
 639         inmw->dma_base = 0;
 640         inmw->size = 0;
 641         inmw->dbgfs_file = NULL;
 642 }
 643
 644 static ssize_t tool_mw_trans_read(struct file *filep, char __user *ubuf,
 645                                   size_t size, loff_t *offp)
 646 {
 647         struct tool_mw *inmw = filep->private_data;
 648         resource_size_t addr_align;
 649         resource_size_t size_align;
 650         resource_size_t size_max;
 651         ssize_t ret, off = 0;
 652         size_t buf_size;
 653         char *buf;
 654
 655         buf_size = min_t(size_t, size, 512);
 656
 657         buf = kmalloc(buf_size, GFP_KERNEL);
 658         if (!buf)
 659                 return -ENOMEM;
 660
 661         ret = ntb_mw_get_align(inmw->tc->ntb, inmw->pidx, inmw->widx,
 662                                &addr_align, &size_align, &size_max);
 663         if (ret)
 664                 goto err;
 665
 666         off += scnprintf(buf + off, buf_size - off,
 667                          "Inbound MW     \t%d\n",
 668                          inmw->widx);
 669
 670         off += scnprintf(buf + off, buf_size - off,
 671                          "Port           \t%d (%d)\n",
 672                          ntb_peer_port_number(inmw->tc->ntb, inmw->pidx),
 673                          inmw->pidx);
 674
 675         off += scnprintf(buf + off, buf_size - off,
 676                          "Window Address \t0x%pK\n", inmw->mm_base);
 677
 678         off += scnprintf(buf + off, buf_size - off,
 679                          "DMA Address    \t%pad\n",
 680                          &inmw->dma_base);
 681
 682         off += scnprintf(buf + off, buf_size - off,
 683                          "Window Size    \t%pap\n",
 684                          &inmw->size);
 685
 686         off += scnprintf(buf + off, buf_size - off,
 687                          "Alignment      \t%pap\n",
 688                          &addr_align);
 689
 690         off += scnprintf(buf + off, buf_size - off,
 691                          "Size Alignment \t%pap\n",
 692                          &size_align);
 693
 694         off += scnprintf(buf + off, buf_size - off,
 695                          "Size Max       \t%pap\n",
 696                          &size_max);
 697
 698         ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 699
 700 err:
 701         kfree(buf);
 702
 703         return ret;
 704 }
 705
 706 static ssize_t tool_mw_trans_write(struct file *filep, const char __user *ubuf,
 707                                    size_t size, loff_t *offp)
 708 {
 709         struct tool_mw *inmw = filep->private_data;
 710         unsigned int val;
 711         int ret;
 712
 713         ret = kstrtouint_from_user(ubuf, size, 0, &val);
 714         if (ret)
 715                 return ret;
 716
 717         tool_free_mw(inmw->tc, inmw->pidx, inmw->widx);
 718         if (val) {
 719                 ret = tool_setup_mw(inmw->tc, inmw->pidx, inmw->widx, val);
 720                 if (ret)
 721                         return ret;
 722         }
 723
 724         return size;
 725 }
 726
 727 static TOOL_FOPS_RDWR(tool_mw_trans_fops,
 728                       tool_mw_trans_read,
 729                       tool_mw_trans_write);
 730
 731 static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
 732                                  size_t size, loff_t *offp)
 733 {
 734         struct tool_mw *outmw = filep->private_data;
 735         loff_t pos = *offp;
 736         ssize_t ret;
 737         void *buf;
 738
 739         if (outmw->io_base == NULL)
 740                 return -EIO;
 741
 742         if (pos >= outmw->size || !size)
 743                 return 0;
 744
 745         if (size > outmw->size - pos)
 746                 size = outmw->size - pos;
 747
 748         buf = kmalloc(size, GFP_KERNEL);
 749         if (!buf)
 750                 return -ENOMEM;
 751
 752         memcpy_fromio(buf, outmw->io_base + pos, size);
 753         ret = copy_to_user(ubuf, buf, size);
 754         if (ret == size) {
 755                 ret = -EFAULT;
 756                 goto err_free;
 757         }
 758
 759         size -= ret;
 760         *offp = pos + size;
 761         ret = size;
 762
 763 err_free:
 764         kfree(buf);
 765
 766         return ret;
 767 }
 768
 769 static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
 770                                   size_t size, loff_t *offp)
 771 {
 772         struct tool_mw *outmw = filep->private_data;
 773         ssize_t ret;
 774         loff_t pos = *offp;
 775         void *buf;
 776
 777         if (outmw->io_base == NULL)
 778                 return -EIO;
 779
 780         if (pos >= outmw->size || !size)
 781                 return 0;
 782         if (size > outmw->size - pos)
 783                 size = outmw->size - pos;
 784
 785         buf = kmalloc(size, GFP_KERNEL);
 786         if (!buf)
 787                 return -ENOMEM;
 788
 789         ret = copy_from_user(buf, ubuf, size);
 790         if (ret == size) {
 791                 ret = -EFAULT;
 792                 goto err_free;
 793         }
 794
 795         size -= ret;
 796         *offp = pos + size;
 797         ret = size;
 798
 799         memcpy_toio(outmw->io_base + pos, buf, size);
 800
 801 err_free:
 802         kfree(buf);
 803
 804         return ret;
 805 }
 806
 807 static TOOL_FOPS_RDWR(tool_peer_mw_fops,
 808                       tool_peer_mw_read,
 809                       tool_peer_mw_write);
 810
 811 static int tool_setup_peer_mw(struct tool_ctx *tc, int pidx, int widx,
 812                               u64 req_addr, size_t req_size)
 813 {
 814         struct tool_mw *outmw = &tc->outmws[widx];
 815         resource_size_t map_size;
 816         phys_addr_t map_base;
 817         char buf[TOOL_BUF_LEN];
 818         int ret;
 819
 820         if (outmw->io_base != NULL)
 821                 return 0;
 822
 823         ret = ntb_peer_mw_get_addr(tc->ntb, widx, &map_base, &map_size);
 824         if (ret)
 825                 return ret;
 826
 827         ret = ntb_peer_mw_set_trans(tc->ntb, pidx, widx, req_addr, req_size);
 828         if (ret)
 829                 return ret;
 830
 831         outmw->io_base = ioremap_wc(map_base, map_size);
 832         if (outmw->io_base == NULL) {
 833                 ret = -EFAULT;
 834                 goto err_clear_trans;
 835         }
 836
 837         outmw->tr_base = req_addr;
 838         outmw->size = req_size;
 839         outmw->pidx = pidx;
 840
 841         snprintf(buf, sizeof(buf), "peer_mw%d", widx);
 842         outmw->dbgfs_file = debugfs_create_file(buf, 0600,
 843                                                tc->peers[pidx].dbgfs_dir, outmw,
 844                                                &tool_peer_mw_fops);
 845
 846         return 0;
 847
 848 err_clear_trans:
 849         ntb_peer_mw_clear_trans(tc->ntb, pidx, widx);
 850
 851         return ret;
 852 }
 853
 854 static void tool_free_peer_mw(struct tool_ctx *tc, int widx)
 855 {
 856         struct tool_mw *outmw = &tc->outmws[widx];
 857
 858         debugfs_remove(outmw->dbgfs_file);
 859
 860         if (outmw->io_base != NULL) {
 861                 iounmap(tc->outmws[widx].io_base);
 862                 ntb_peer_mw_clear_trans(tc->ntb, outmw->pidx, widx);
 863         }
 864
 865         outmw->io_base = NULL;
 866         outmw->tr_base = 0;
 867         outmw->size = 0;
 868         outmw->pidx = -1;
 869         outmw->dbgfs_file = NULL;
 870 }
 871
 872 static ssize_t tool_peer_mw_trans_read(struct file *filep, char __user *ubuf,
 873                                         size_t size, loff_t *offp)
 874 {
 875         struct tool_mw_wrap *outmw_wrap = filep->private_data;
 876         struct tool_mw *outmw = outmw_wrap->mw;
 877         resource_size_t map_size;
 878         phys_addr_t map_base;
 879         ssize_t off = 0;
 880         size_t buf_size;
 881         char *buf;
 882         int ret;
 883
 884         ret = ntb_peer_mw_get_addr(outmw->tc->ntb, outmw->widx,
 885                                   &map_base, &map_size);
 886         if (ret)
 887                 return ret;
 888
 889         buf_size = min_t(size_t, size, 512);
 890
 891         buf = kmalloc(buf_size, GFP_KERNEL);
 892         if (!buf)
 893                 return -ENOMEM;
 894
 895         off += scnprintf(buf + off, buf_size - off,
 896                          "Outbound MW:        \t%d\n", outmw->widx);
 897
 898         if (outmw->io_base != NULL) {
 899                 off += scnprintf(buf + off, buf_size - off,
 900                         "Port attached       \t%d (%d)\n",
 901                         ntb_peer_port_number(outmw->tc->ntb, outmw->pidx),
 902                         outmw->pidx);
 903         } else {
 904                 off += scnprintf(buf + off, buf_size - off,
 905                                  "Port attached       \t-1 (-1)\n");
 906         }
 907
 908         off += scnprintf(buf + off, buf_size - off,
 909                          "Virtual address     \t0x%pK\n", outmw->io_base);
 910
 911         off += scnprintf(buf + off, buf_size - off,
 912                          "Phys Address        \t%pap\n", &map_base);
 913
 914         off += scnprintf(buf + off, buf_size - off,
 915                          "Mapping Size        \t%pap\n", &map_size);
 916
 917         off += scnprintf(buf + off, buf_size - off,
 918                          "Translation Address \t0x%016llx\n", outmw->tr_base);
 919
 920         off += scnprintf(buf + off, buf_size - off,
 921                          "Window Size         \t%pap\n", &outmw->size);
 922
 923         ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 924         kfree(buf);
 925
 926         return ret;
 927 }
 928
 929 static ssize_t tool_peer_mw_trans_write(struct file *filep,
 930                                         const char __user *ubuf,
 931                                         size_t size, loff_t *offp)
 932 {
 933         struct tool_mw_wrap *outmw_wrap = filep->private_data;
 934         struct tool_mw *outmw = outmw_wrap->mw;
 935         size_t buf_size, wsize;
 936         char buf[TOOL_BUF_LEN];
 937         int ret, n;
 938         u64 addr;
 939
 940         buf_size = min(size, (sizeof(buf) - 1));
 941         if (copy_from_user(buf, ubuf, buf_size))
 942                 return -EFAULT;
 943
 944         buf[buf_size] = '\0';
 945
 946         n = sscanf(buf, "%lli:%zi", &addr, &wsize);
 947         if (n != 2)
 948                 return -EINVAL;
 949
 950         tool_free_peer_mw(outmw->tc, outmw->widx);
 951         if (wsize) {
 952                 ret = tool_setup_peer_mw(outmw->tc, outmw_wrap->pidx,
 953                                          outmw->widx, addr, wsize);
 954                 if (ret)
 955                         return ret;
 956         }
 957
 958         return size;
 959 }
 960
 961 static TOOL_FOPS_RDWR(tool_peer_mw_trans_fops,
 962                       tool_peer_mw_trans_read,
 963                       tool_peer_mw_trans_write);
 964
 965 static int tool_init_mws(struct tool_ctx *tc)
 966 {
 967         int widx, pidx;
 968
 969         /* Initialize outbound memory windows */
 970         tc->outmw_cnt = ntb_peer_mw_count(tc->ntb);
 971         tc->outmws = devm_kcalloc(&tc->ntb->dev, tc->outmw_cnt,
 972                                   sizeof(*tc->outmws), GFP_KERNEL);
 973         if (tc->outmws == NULL)
 974                 return -ENOMEM;
 975
 976         for (widx = 0; widx < tc->outmw_cnt; widx++) {
 977                 tc->outmws[widx].widx = widx;
 978                 tc->outmws[widx].pidx = -1;
 979                 tc->outmws[widx].tc = tc;
 980         }
 981
 982         /* Initialize inbound memory windows and outbound MWs wrapper */
 983         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
 984                 tc->peers[pidx].inmw_cnt = ntb_mw_count(tc->ntb, pidx);
 985                 tc->peers[pidx].inmws =
 986                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].inmw_cnt,
 987                                     sizeof(*tc->peers[pidx].inmws), GFP_KERNEL);
 988                 if (tc->peers[pidx].inmws == NULL)
 989                         return -ENOMEM;
 990
 991                 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
 992                         tc->peers[pidx].inmws[widx].widx = widx;
 993                         tc->peers[pidx].inmws[widx].pidx = pidx;
 994                         tc->peers[pidx].inmws[widx].tc = tc;
 995                 }
 996
 997                 tc->peers[pidx].outmw_cnt = ntb_peer_mw_count(tc->ntb);
 998                 tc->peers[pidx].outmws =
 999                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmw_cnt,
1000                                    sizeof(*tc->peers[pidx].outmws), GFP_KERNEL);
1001                 if (tc->peers[pidx].outmws == NULL)
1002                         return -ENOMEM;
1003
1004                 for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
1005                         tc->peers[pidx].outmws[widx].pidx = pidx;
1006                         tc->peers[pidx].outmws[widx].mw = &tc->outmws[widx];
1007                 }
1008         }
1009
1010         return 0;
1011 }
1012
1013 static void tool_clear_mws(struct tool_ctx *tc)
1014 {
1015         int widx, pidx;
1016
1017         /* Free outbound memory windows */
1018         for (widx = 0; widx < tc->outmw_cnt; widx++)
1019                 tool_free_peer_mw(tc, widx);
1020
1021         /* Free outbound memory windows */
1022         for (pidx = 0; pidx < tc->peer_cnt; pidx++)
1023                 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++)
1024                         tool_free_mw(tc, pidx, widx);
1025 }
1026
1027 /*==============================================================================
1028  *                       Doorbell read/write methods
1029  *==============================================================================
1030  */
1031
1032 static ssize_t tool_db_read(struct file *filep, char __user *ubuf,
1033                             size_t size, loff_t *offp)
1034 {
1035         struct tool_ctx *tc = filep->private_data;
1036
1037         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read);
1038 }
1039
1040 static ssize_t tool_db_write(struct file *filep, const char __user *ubuf,
1041                              size_t size, loff_t *offp)
1042 {
1043         struct tool_ctx *tc = filep->private_data;
1044
1045         return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set,
1046                              tc->ntb->ops->db_clear);
1047 }
1048
1049 static TOOL_FOPS_RDWR(tool_db_fops,
1050                       tool_db_read,
1051                       tool_db_write);
1052
1053 static ssize_t tool_db_valid_mask_read(struct file *filep, char __user *ubuf,
1054                                        size_t size, loff_t *offp)
1055 {
1056         struct tool_ctx *tc = filep->private_data;
1057
1058         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_valid_mask);
1059 }
1060
1061 static TOOL_FOPS_RDWR(tool_db_valid_mask_fops,
1062                       tool_db_valid_mask_read,
1063                       NULL);
1064
1065 static ssize_t tool_db_mask_read(struct file *filep, char __user *ubuf,
1066                                  size_t size, loff_t *offp)
1067 {
1068         struct tool_ctx *tc = filep->private_data;
1069
1070         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read_mask);
1071 }
1072
1073 static ssize_t tool_db_mask_write(struct file *filep, const char __user *ubuf,
1074                                size_t size, loff_t *offp)
1075 {
1076         struct tool_ctx *tc = filep->private_data;
1077
1078         return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set_mask,
1079                              tc->ntb->ops->db_clear_mask);
1080 }
1081
1082 static TOOL_FOPS_RDWR(tool_db_mask_fops,
1083                       tool_db_mask_read,
1084                       tool_db_mask_write);
1085
1086 static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf,
1087                                  size_t size, loff_t *offp)
1088 {
1089         struct tool_ctx *tc = filep->private_data;
1090
1091         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->peer_db_read);
1092 }
1093
1094 static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf,
1095                                   size_t size, loff_t *offp)
1096 {
1097         struct tool_ctx *tc = filep->private_data;
1098
1099         return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->peer_db_set,
1100                              tc->ntb->ops->peer_db_clear);
1101 }
1102
1103 static TOOL_FOPS_RDWR(tool_peer_db_fops,
1104                       tool_peer_db_read,
1105                       tool_peer_db_write);
1106
1107 static ssize_t tool_peer_db_mask_read(struct file *filep, char __user *ubuf,
1108                                    size_t size, loff_t *offp)
1109 {
1110         struct tool_ctx *tc = filep->private_data;
1111
1112         return tool_fn_read(tc, ubuf, size, offp,
1113                             tc->ntb->ops->peer_db_read_mask);
1114 }
1115
1116 static ssize_t tool_peer_db_mask_write(struct file *filep,
1117                                        const char __user *ubuf,
1118                                        size_t size, loff_t *offp)
1119 {
1120         struct tool_ctx *tc = filep->private_data;
1121
1122         return tool_fn_write(tc, ubuf, size, offp,
1123                              tc->ntb->ops->peer_db_set_mask,
1124                              tc->ntb->ops->peer_db_clear_mask);
1125 }
1126
1127 static TOOL_FOPS_RDWR(tool_peer_db_mask_fops,
1128                       tool_peer_db_mask_read,
1129                       tool_peer_db_mask_write);
1130
1131 static ssize_t tool_db_event_write(struct file *filep,
1132                                    const char __user *ubuf,
1133                                    size_t size, loff_t *offp)
1134 {
1135         struct tool_ctx *tc = filep->private_data;
1136         u64 val;
1137         int ret;
1138
1139         ret = kstrtou64_from_user(ubuf, size, 0, &val);
1140         if (ret)
1141                 return ret;
1142
1143         if (wait_event_interruptible(tc->db_wq, ntb_db_read(tc->ntb) == val))
1144                 return -ERESTART;
1145
1146         return size;
1147 }
1148
1149 static TOOL_FOPS_RDWR(tool_db_event_fops,
1150                       NULL,
1151                       tool_db_event_write);
1152
1153 /*==============================================================================
1154  *                       Scratchpads read/write methods
1155  *==============================================================================
1156  */
1157
1158 static ssize_t tool_spad_read(struct file *filep, char __user *ubuf,
1159                               size_t size, loff_t *offp)
1160 {
1161         struct tool_spad *spad = filep->private_data;
1162         char buf[TOOL_BUF_LEN];
1163         ssize_t pos;
1164
1165         if (!spad->tc->ntb->ops->spad_read)
1166                 return -EINVAL;
1167
1168         pos = scnprintf(buf, sizeof(buf), "%#x\n",
1169                 ntb_spad_read(spad->tc->ntb, spad->sidx));
1170
1171         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1172 }
1173
1174 static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
1175                                size_t size, loff_t *offp)
1176 {
1177         struct tool_spad *spad = filep->private_data;
1178         u32 val;
1179         int ret;
1180
1181         if (!spad->tc->ntb->ops->spad_write) {
1182                 dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1183                 return -EINVAL;
1184         }
1185
1186         ret = kstrtou32_from_user(ubuf, size, 0, &val);
1187         if (ret)
1188                 return ret;
1189
1190         ret = ntb_spad_write(spad->tc->ntb, spad->sidx, val);
1191
1192         return ret ?: size;
1193 }
1194
1195 static TOOL_FOPS_RDWR(tool_spad_fops,
1196                       tool_spad_read,
1197                       tool_spad_write);
1198
1199 static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf,
1200                                    size_t size, loff_t *offp)
1201 {
1202         struct tool_spad *spad = filep->private_data;
1203         char buf[TOOL_BUF_LEN];
1204         ssize_t pos;
1205
1206         if (!spad->tc->ntb->ops->peer_spad_read)
1207                 return -EINVAL;
1208
1209         pos = scnprintf(buf, sizeof(buf), "%#x\n",
1210                 ntb_peer_spad_read(spad->tc->ntb, spad->pidx, spad->sidx));
1211
1212         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1213 }
1214
1215 static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf,
1216                                     size_t size, loff_t *offp)
1217 {
1218         struct tool_spad *spad = filep->private_data;
1219         u32 val;
1220         int ret;
1221
1222         if (!spad->tc->ntb->ops->peer_spad_write) {
1223                 dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1224                 return -EINVAL;
1225         }
1226
1227         ret = kstrtou32_from_user(ubuf, size, 0, &val);
1228         if (ret)
1229                 return ret;
1230
1231         ret = ntb_peer_spad_write(spad->tc->ntb, spad->pidx, spad->sidx, val);
1232
1233         return ret ?: size;
1234 }
1235
1236 static TOOL_FOPS_RDWR(tool_peer_spad_fops,
1237                       tool_peer_spad_read,
1238                       tool_peer_spad_write);
1239
1240 static int tool_init_spads(struct tool_ctx *tc)
1241 {
1242         int sidx, pidx;
1243
1244         /* Initialize inbound scratchpad structures */
1245         tc->inspad_cnt = ntb_spad_count(tc->ntb);
1246         tc->inspads = devm_kcalloc(&tc->ntb->dev, tc->inspad_cnt,
1247                                    sizeof(*tc->inspads), GFP_KERNEL);
1248         if (tc->inspads == NULL)
1249                 return -ENOMEM;
1250
1251         for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1252                 tc->inspads[sidx].sidx = sidx;
1253                 tc->inspads[sidx].pidx = -1;
1254                 tc->inspads[sidx].tc = tc;
1255         }
1256
1257         /* Initialize outbound scratchpad structures */
1258         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1259                 tc->peers[pidx].outspad_cnt = ntb_spad_count(tc->ntb);
1260                 tc->peers[pidx].outspads =
1261                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outspad_cnt,
1262                                 sizeof(*tc->peers[pidx].outspads), GFP_KERNEL);
1263                 if (tc->peers[pidx].outspads == NULL)
1264                         return -ENOMEM;
1265
1266                 for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1267                         tc->peers[pidx].outspads[sidx].sidx = sidx;
1268                         tc->peers[pidx].outspads[sidx].pidx = pidx;
1269                         tc->peers[pidx].outspads[sidx].tc = tc;
1270                 }
1271         }
1272
1273         return 0;
1274 }
1275
1276 /*==============================================================================
1277  *                       Messages read/write methods
1278  *==============================================================================
1279  */
1280
1281 static ssize_t tool_inmsg_read(struct file *filep, char __user *ubuf,
1282                                size_t size, loff_t *offp)
1283 {
1284         struct tool_msg *msg = filep->private_data;
1285         char buf[TOOL_BUF_LEN];
1286         ssize_t pos;
1287         u32 data;
1288         int pidx;
1289
1290         data = ntb_msg_read(msg->tc->ntb, &pidx, msg->midx);
1291
1292         pos = scnprintf(buf, sizeof(buf), "0x%08x<-%d\n", data, pidx);
1293
1294         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1295 }
1296
1297 static TOOL_FOPS_RDWR(tool_inmsg_fops,
1298                       tool_inmsg_read,
1299                       NULL);
1300
1301 static ssize_t tool_outmsg_write(struct file *filep,
1302                                  const char __user *ubuf,
1303                                  size_t size, loff_t *offp)
1304 {
1305         struct tool_msg *msg = filep->private_data;
1306         u32 val;
1307         int ret;
1308
1309         ret = kstrtou32_from_user(ubuf, size, 0, &val);
1310         if (ret)
1311                 return ret;
1312
1313         ret = ntb_peer_msg_write(msg->tc->ntb, msg->pidx, msg->midx, val);
1314
1315         return ret ? : size;
1316 }
1317
1318 static TOOL_FOPS_RDWR(tool_outmsg_fops,
1319                       NULL,
1320                       tool_outmsg_write);
1321
1322 static ssize_t tool_msg_sts_read(struct file *filep, char __user *ubuf,
1323                                  size_t size, loff_t *offp)
1324 {
1325         struct tool_ctx *tc = filep->private_data;
1326
1327         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_read_sts);
1328 }
1329
1330 static ssize_t tool_msg_sts_write(struct file *filep, const char __user *ubuf,
1331                                   size_t size, loff_t *offp)
1332 {
1333         struct tool_ctx *tc = filep->private_data;
1334
1335         return tool_fn_write(tc, ubuf, size, offp, NULL,
1336                              tc->ntb->ops->msg_clear_sts);
1337 }
1338
1339 static TOOL_FOPS_RDWR(tool_msg_sts_fops,
1340                       tool_msg_sts_read,
1341                       tool_msg_sts_write);
1342
1343 static ssize_t tool_msg_inbits_read(struct file *filep, char __user *ubuf,
1344                                     size_t size, loff_t *offp)
1345 {
1346         struct tool_ctx *tc = filep->private_data;
1347
1348         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_inbits);
1349 }
1350
1351 static TOOL_FOPS_RDWR(tool_msg_inbits_fops,
1352                       tool_msg_inbits_read,
1353                       NULL);
1354
1355 static ssize_t tool_msg_outbits_read(struct file *filep, char __user *ubuf,
1356                                      size_t size, loff_t *offp)
1357 {
1358         struct tool_ctx *tc = filep->private_data;
1359
1360         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_outbits);
1361 }
1362
1363 static TOOL_FOPS_RDWR(tool_msg_outbits_fops,
1364                       tool_msg_outbits_read,
1365                       NULL);
1366
1367 static ssize_t tool_msg_mask_write(struct file *filep, const char __user *ubuf,
1368                                    size_t size, loff_t *offp)
1369 {
1370         struct tool_ctx *tc = filep->private_data;
1371
1372         return tool_fn_write(tc, ubuf, size, offp,
1373                              tc->ntb->ops->msg_set_mask,
1374                              tc->ntb->ops->msg_clear_mask);
1375 }
1376
1377 static TOOL_FOPS_RDWR(tool_msg_mask_fops,
1378                       NULL,
1379                       tool_msg_mask_write);
1380
1381 static ssize_t tool_msg_event_write(struct file *filep,
1382                                     const char __user *ubuf,
1383                                     size_t size, loff_t *offp)
1384 {
1385         struct tool_ctx *tc = filep->private_data;
1386         u64 val;
1387         int ret;
1388
1389         ret = kstrtou64_from_user(ubuf, size, 0, &val);
1390         if (ret)
1391                 return ret;
1392
1393         if (wait_event_interruptible(tc->msg_wq,
1394                 ntb_msg_read_sts(tc->ntb) == val))
1395                 return -ERESTART;
1396
1397         return size;
1398 }
1399
1400 static TOOL_FOPS_RDWR(tool_msg_event_fops,
1401                       NULL,
1402                       tool_msg_event_write);
1403
1404 static int tool_init_msgs(struct tool_ctx *tc)
1405 {
1406         int midx, pidx;
1407
1408         /* Initialize inbound message structures */
1409         tc->inmsg_cnt = ntb_msg_count(tc->ntb);
1410         tc->inmsgs = devm_kcalloc(&tc->ntb->dev, tc->inmsg_cnt,
1411                                    sizeof(*tc->inmsgs), GFP_KERNEL);
1412         if (tc->inmsgs == NULL)
1413                 return -ENOMEM;
1414
1415         for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1416                 tc->inmsgs[midx].midx = midx;
1417                 tc->inmsgs[midx].pidx = -1;
1418                 tc->inmsgs[midx].tc = tc;
1419         }
1420
1421         /* Initialize outbound message structures */
1422         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1423                 tc->peers[pidx].outmsg_cnt = ntb_msg_count(tc->ntb);
1424                 tc->peers[pidx].outmsgs =
1425                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmsg_cnt,
1426                                 sizeof(*tc->peers[pidx].outmsgs), GFP_KERNEL);
1427                 if (tc->peers[pidx].outmsgs == NULL)
1428                         return -ENOMEM;
1429
1430                 for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1431                         tc->peers[pidx].outmsgs[midx].midx = midx;
1432                         tc->peers[pidx].outmsgs[midx].pidx = pidx;
1433                         tc->peers[pidx].outmsgs[midx].tc = tc;
1434                 }
1435         }
1436
1437         return 0;
1438 }
1439
1440 /*==============================================================================
1441  *                          Initialization methods
1442  *==============================================================================
1443  */
1444
1445 static struct tool_ctx *tool_create_data(struct ntb_dev *ntb)
1446 {
1447         struct tool_ctx *tc;
1448
1449         tc = devm_kzalloc(&ntb->dev, sizeof(*tc), GFP_KERNEL);
1450         if (tc == NULL)
1451                 return ERR_PTR(-ENOMEM);
1452
1453         tc->ntb = ntb;
1454         init_waitqueue_head(&tc->link_wq);
1455         init_waitqueue_head(&tc->db_wq);
1456         init_waitqueue_head(&tc->msg_wq);
1457
1458         if (ntb_db_is_unsafe(ntb))
1459                 dev_dbg(&ntb->dev, "doorbell is unsafe\n");
1460
1461         if (ntb_spad_is_unsafe(ntb))
1462                 dev_dbg(&ntb->dev, "scratchpad is unsafe\n");
1463
1464         return tc;
1465 }
1466
1467 static void tool_clear_data(struct tool_ctx *tc)
1468 {
1469         wake_up(&tc->link_wq);
1470         wake_up(&tc->db_wq);
1471         wake_up(&tc->msg_wq);
1472 }
1473
1474 static int tool_init_ntb(struct tool_ctx *tc)
1475 {
1476         return ntb_set_ctx(tc->ntb, tc, &tool_ops);
1477 }
1478
1479 static void tool_clear_ntb(struct tool_ctx *tc)
1480 {
1481         ntb_clear_ctx(tc->ntb);
1482         ntb_link_disable(tc->ntb);
1483 }
1484
1485 static void tool_setup_dbgfs(struct tool_ctx *tc)
1486 {
1487         int pidx, widx, sidx, midx;
1488         char buf[TOOL_BUF_LEN];
1489
1490         /* This modules is useless without dbgfs... */
1491         if (!tool_dbgfs_topdir) {
1492                 tc->dbgfs_dir = NULL;
1493                 return;
1494         }
1495
1496         tc->dbgfs_dir = debugfs_create_dir(dev_name(&tc->ntb->dev),
1497                                            tool_dbgfs_topdir);
1498         if (!tc->dbgfs_dir)
1499                 return;
1500
1501         debugfs_create_file("port", 0600, tc->dbgfs_dir,
1502                             tc, &tool_port_fops);
1503
1504         debugfs_create_file("link", 0600, tc->dbgfs_dir,
1505                             tc, &tool_link_fops);
1506
1507         debugfs_create_file("db", 0600, tc->dbgfs_dir,
1508                             tc, &tool_db_fops);
1509
1510         debugfs_create_file("db_valid_mask", 0600, tc->dbgfs_dir,
1511                             tc, &tool_db_valid_mask_fops);
1512
1513         debugfs_create_file("db_mask", 0600, tc->dbgfs_dir,
1514                             tc, &tool_db_mask_fops);
1515
1516         debugfs_create_file("db_event", 0600, tc->dbgfs_dir,
1517                             tc, &tool_db_event_fops);
1518
1519         debugfs_create_file("peer_db", 0600, tc->dbgfs_dir,
1520                             tc, &tool_peer_db_fops);
1521
1522         debugfs_create_file("peer_db_mask", 0600, tc->dbgfs_dir,
1523                             tc, &tool_peer_db_mask_fops);
1524
1525         if (tc->inspad_cnt != 0) {
1526                 for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1527                         snprintf(buf, sizeof(buf), "spad%d", sidx);
1528
1529                         debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1530                                            &tc->inspads[sidx], &tool_spad_fops);
1531                 }
1532         }
1533
1534         if (tc->inmsg_cnt != 0) {
1535                 for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1536                         snprintf(buf, sizeof(buf), "msg%d", midx);
1537                         debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1538                                            &tc->inmsgs[midx], &tool_inmsg_fops);
1539                 }
1540
1541                 debugfs_create_file("msg_sts", 0600, tc->dbgfs_dir,
1542                                     tc, &tool_msg_sts_fops);
1543
1544                 debugfs_create_file("msg_inbits", 0600, tc->dbgfs_dir,
1545                                     tc, &tool_msg_inbits_fops);
1546
1547                 debugfs_create_file("msg_outbits", 0600, tc->dbgfs_dir,
1548                                     tc, &tool_msg_outbits_fops);
1549
1550                 debugfs_create_file("msg_mask", 0600, tc->dbgfs_dir,
1551                                     tc, &tool_msg_mask_fops);
1552
1553                 debugfs_create_file("msg_event", 0600, tc->dbgfs_dir,
1554                                     tc, &tool_msg_event_fops);
1555         }
1556
1557         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1558                 snprintf(buf, sizeof(buf), "peer%d", pidx);
1559                 tc->peers[pidx].dbgfs_dir =
1560                         debugfs_create_dir(buf, tc->dbgfs_dir);
1561
1562                 debugfs_create_file("port", 0600,
1563                                     tc->peers[pidx].dbgfs_dir,
1564                                     &tc->peers[pidx], &tool_peer_port_fops);
1565
1566                 debugfs_create_file("link", 0200,
1567                                     tc->peers[pidx].dbgfs_dir,
1568                                     &tc->peers[pidx], &tool_peer_link_fops);
1569
1570                 debugfs_create_file("link_event", 0200,
1571                                   tc->peers[pidx].dbgfs_dir,
1572                                   &tc->peers[pidx], &tool_peer_link_event_fops);
1573
1574                 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
1575                         snprintf(buf, sizeof(buf), "mw_trans%d", widx);
1576                         debugfs_create_file(buf, 0600,
1577                                             tc->peers[pidx].dbgfs_dir,
1578                                             &tc->peers[pidx].inmws[widx],
1579                                             &tool_mw_trans_fops);
1580                 }
1581
1582                 for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
1583                         snprintf(buf, sizeof(buf), "peer_mw_trans%d", widx);
1584                         debugfs_create_file(buf, 0600,
1585                                             tc->peers[pidx].dbgfs_dir,
1586                                             &tc->peers[pidx].outmws[widx],
1587                                             &tool_peer_mw_trans_fops);
1588                 }
1589
1590                 for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1591                         snprintf(buf, sizeof(buf), "spad%d", sidx);
1592
1593                         debugfs_create_file(buf, 0600,
1594                                             tc->peers[pidx].dbgfs_dir,
1595                                             &tc->peers[pidx].outspads[sidx],
1596                                             &tool_peer_spad_fops);
1597                 }
1598
1599                 for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1600                         snprintf(buf, sizeof(buf), "msg%d", midx);
1601                         debugfs_create_file(buf, 0600,
1602                                             tc->peers[pidx].dbgfs_dir,
1603                                             &tc->peers[pidx].outmsgs[midx],
1604                                             &tool_outmsg_fops);
1605                 }
1606         }
1607 }
1608
1609 static void tool_clear_dbgfs(struct tool_ctx *tc)
1610 {
1611         debugfs_remove_recursive(tc->dbgfs_dir);
1612 }
1613
1614 static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
1615 {
1616         struct tool_ctx *tc;
1617         int ret;
1618
1619         tc = tool_create_data(ntb);
1620         if (IS_ERR(tc))
1621                 return PTR_ERR(tc);
1622
1623         ret = tool_init_peers(tc);
1624         if (ret != 0)
1625                 goto err_clear_data;
1626
1627         ret = tool_init_mws(tc);
1628         if (ret != 0)
1629                 goto err_clear_data;
1630
1631         ret = tool_init_spads(tc);
1632         if (ret != 0)
1633                 goto err_clear_mws;
1634
1635         ret = tool_init_msgs(tc);
1636         if (ret != 0)
1637                 goto err_clear_mws;
1638
1639         ret = tool_init_ntb(tc);
1640         if (ret != 0)
1641                 goto err_clear_mws;
1642
1643         tool_setup_dbgfs(tc);
1644
1645         return 0;
1646
1647 err_clear_mws:
1648         tool_clear_mws(tc);
1649
1650 err_clear_data:
1651         tool_clear_data(tc);
1652
1653         return ret;
1654 }
1655
1656 static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb)
1657 {
1658         struct tool_ctx *tc = ntb->ctx;
1659
1660         tool_clear_dbgfs(tc);
1661
1662         tool_clear_ntb(tc);
1663
1664         tool_clear_mws(tc);
1665
1666         tool_clear_data(tc);
1667 }
1668
1669 static struct ntb_client tool_client = {
1670         .ops = {
1671                 .probe = tool_probe,
1672                 .remove = tool_remove,
1673         }
1674 };
1675
1676 static int __init tool_init(void)
1677 {
1678         int ret;
1679
1680         if (debugfs_initialized())
1681                 tool_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1682
1683         ret = ntb_register_client(&tool_client);
1684         if (ret)
1685                 debugfs_remove_recursive(tool_dbgfs_topdir);
1686
1687         return ret;
1688 }
1689 module_init(tool_init);
1690
1691 static void __exit tool_exit(void)
1692 {
1693         ntb_unregister_client(&tool_client);
1694         debugfs_remove_recursive(tool_dbgfs_topdir);
1695 }
1696 module_exit(tool_exit);