include/trace/events/rpcrdma.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Copyright (c) 2017, 2018 Oracle.  All rights reserved.
   4  *
   5  * Trace point definitions for the "rpcrdma" subsystem.
   6  */
   7 #undef TRACE_SYSTEM
   8 #define TRACE_SYSTEM rpcrdma
   9
  10 #if !defined(_TRACE_RPCRDMA_H) || defined(TRACE_HEADER_MULTI_READ)
  11 #define _TRACE_RPCRDMA_H
  12
  13 #include <linux/scatterlist.h>
  14 #include <linux/tracepoint.h>
  15 #include <trace/events/rdma.h>
  16
  17 /**
  18  ** Event classes
  19  **/
  20
  21 DECLARE_EVENT_CLASS(xprtrdma_reply_event,
  22         TP_PROTO(
  23                 const struct rpcrdma_rep *rep
  24         ),
  25
  26         TP_ARGS(rep),
  27
  28         TP_STRUCT__entry(
  29                 __field(const void *, rep)
  30                 __field(const void *, r_xprt)
  31                 __field(u32, xid)
  32                 __field(u32, version)
  33                 __field(u32, proc)
  34         ),
  35
  36         TP_fast_assign(
  37                 __entry->rep = rep;
  38                 __entry->r_xprt = rep->rr_rxprt;
  39                 __entry->xid = be32_to_cpu(rep->rr_xid);
  40                 __entry->version = be32_to_cpu(rep->rr_vers);
  41                 __entry->proc = be32_to_cpu(rep->rr_proc);
  42         ),
  43
  44         TP_printk("rxprt %p xid=0x%08x rep=%p: version %u proc %u",
  45                 __entry->r_xprt, __entry->xid, __entry->rep,
  46                 __entry->version, __entry->proc
  47         )
  48 );
  49
  50 #define DEFINE_REPLY_EVENT(name)                                        \
  51                 DEFINE_EVENT(xprtrdma_reply_event, name,                \
  52                                 TP_PROTO(                               \
  53                                         const struct rpcrdma_rep *rep   \
  54                                 ),                                      \
  55                                 TP_ARGS(rep))
  56
  57 DECLARE_EVENT_CLASS(xprtrdma_rxprt,
  58         TP_PROTO(
  59                 const struct rpcrdma_xprt *r_xprt
  60         ),
  61
  62         TP_ARGS(r_xprt),
  63
  64         TP_STRUCT__entry(
  65                 __field(const void *, r_xprt)
  66                 __string(addr, rpcrdma_addrstr(r_xprt))
  67                 __string(port, rpcrdma_portstr(r_xprt))
  68         ),
  69
  70         TP_fast_assign(
  71                 __entry->r_xprt = r_xprt;
  72                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
  73                 __assign_str(port, rpcrdma_portstr(r_xprt));
  74         ),
  75
  76         TP_printk("peer=[%s]:%s r_xprt=%p",
  77                 __get_str(addr), __get_str(port), __entry->r_xprt
  78         )
  79 );
  80
  81 #define DEFINE_RXPRT_EVENT(name)                                        \
  82                 DEFINE_EVENT(xprtrdma_rxprt, name,                      \
  83                                 TP_PROTO(                               \
  84                                         const struct rpcrdma_xprt *r_xprt \
  85                                 ),                                      \
  86                                 TP_ARGS(r_xprt))
  87
  88 DECLARE_EVENT_CLASS(xprtrdma_connect_class,
  89         TP_PROTO(
  90                 const struct rpcrdma_xprt *r_xprt,
  91                 int rc
  92         ),
  93
  94         TP_ARGS(r_xprt, rc),
  95
  96         TP_STRUCT__entry(
  97                 __field(const void *, r_xprt)
  98                 __field(int, rc)
  99                 __field(int, connect_status)
 100                 __string(addr, rpcrdma_addrstr(r_xprt))
 101                 __string(port, rpcrdma_portstr(r_xprt))
 102         ),
 103
 104         TP_fast_assign(
 105                 __entry->r_xprt = r_xprt;
 106                 __entry->rc = rc;
 107                 __entry->connect_status = r_xprt->rx_ep->re_connect_status;
 108                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 109                 __assign_str(port, rpcrdma_portstr(r_xprt));
 110         ),
 111
 112         TP_printk("peer=[%s]:%s r_xprt=%p: rc=%d connection status=%d",
 113                 __get_str(addr), __get_str(port), __entry->r_xprt,
 114                 __entry->rc, __entry->connect_status
 115         )
 116 );
 117
 118 #define DEFINE_CONN_EVENT(name)                                         \
 119                 DEFINE_EVENT(xprtrdma_connect_class, xprtrdma_##name,   \
 120                                 TP_PROTO(                               \
 121                                         const struct rpcrdma_xprt *r_xprt, \
 122                                         int rc                          \
 123                                 ),                                      \
 124                                 TP_ARGS(r_xprt, rc))
 125
 126 DECLARE_EVENT_CLASS(xprtrdma_rdch_event,
 127         TP_PROTO(
 128                 const struct rpc_task *task,
 129                 unsigned int pos,
 130                 struct rpcrdma_mr *mr,
 131                 int nsegs
 132         ),
 133
 134         TP_ARGS(task, pos, mr, nsegs),
 135
 136         TP_STRUCT__entry(
 137                 __field(unsigned int, task_id)
 138                 __field(unsigned int, client_id)
 139                 __field(unsigned int, pos)
 140                 __field(int, nents)
 141                 __field(u32, handle)
 142                 __field(u32, length)
 143                 __field(u64, offset)
 144                 __field(int, nsegs)
 145         ),
 146
 147         TP_fast_assign(
 148                 __entry->task_id = task->tk_pid;
 149                 __entry->client_id = task->tk_client->cl_clid;
 150                 __entry->pos = pos;
 151                 __entry->nents = mr->mr_nents;
 152                 __entry->handle = mr->mr_handle;
 153                 __entry->length = mr->mr_length;
 154                 __entry->offset = mr->mr_offset;
 155                 __entry->nsegs = nsegs;
 156         ),
 157
 158         TP_printk("task:%u@%u pos=%u %u@0x%016llx:0x%08x (%s)",
 159                 __entry->task_id, __entry->client_id,
 160                 __entry->pos, __entry->length,
 161                 (unsigned long long)__entry->offset, __entry->handle,
 162                 __entry->nents < __entry->nsegs ? "more" : "last"
 163         )
 164 );
 165
 166 #define DEFINE_RDCH_EVENT(name)                                         \
 167                 DEFINE_EVENT(xprtrdma_rdch_event, xprtrdma_chunk_##name,\
 168                                 TP_PROTO(                               \
 169                                         const struct rpc_task *task,    \
 170                                         unsigned int pos,               \
 171                                         struct rpcrdma_mr *mr,          \
 172                                         int nsegs                       \
 173                                 ),                                      \
 174                                 TP_ARGS(task, pos, mr, nsegs))
 175
 176 DECLARE_EVENT_CLASS(xprtrdma_wrch_event,
 177         TP_PROTO(
 178                 const struct rpc_task *task,
 179                 struct rpcrdma_mr *mr,
 180                 int nsegs
 181         ),
 182
 183         TP_ARGS(task, mr, nsegs),
 184
 185         TP_STRUCT__entry(
 186                 __field(unsigned int, task_id)
 187                 __field(unsigned int, client_id)
 188                 __field(int, nents)
 189                 __field(u32, handle)
 190                 __field(u32, length)
 191                 __field(u64, offset)
 192                 __field(int, nsegs)
 193         ),
 194
 195         TP_fast_assign(
 196                 __entry->task_id = task->tk_pid;
 197                 __entry->client_id = task->tk_client->cl_clid;
 198                 __entry->nents = mr->mr_nents;
 199                 __entry->handle = mr->mr_handle;
 200                 __entry->length = mr->mr_length;
 201                 __entry->offset = mr->mr_offset;
 202                 __entry->nsegs = nsegs;
 203         ),
 204
 205         TP_printk("task:%u@%u %u@0x%016llx:0x%08x (%s)",
 206                 __entry->task_id, __entry->client_id,
 207                 __entry->length, (unsigned long long)__entry->offset,
 208                 __entry->handle,
 209                 __entry->nents < __entry->nsegs ? "more" : "last"
 210         )
 211 );
 212
 213 #define DEFINE_WRCH_EVENT(name)                                         \
 214                 DEFINE_EVENT(xprtrdma_wrch_event, xprtrdma_chunk_##name,\
 215                                 TP_PROTO(                               \
 216                                         const struct rpc_task *task,    \
 217                                         struct rpcrdma_mr *mr,          \
 218                                         int nsegs                       \
 219                                 ),                                      \
 220                                 TP_ARGS(task, mr, nsegs))
 221
 222 DECLARE_EVENT_CLASS(xprtrdma_frwr_done,
 223         TP_PROTO(
 224                 const struct ib_wc *wc,
 225                 const struct rpcrdma_frwr *frwr
 226         ),
 227
 228         TP_ARGS(wc, frwr),
 229
 230         TP_STRUCT__entry(
 231                 __field(u32, mr_id)
 232                 __field(unsigned int, status)
 233                 __field(unsigned int, vendor_err)
 234         ),
 235
 236         TP_fast_assign(
 237                 __entry->mr_id = frwr->fr_mr->res.id;
 238                 __entry->status = wc->status;
 239                 __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
 240         ),
 241
 242         TP_printk(
 243                 "mr.id=%u: %s (%u/0x%x)",
 244                 __entry->mr_id, rdma_show_wc_status(__entry->status),
 245                 __entry->status, __entry->vendor_err
 246         )
 247 );
 248
 249 #define DEFINE_FRWR_DONE_EVENT(name)                                    \
 250                 DEFINE_EVENT(xprtrdma_frwr_done, name,                  \
 251                                 TP_PROTO(                               \
 252                                         const struct ib_wc *wc,         \
 253                                         const struct rpcrdma_frwr *frwr \
 254                                 ),                                      \
 255                                 TP_ARGS(wc, frwr))
 256
 257 TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
 258 TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
 259 TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
 260 TRACE_DEFINE_ENUM(DMA_NONE);
 261
 262 #define xprtrdma_show_direction(x)                                      \
 263                 __print_symbolic(x,                                     \
 264                                 { DMA_BIDIRECTIONAL, "BIDIR" },         \
 265                                 { DMA_TO_DEVICE, "TO_DEVICE" },         \
 266                                 { DMA_FROM_DEVICE, "FROM_DEVICE" },     \
 267                                 { DMA_NONE, "NONE" })
 268
 269 DECLARE_EVENT_CLASS(xprtrdma_mr,
 270         TP_PROTO(
 271                 const struct rpcrdma_mr *mr
 272         ),
 273
 274         TP_ARGS(mr),
 275
 276         TP_STRUCT__entry(
 277                 __field(u32, mr_id)
 278                 __field(int, nents)
 279                 __field(u32, handle)
 280                 __field(u32, length)
 281                 __field(u64, offset)
 282                 __field(u32, dir)
 283         ),
 284
 285         TP_fast_assign(
 286                 __entry->mr_id  = mr->frwr.fr_mr->res.id;
 287                 __entry->nents  = mr->mr_nents;
 288                 __entry->handle = mr->mr_handle;
 289                 __entry->length = mr->mr_length;
 290                 __entry->offset = mr->mr_offset;
 291                 __entry->dir    = mr->mr_dir;
 292         ),
 293
 294         TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
 295                 __entry->mr_id, __entry->nents, __entry->length,
 296                 (unsigned long long)__entry->offset, __entry->handle,
 297                 xprtrdma_show_direction(__entry->dir)
 298         )
 299 );
 300
 301 #define DEFINE_MR_EVENT(name) \
 302                 DEFINE_EVENT(xprtrdma_mr, xprtrdma_mr_##name, \
 303                                 TP_PROTO( \
 304                                         const struct rpcrdma_mr *mr \
 305                                 ), \
 306                                 TP_ARGS(mr))
 307
 308 DECLARE_EVENT_CLASS(xprtrdma_cb_event,
 309         TP_PROTO(
 310                 const struct rpc_rqst *rqst
 311         ),
 312
 313         TP_ARGS(rqst),
 314
 315         TP_STRUCT__entry(
 316                 __field(const void *, rqst)
 317                 __field(const void *, rep)
 318                 __field(const void *, req)
 319                 __field(u32, xid)
 320         ),
 321
 322         TP_fast_assign(
 323                 __entry->rqst = rqst;
 324                 __entry->req = rpcr_to_rdmar(rqst);
 325                 __entry->rep = rpcr_to_rdmar(rqst)->rl_reply;
 326                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 327         ),
 328
 329         TP_printk("xid=0x%08x, rqst=%p req=%p rep=%p",
 330                 __entry->xid, __entry->rqst, __entry->req, __entry->rep
 331         )
 332 );
 333
 334 #define DEFINE_CB_EVENT(name)                                           \
 335                 DEFINE_EVENT(xprtrdma_cb_event, name,                   \
 336                                 TP_PROTO(                               \
 337                                         const struct rpc_rqst *rqst     \
 338                                 ),                                      \
 339                                 TP_ARGS(rqst))
 340
 341 /**
 342  ** Connection events
 343  **/
 344
 345 TRACE_EVENT(xprtrdma_inline_thresh,
 346         TP_PROTO(
 347                 const struct rpcrdma_ep *ep
 348         ),
 349
 350         TP_ARGS(ep),
 351
 352         TP_STRUCT__entry(
 353                 __field(unsigned int, inline_send)
 354                 __field(unsigned int, inline_recv)
 355                 __field(unsigned int, max_send)
 356                 __field(unsigned int, max_recv)
 357                 __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
 358                 __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
 359         ),
 360
 361         TP_fast_assign(
 362                 const struct rdma_cm_id *id = ep->re_id;
 363
 364                 __entry->inline_send = ep->re_inline_send;
 365                 __entry->inline_recv = ep->re_inline_recv;
 366                 __entry->max_send = ep->re_max_inline_send;
 367                 __entry->max_recv = ep->re_max_inline_recv;
 368                 memcpy(__entry->srcaddr, &id->route.addr.src_addr,
 369                        sizeof(struct sockaddr_in6));
 370                 memcpy(__entry->dstaddr, &id->route.addr.dst_addr,
 371                        sizeof(struct sockaddr_in6));
 372         ),
 373
 374         TP_printk("%pISpc -> %pISpc neg send/recv=%u/%u, calc send/recv=%u/%u",
 375                 __entry->srcaddr, __entry->dstaddr,
 376                 __entry->inline_send, __entry->inline_recv,
 377                 __entry->max_send, __entry->max_recv
 378         )
 379 );
 380
 381 DEFINE_CONN_EVENT(connect);
 382 DEFINE_CONN_EVENT(disconnect);
 383 DEFINE_CONN_EVENT(flush_dct);
 384
 385 DEFINE_RXPRT_EVENT(xprtrdma_create);
 386 DEFINE_RXPRT_EVENT(xprtrdma_op_destroy);
 387 DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
 388 DEFINE_RXPRT_EVENT(xprtrdma_op_close);
 389 DEFINE_RXPRT_EVENT(xprtrdma_op_setport);
 390
 391 TRACE_EVENT(xprtrdma_op_connect,
 392         TP_PROTO(
 393                 const struct rpcrdma_xprt *r_xprt,
 394                 unsigned long delay
 395         ),
 396
 397         TP_ARGS(r_xprt, delay),
 398
 399         TP_STRUCT__entry(
 400                 __field(const void *, r_xprt)
 401                 __field(unsigned long, delay)
 402                 __string(addr, rpcrdma_addrstr(r_xprt))
 403                 __string(port, rpcrdma_portstr(r_xprt))
 404         ),
 405
 406         TP_fast_assign(
 407                 __entry->r_xprt = r_xprt;
 408                 __entry->delay = delay;
 409                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 410                 __assign_str(port, rpcrdma_portstr(r_xprt));
 411         ),
 412
 413         TP_printk("peer=[%s]:%s r_xprt=%p delay=%lu",
 414                 __get_str(addr), __get_str(port), __entry->r_xprt,
 415                 __entry->delay
 416         )
 417 );
 418
 419
 420 TRACE_EVENT(xprtrdma_op_set_cto,
 421         TP_PROTO(
 422                 const struct rpcrdma_xprt *r_xprt,
 423                 unsigned long connect,
 424                 unsigned long reconnect
 425         ),
 426
 427         TP_ARGS(r_xprt, connect, reconnect),
 428
 429         TP_STRUCT__entry(
 430                 __field(const void *, r_xprt)
 431                 __field(unsigned long, connect)
 432                 __field(unsigned long, reconnect)
 433                 __string(addr, rpcrdma_addrstr(r_xprt))
 434                 __string(port, rpcrdma_portstr(r_xprt))
 435         ),
 436
 437         TP_fast_assign(
 438                 __entry->r_xprt = r_xprt;
 439                 __entry->connect = connect;
 440                 __entry->reconnect = reconnect;
 441                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 442                 __assign_str(port, rpcrdma_portstr(r_xprt));
 443         ),
 444
 445         TP_printk("peer=[%s]:%s r_xprt=%p: connect=%lu reconnect=%lu",
 446                 __get_str(addr), __get_str(port), __entry->r_xprt,
 447                 __entry->connect / HZ, __entry->reconnect / HZ
 448         )
 449 );
 450
 451 TRACE_EVENT(xprtrdma_qp_event,
 452         TP_PROTO(
 453                 const struct rpcrdma_ep *ep,
 454                 const struct ib_event *event
 455         ),
 456
 457         TP_ARGS(ep, event),
 458
 459         TP_STRUCT__entry(
 460                 __field(unsigned long, event)
 461                 __string(name, event->device->name)
 462                 __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
 463                 __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
 464         ),
 465
 466         TP_fast_assign(
 467                 const struct rdma_cm_id *id = ep->re_id;
 468
 469                 __entry->event = event->event;
 470                 __assign_str(name, event->device->name);
 471                 memcpy(__entry->srcaddr, &id->route.addr.src_addr,
 472                        sizeof(struct sockaddr_in6));
 473                 memcpy(__entry->dstaddr, &id->route.addr.dst_addr,
 474                        sizeof(struct sockaddr_in6));
 475         ),
 476
 477         TP_printk("%pISpc -> %pISpc device=%s %s (%lu)",
 478                 __entry->srcaddr, __entry->dstaddr, __get_str(name),
 479                 rdma_show_ib_event(__entry->event), __entry->event
 480         )
 481 );
 482
 483 /**
 484  ** Call events
 485  **/
 486
 487 TRACE_EVENT(xprtrdma_createmrs,
 488         TP_PROTO(
 489                 const struct rpcrdma_xprt *r_xprt,
 490                 unsigned int count
 491         ),
 492
 493         TP_ARGS(r_xprt, count),
 494
 495         TP_STRUCT__entry(
 496                 __field(const void *, r_xprt)
 497                 __string(addr, rpcrdma_addrstr(r_xprt))
 498                 __string(port, rpcrdma_portstr(r_xprt))
 499                 __field(unsigned int, count)
 500         ),
 501
 502         TP_fast_assign(
 503                 __entry->r_xprt = r_xprt;
 504                 __entry->count = count;
 505                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 506                 __assign_str(port, rpcrdma_portstr(r_xprt));
 507         ),
 508
 509         TP_printk("peer=[%s]:%s r_xprt=%p: created %u MRs",
 510                 __get_str(addr), __get_str(port), __entry->r_xprt,
 511                 __entry->count
 512         )
 513 );
 514
 515 TRACE_EVENT(xprtrdma_mr_get,
 516         TP_PROTO(
 517                 const struct rpcrdma_req *req
 518         ),
 519
 520         TP_ARGS(req),
 521
 522         TP_STRUCT__entry(
 523                 __field(const void *, req)
 524                 __field(unsigned int, task_id)
 525                 __field(unsigned int, client_id)
 526                 __field(u32, xid)
 527         ),
 528
 529         TP_fast_assign(
 530                 const struct rpc_rqst *rqst = &req->rl_slot;
 531
 532                 __entry->req = req;
 533                 __entry->task_id = rqst->rq_task->tk_pid;
 534                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 535                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 536         ),
 537
 538         TP_printk("task:%u@%u xid=0x%08x req=%p",
 539                 __entry->task_id, __entry->client_id, __entry->xid,
 540                 __entry->req
 541         )
 542 );
 543
 544 TRACE_EVENT(xprtrdma_nomrs,
 545         TP_PROTO(
 546                 const struct rpcrdma_req *req
 547         ),
 548
 549         TP_ARGS(req),
 550
 551         TP_STRUCT__entry(
 552                 __field(const void *, req)
 553                 __field(unsigned int, task_id)
 554                 __field(unsigned int, client_id)
 555                 __field(u32, xid)
 556         ),
 557
 558         TP_fast_assign(
 559                 const struct rpc_rqst *rqst = &req->rl_slot;
 560
 561                 __entry->req = req;
 562                 __entry->task_id = rqst->rq_task->tk_pid;
 563                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 564                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 565         ),
 566
 567         TP_printk("task:%u@%u xid=0x%08x req=%p",
 568                 __entry->task_id, __entry->client_id, __entry->xid,
 569                 __entry->req
 570         )
 571 );
 572
 573 DEFINE_RDCH_EVENT(read);
 574 DEFINE_WRCH_EVENT(write);
 575 DEFINE_WRCH_EVENT(reply);
 576
 577 TRACE_DEFINE_ENUM(rpcrdma_noch);
 578 TRACE_DEFINE_ENUM(rpcrdma_noch_pullup);
 579 TRACE_DEFINE_ENUM(rpcrdma_noch_mapped);
 580 TRACE_DEFINE_ENUM(rpcrdma_readch);
 581 TRACE_DEFINE_ENUM(rpcrdma_areadch);
 582 TRACE_DEFINE_ENUM(rpcrdma_writech);
 583 TRACE_DEFINE_ENUM(rpcrdma_replych);
 584
 585 #define xprtrdma_show_chunktype(x)                                      \
 586                 __print_symbolic(x,                                     \
 587                                 { rpcrdma_noch, "inline" },             \
 588                                 { rpcrdma_noch_pullup, "pullup" },      \
 589                                 { rpcrdma_noch_mapped, "mapped" },      \
 590                                 { rpcrdma_readch, "read list" },        \
 591                                 { rpcrdma_areadch, "*read list" },      \
 592                                 { rpcrdma_writech, "write list" },      \
 593                                 { rpcrdma_replych, "reply chunk" })
 594
 595 TRACE_EVENT(xprtrdma_marshal,
 596         TP_PROTO(
 597                 const struct rpcrdma_req *req,
 598                 unsigned int rtype,
 599                 unsigned int wtype
 600         ),
 601
 602         TP_ARGS(req, rtype, wtype),
 603
 604         TP_STRUCT__entry(
 605                 __field(unsigned int, task_id)
 606                 __field(unsigned int, client_id)
 607                 __field(u32, xid)
 608                 __field(unsigned int, hdrlen)
 609                 __field(unsigned int, headlen)
 610                 __field(unsigned int, pagelen)
 611                 __field(unsigned int, taillen)
 612                 __field(unsigned int, rtype)
 613                 __field(unsigned int, wtype)
 614         ),
 615
 616         TP_fast_assign(
 617                 const struct rpc_rqst *rqst = &req->rl_slot;
 618
 619                 __entry->task_id = rqst->rq_task->tk_pid;
 620                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 621                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 622                 __entry->hdrlen = req->rl_hdrbuf.len;
 623                 __entry->headlen = rqst->rq_snd_buf.head[0].iov_len;
 624                 __entry->pagelen = rqst->rq_snd_buf.page_len;
 625                 __entry->taillen = rqst->rq_snd_buf.tail[0].iov_len;
 626                 __entry->rtype = rtype;
 627                 __entry->wtype = wtype;
 628         ),
 629
 630         TP_printk("task:%u@%u xid=0x%08x: hdr=%u xdr=%u/%u/%u %s/%s",
 631                 __entry->task_id, __entry->client_id, __entry->xid,
 632                 __entry->hdrlen,
 633                 __entry->headlen, __entry->pagelen, __entry->taillen,
 634                 xprtrdma_show_chunktype(__entry->rtype),
 635                 xprtrdma_show_chunktype(__entry->wtype)
 636         )
 637 );
 638
 639 TRACE_EVENT(xprtrdma_marshal_failed,
 640         TP_PROTO(const struct rpc_rqst *rqst,
 641                  int ret
 642         ),
 643
 644         TP_ARGS(rqst, ret),
 645
 646         TP_STRUCT__entry(
 647                 __field(unsigned int, task_id)
 648                 __field(unsigned int, client_id)
 649                 __field(u32, xid)
 650                 __field(int, ret)
 651         ),
 652
 653         TP_fast_assign(
 654                 __entry->task_id = rqst->rq_task->tk_pid;
 655                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 656                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 657                 __entry->ret = ret;
 658         ),
 659
 660         TP_printk("task:%u@%u xid=0x%08x: ret=%d",
 661                 __entry->task_id, __entry->client_id, __entry->xid,
 662                 __entry->ret
 663         )
 664 );
 665
 666 TRACE_EVENT(xprtrdma_prepsend_failed,
 667         TP_PROTO(const struct rpc_rqst *rqst,
 668                  int ret
 669         ),
 670
 671         TP_ARGS(rqst, ret),
 672
 673         TP_STRUCT__entry(
 674                 __field(unsigned int, task_id)
 675                 __field(unsigned int, client_id)
 676                 __field(u32, xid)
 677                 __field(int, ret)
 678         ),
 679
 680         TP_fast_assign(
 681                 __entry->task_id = rqst->rq_task->tk_pid;
 682                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 683                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 684                 __entry->ret = ret;
 685         ),
 686
 687         TP_printk("task:%u@%u xid=0x%08x: ret=%d",
 688                 __entry->task_id, __entry->client_id, __entry->xid,
 689                 __entry->ret
 690         )
 691 );
 692
 693 TRACE_EVENT(xprtrdma_post_send,
 694         TP_PROTO(
 695                 const struct rpcrdma_req *req,
 696                 int status
 697         ),
 698
 699         TP_ARGS(req, status),
 700
 701         TP_STRUCT__entry(
 702                 __field(const void *, req)
 703                 __field(const void *, sc)
 704                 __field(unsigned int, task_id)
 705                 __field(unsigned int, client_id)
 706                 __field(int, num_sge)
 707                 __field(int, signaled)
 708                 __field(int, status)
 709         ),
 710
 711         TP_fast_assign(
 712                 const struct rpc_rqst *rqst = &req->rl_slot;
 713
 714                 __entry->task_id = rqst->rq_task->tk_pid;
 715                 __entry->client_id = rqst->rq_task->tk_client ?
 716                                      rqst->rq_task->tk_client->cl_clid : -1;
 717                 __entry->req = req;
 718                 __entry->sc = req->rl_sendctx;
 719                 __entry->num_sge = req->rl_wr.num_sge;
 720                 __entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
 721                 __entry->status = status;
 722         ),
 723
 724         TP_printk("task:%u@%u req=%p sc=%p (%d SGE%s) %sstatus=%d",
 725                 __entry->task_id, __entry->client_id,
 726                 __entry->req, __entry->sc, __entry->num_sge,
 727                 (__entry->num_sge == 1 ? "" : "s"),
 728                 (__entry->signaled ? "signaled " : ""),
 729                 __entry->status
 730         )
 731 );
 732
 733 TRACE_EVENT(xprtrdma_post_recv,
 734         TP_PROTO(
 735                 const struct rpcrdma_rep *rep
 736         ),
 737
 738         TP_ARGS(rep),
 739
 740         TP_STRUCT__entry(
 741                 __field(const void *, rep)
 742         ),
 743
 744         TP_fast_assign(
 745                 __entry->rep = rep;
 746         ),
 747
 748         TP_printk("rep=%p",
 749                 __entry->rep
 750         )
 751 );
 752
 753 TRACE_EVENT(xprtrdma_post_recvs,
 754         TP_PROTO(
 755                 const struct rpcrdma_xprt *r_xprt,
 756                 unsigned int count,
 757                 int status
 758         ),
 759
 760         TP_ARGS(r_xprt, count, status),
 761
 762         TP_STRUCT__entry(
 763                 __field(const void *, r_xprt)
 764                 __field(unsigned int, count)
 765                 __field(int, status)
 766                 __field(int, posted)
 767                 __string(addr, rpcrdma_addrstr(r_xprt))
 768                 __string(port, rpcrdma_portstr(r_xprt))
 769         ),
 770
 771         TP_fast_assign(
 772                 __entry->r_xprt = r_xprt;
 773                 __entry->count = count;
 774                 __entry->status = status;
 775                 __entry->posted = r_xprt->rx_ep->re_receive_count;
 776                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 777                 __assign_str(port, rpcrdma_portstr(r_xprt));
 778         ),
 779
 780         TP_printk("peer=[%s]:%s r_xprt=%p: %u new recvs, %d active (rc %d)",
 781                 __get_str(addr), __get_str(port), __entry->r_xprt,
 782                 __entry->count, __entry->posted, __entry->status
 783         )
 784 );
 785
 786 TRACE_EVENT(xprtrdma_post_linv,
 787         TP_PROTO(
 788                 const struct rpcrdma_req *req,
 789                 int status
 790         ),
 791
 792         TP_ARGS(req, status),
 793
 794         TP_STRUCT__entry(
 795                 __field(const void *, req)
 796                 __field(int, status)
 797                 __field(u32, xid)
 798         ),
 799
 800         TP_fast_assign(
 801                 __entry->req = req;
 802                 __entry->status = status;
 803                 __entry->xid = be32_to_cpu(req->rl_slot.rq_xid);
 804         ),
 805
 806         TP_printk("req=%p xid=0x%08x status=%d",
 807                 __entry->req, __entry->xid, __entry->status
 808         )
 809 );
 810
 811 /**
 812  ** Completion events
 813  **/
 814
 815 TRACE_EVENT(xprtrdma_wc_send,
 816         TP_PROTO(
 817                 const struct rpcrdma_sendctx *sc,
 818                 const struct ib_wc *wc
 819         ),
 820
 821         TP_ARGS(sc, wc),
 822
 823         TP_STRUCT__entry(
 824                 __field(const void *, req)
 825                 __field(const void *, sc)
 826                 __field(unsigned int, unmap_count)
 827                 __field(unsigned int, status)
 828                 __field(unsigned int, vendor_err)
 829         ),
 830
 831         TP_fast_assign(
 832                 __entry->req = sc->sc_req;
 833                 __entry->sc = sc;
 834                 __entry->unmap_count = sc->sc_unmap_count;
 835                 __entry->status = wc->status;
 836                 __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
 837         ),
 838
 839         TP_printk("req=%p sc=%p unmapped=%u: %s (%u/0x%x)",
 840                 __entry->req, __entry->sc, __entry->unmap_count,
 841                 rdma_show_wc_status(__entry->status),
 842                 __entry->status, __entry->vendor_err
 843         )
 844 );
 845
 846 TRACE_EVENT(xprtrdma_wc_receive,
 847         TP_PROTO(
 848                 const struct ib_wc *wc
 849         ),
 850
 851         TP_ARGS(wc),
 852
 853         TP_STRUCT__entry(
 854                 __field(const void *, rep)
 855                 __field(u32, byte_len)
 856                 __field(unsigned int, status)
 857                 __field(u32, vendor_err)
 858         ),
 859
 860         TP_fast_assign(
 861                 __entry->rep = container_of(wc->wr_cqe, struct rpcrdma_rep,
 862                                             rr_cqe);
 863                 __entry->status = wc->status;
 864                 if (wc->status) {
 865                         __entry->byte_len = 0;
 866                         __entry->vendor_err = wc->vendor_err;
 867                 } else {
 868                         __entry->byte_len = wc->byte_len;
 869                         __entry->vendor_err = 0;
 870                 }
 871         ),
 872
 873         TP_printk("rep=%p %u bytes: %s (%u/0x%x)",
 874                 __entry->rep, __entry->byte_len,
 875                 rdma_show_wc_status(__entry->status),
 876                 __entry->status, __entry->vendor_err
 877         )
 878 );
 879
 880 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg);
 881 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);
 882 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);
 883 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_done);
 884
 885 TRACE_EVENT(xprtrdma_frwr_alloc,
 886         TP_PROTO(
 887                 const struct rpcrdma_mr *mr,
 888                 int rc
 889         ),
 890
 891         TP_ARGS(mr, rc),
 892
 893         TP_STRUCT__entry(
 894                 __field(u32, mr_id)
 895                 __field(int, rc)
 896         ),
 897
 898         TP_fast_assign(
 899                 __entry->mr_id = mr->frwr.fr_mr->res.id;
 900                 __entry->rc = rc;
 901         ),
 902
 903         TP_printk("mr.id=%u: rc=%d",
 904                 __entry->mr_id, __entry->rc
 905         )
 906 );
 907
 908 TRACE_EVENT(xprtrdma_frwr_dereg,
 909         TP_PROTO(
 910                 const struct rpcrdma_mr *mr,
 911                 int rc
 912         ),
 913
 914         TP_ARGS(mr, rc),
 915
 916         TP_STRUCT__entry(
 917                 __field(u32, mr_id)
 918                 __field(int, nents)
 919                 __field(u32, handle)
 920                 __field(u32, length)
 921                 __field(u64, offset)
 922                 __field(u32, dir)
 923                 __field(int, rc)
 924         ),
 925
 926         TP_fast_assign(
 927                 __entry->mr_id  = mr->frwr.fr_mr->res.id;
 928                 __entry->nents  = mr->mr_nents;
 929                 __entry->handle = mr->mr_handle;
 930                 __entry->length = mr->mr_length;
 931                 __entry->offset = mr->mr_offset;
 932                 __entry->dir    = mr->mr_dir;
 933                 __entry->rc     = rc;
 934         ),
 935
 936         TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s): rc=%d",
 937                 __entry->mr_id, __entry->nents, __entry->length,
 938                 (unsigned long long)__entry->offset, __entry->handle,
 939                 xprtrdma_show_direction(__entry->dir),
 940                 __entry->rc
 941         )
 942 );
 943
 944 TRACE_EVENT(xprtrdma_frwr_sgerr,
 945         TP_PROTO(
 946                 const struct rpcrdma_mr *mr,
 947                 int sg_nents
 948         ),
 949
 950         TP_ARGS(mr, sg_nents),
 951
 952         TP_STRUCT__entry(
 953                 __field(u32, mr_id)
 954                 __field(u64, addr)
 955                 __field(u32, dir)
 956                 __field(int, nents)
 957         ),
 958
 959         TP_fast_assign(
 960                 __entry->mr_id = mr->frwr.fr_mr->res.id;
 961                 __entry->addr = mr->mr_sg->dma_address;
 962                 __entry->dir = mr->mr_dir;
 963                 __entry->nents = sg_nents;
 964         ),
 965
 966         TP_printk("mr.id=%u DMA addr=0x%llx (%s) sg_nents=%d",
 967                 __entry->mr_id, __entry->addr,
 968                 xprtrdma_show_direction(__entry->dir),
 969                 __entry->nents
 970         )
 971 );
 972
 973 TRACE_EVENT(xprtrdma_frwr_maperr,
 974         TP_PROTO(
 975                 const struct rpcrdma_mr *mr,
 976                 int num_mapped
 977         ),
 978
 979         TP_ARGS(mr, num_mapped),
 980
 981         TP_STRUCT__entry(
 982                 __field(u32, mr_id)
 983                 __field(u64, addr)
 984                 __field(u32, dir)
 985                 __field(int, num_mapped)
 986                 __field(int, nents)
 987         ),
 988
 989         TP_fast_assign(
 990                 __entry->mr_id = mr->frwr.fr_mr->res.id;
 991                 __entry->addr = mr->mr_sg->dma_address;
 992                 __entry->dir = mr->mr_dir;
 993                 __entry->num_mapped = num_mapped;
 994                 __entry->nents = mr->mr_nents;
 995         ),
 996
 997         TP_printk("mr.id=%u DMA addr=0x%llx (%s) nents=%d of %d",
 998                 __entry->mr_id, __entry->addr,
 999                 xprtrdma_show_direction(__entry->dir),
1000                 __entry->num_mapped, __entry->nents
1001         )
1002 );
1003
1004 DEFINE_MR_EVENT(localinv);
1005 DEFINE_MR_EVENT(map);
1006 DEFINE_MR_EVENT(unmap);
1007 DEFINE_MR_EVENT(reminv);
1008 DEFINE_MR_EVENT(recycle);
1009
1010 TRACE_EVENT(xprtrdma_dma_maperr,
1011         TP_PROTO(
1012                 u64 addr
1013         ),
1014
1015         TP_ARGS(addr),
1016
1017         TP_STRUCT__entry(
1018                 __field(u64, addr)
1019         ),
1020
1021         TP_fast_assign(
1022                 __entry->addr = addr;
1023         ),
1024
1025         TP_printk("dma addr=0x%llx\n", __entry->addr)
1026 );
1027
1028 /**
1029  ** Reply events
1030  **/
1031
1032 TRACE_EVENT(xprtrdma_reply,
1033         TP_PROTO(
1034                 const struct rpc_task *task,
1035                 const struct rpcrdma_rep *rep,
1036                 const struct rpcrdma_req *req,
1037                 unsigned int credits
1038         ),
1039
1040         TP_ARGS(task, rep, req, credits),
1041
1042         TP_STRUCT__entry(
1043                 __field(unsigned int, task_id)
1044                 __field(unsigned int, client_id)
1045                 __field(const void *, rep)
1046                 __field(const void *, req)
1047                 __field(u32, xid)
1048                 __field(unsigned int, credits)
1049         ),
1050
1051         TP_fast_assign(
1052                 __entry->task_id = task->tk_pid;
1053                 __entry->client_id = task->tk_client->cl_clid;
1054                 __entry->rep = rep;
1055                 __entry->req = req;
1056                 __entry->xid = be32_to_cpu(rep->rr_xid);
1057                 __entry->credits = credits;
1058         ),
1059
1060         TP_printk("task:%u@%u xid=0x%08x, %u credits, rep=%p -> req=%p",
1061                 __entry->task_id, __entry->client_id, __entry->xid,
1062                 __entry->credits, __entry->rep, __entry->req
1063         )
1064 );
1065
1066 TRACE_EVENT(xprtrdma_defer_cmp,
1067         TP_PROTO(
1068                 const struct rpcrdma_rep *rep
1069         ),
1070
1071         TP_ARGS(rep),
1072
1073         TP_STRUCT__entry(
1074                 __field(unsigned int, task_id)
1075                 __field(unsigned int, client_id)
1076                 __field(const void *, rep)
1077                 __field(u32, xid)
1078         ),
1079
1080         TP_fast_assign(
1081                 __entry->task_id = rep->rr_rqst->rq_task->tk_pid;
1082                 __entry->client_id = rep->rr_rqst->rq_task->tk_client->cl_clid;
1083                 __entry->rep = rep;
1084                 __entry->xid = be32_to_cpu(rep->rr_xid);
1085         ),
1086
1087         TP_printk("task:%u@%u xid=0x%08x rep=%p",
1088                 __entry->task_id, __entry->client_id, __entry->xid,
1089                 __entry->rep
1090         )
1091 );
1092
1093 DEFINE_REPLY_EVENT(xprtrdma_reply_vers);
1094 DEFINE_REPLY_EVENT(xprtrdma_reply_rqst);
1095 DEFINE_REPLY_EVENT(xprtrdma_reply_short);
1096 DEFINE_REPLY_EVENT(xprtrdma_reply_hdr);
1097
1098 TRACE_EVENT(xprtrdma_fixup,
1099         TP_PROTO(
1100                 const struct rpc_rqst *rqst,
1101                 unsigned long fixup
1102         ),
1103
1104         TP_ARGS(rqst, fixup),
1105
1106         TP_STRUCT__entry(
1107                 __field(unsigned int, task_id)
1108                 __field(unsigned int, client_id)
1109                 __field(unsigned long, fixup)
1110                 __field(size_t, headlen)
1111                 __field(unsigned int, pagelen)
1112                 __field(size_t, taillen)
1113         ),
1114
1115         TP_fast_assign(
1116                 __entry->task_id = rqst->rq_task->tk_pid;
1117                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1118                 __entry->fixup = fixup;
1119                 __entry->headlen = rqst->rq_rcv_buf.head[0].iov_len;
1120                 __entry->pagelen = rqst->rq_rcv_buf.page_len;
1121                 __entry->taillen = rqst->rq_rcv_buf.tail[0].iov_len;
1122         ),
1123
1124         TP_printk("task:%u@%u fixup=%lu xdr=%zu/%u/%zu",
1125                 __entry->task_id, __entry->client_id, __entry->fixup,
1126                 __entry->headlen, __entry->pagelen, __entry->taillen
1127         )
1128 );
1129
1130 TRACE_EVENT(xprtrdma_decode_seg,
1131         TP_PROTO(
1132                 u32 handle,
1133                 u32 length,
1134                 u64 offset
1135         ),
1136
1137         TP_ARGS(handle, length, offset),
1138
1139         TP_STRUCT__entry(
1140                 __field(u32, handle)
1141                 __field(u32, length)
1142                 __field(u64, offset)
1143         ),
1144
1145         TP_fast_assign(
1146                 __entry->handle = handle;
1147                 __entry->length = length;
1148                 __entry->offset = offset;
1149         ),
1150
1151         TP_printk("%u@0x%016llx:0x%08x",
1152                 __entry->length, (unsigned long long)__entry->offset,
1153                 __entry->handle
1154         )
1155 );
1156
1157 /**
1158  ** Allocation/release of rpcrdma_reqs and rpcrdma_reps
1159  **/
1160
1161 TRACE_EVENT(xprtrdma_op_allocate,
1162         TP_PROTO(
1163                 const struct rpc_task *task,
1164                 const struct rpcrdma_req *req
1165         ),
1166
1167         TP_ARGS(task, req),
1168
1169         TP_STRUCT__entry(
1170                 __field(unsigned int, task_id)
1171                 __field(unsigned int, client_id)
1172                 __field(const void *, req)
1173                 __field(size_t, callsize)
1174                 __field(size_t, rcvsize)
1175         ),
1176
1177         TP_fast_assign(
1178                 __entry->task_id = task->tk_pid;
1179                 __entry->client_id = task->tk_client->cl_clid;
1180                 __entry->req = req;
1181                 __entry->callsize = task->tk_rqstp->rq_callsize;
1182                 __entry->rcvsize = task->tk_rqstp->rq_rcvsize;
1183         ),
1184
1185         TP_printk("task:%u@%u req=%p (%zu, %zu)",
1186                 __entry->task_id, __entry->client_id,
1187                 __entry->req, __entry->callsize, __entry->rcvsize
1188         )
1189 );
1190
1191 TRACE_EVENT(xprtrdma_op_free,
1192         TP_PROTO(
1193                 const struct rpc_task *task,
1194                 const struct rpcrdma_req *req
1195         ),
1196
1197         TP_ARGS(task, req),
1198
1199         TP_STRUCT__entry(
1200                 __field(unsigned int, task_id)
1201                 __field(unsigned int, client_id)
1202                 __field(const void *, req)
1203                 __field(const void *, rep)
1204         ),
1205
1206         TP_fast_assign(
1207                 __entry->task_id = task->tk_pid;
1208                 __entry->client_id = task->tk_client->cl_clid;
1209                 __entry->req = req;
1210                 __entry->rep = req->rl_reply;
1211         ),
1212
1213         TP_printk("task:%u@%u req=%p rep=%p",
1214                 __entry->task_id, __entry->client_id,
1215                 __entry->req, __entry->rep
1216         )
1217 );
1218
1219 /**
1220  ** Callback events
1221  **/
1222
1223 TRACE_EVENT(xprtrdma_cb_setup,
1224         TP_PROTO(
1225                 const struct rpcrdma_xprt *r_xprt,
1226                 unsigned int reqs
1227         ),
1228
1229         TP_ARGS(r_xprt, reqs),
1230
1231         TP_STRUCT__entry(
1232                 __field(const void *, r_xprt)
1233                 __field(unsigned int, reqs)
1234                 __string(addr, rpcrdma_addrstr(r_xprt))
1235                 __string(port, rpcrdma_portstr(r_xprt))
1236         ),
1237
1238         TP_fast_assign(
1239                 __entry->r_xprt = r_xprt;
1240                 __entry->reqs = reqs;
1241                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
1242                 __assign_str(port, rpcrdma_portstr(r_xprt));
1243         ),
1244
1245         TP_printk("peer=[%s]:%s r_xprt=%p: %u reqs",
1246                 __get_str(addr), __get_str(port),
1247                 __entry->r_xprt, __entry->reqs
1248         )
1249 );
1250
1251 DEFINE_CB_EVENT(xprtrdma_cb_call);
1252 DEFINE_CB_EVENT(xprtrdma_cb_reply);
1253
1254 TRACE_EVENT(xprtrdma_leaked_rep,
1255         TP_PROTO(
1256                 const struct rpc_rqst *rqst,
1257                 const struct rpcrdma_rep *rep
1258         ),
1259
1260         TP_ARGS(rqst, rep),
1261
1262         TP_STRUCT__entry(
1263                 __field(unsigned int, task_id)
1264                 __field(unsigned int, client_id)
1265                 __field(u32, xid)
1266                 __field(const void *, rep)
1267         ),
1268
1269         TP_fast_assign(
1270                 __entry->task_id = rqst->rq_task->tk_pid;
1271                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1272                 __entry->xid = be32_to_cpu(rqst->rq_xid);
1273                 __entry->rep = rep;
1274         ),
1275
1276         TP_printk("task:%u@%u xid=0x%08x rep=%p",
1277                 __entry->task_id, __entry->client_id, __entry->xid,
1278                 __entry->rep
1279         )
1280 );
1281
1282 /**
1283  ** Server-side RPC/RDMA events
1284  **/
1285
1286 DECLARE_EVENT_CLASS(svcrdma_xprt_event,
1287         TP_PROTO(
1288                 const struct svc_xprt *xprt
1289         ),
1290
1291         TP_ARGS(xprt),
1292
1293         TP_STRUCT__entry(
1294                 __field(const void *, xprt)
1295                 __string(addr, xprt->xpt_remotebuf)
1296         ),
1297
1298         TP_fast_assign(
1299                 __entry->xprt = xprt;
1300                 __assign_str(addr, xprt->xpt_remotebuf);
1301         ),
1302
1303         TP_printk("xprt=%p addr=%s",
1304                 __entry->xprt, __get_str(addr)
1305         )
1306 );
1307
1308 #define DEFINE_XPRT_EVENT(name)                                         \
1309                 DEFINE_EVENT(svcrdma_xprt_event, svcrdma_xprt_##name,   \
1310                                 TP_PROTO(                               \
1311                                         const struct svc_xprt *xprt     \
1312                                 ),                                      \
1313                                 TP_ARGS(xprt))
1314
1315 DEFINE_XPRT_EVENT(accept);
1316 DEFINE_XPRT_EVENT(fail);
1317 DEFINE_XPRT_EVENT(free);
1318
1319 TRACE_DEFINE_ENUM(RDMA_MSG);
1320 TRACE_DEFINE_ENUM(RDMA_NOMSG);
1321 TRACE_DEFINE_ENUM(RDMA_MSGP);
1322 TRACE_DEFINE_ENUM(RDMA_DONE);
1323 TRACE_DEFINE_ENUM(RDMA_ERROR);
1324
1325 #define show_rpcrdma_proc(x)                                            \
1326                 __print_symbolic(x,                                     \
1327                                 { RDMA_MSG, "RDMA_MSG" },               \
1328                                 { RDMA_NOMSG, "RDMA_NOMSG" },           \
1329                                 { RDMA_MSGP, "RDMA_MSGP" },             \
1330                                 { RDMA_DONE, "RDMA_DONE" },             \
1331                                 { RDMA_ERROR, "RDMA_ERROR" })
1332
1333 TRACE_EVENT(svcrdma_decode_rqst,
1334         TP_PROTO(
1335                 __be32 *p,
1336                 unsigned int hdrlen
1337         ),
1338
1339         TP_ARGS(p, hdrlen),
1340
1341         TP_STRUCT__entry(
1342                 __field(u32, xid)
1343                 __field(u32, vers)
1344                 __field(u32, proc)
1345                 __field(u32, credits)
1346                 __field(unsigned int, hdrlen)
1347         ),
1348
1349         TP_fast_assign(
1350                 __entry->xid = be32_to_cpup(p++);
1351                 __entry->vers = be32_to_cpup(p++);
1352                 __entry->credits = be32_to_cpup(p++);
1353                 __entry->proc = be32_to_cpup(p);
1354                 __entry->hdrlen = hdrlen;
1355         ),
1356
1357         TP_printk("xid=0x%08x vers=%u credits=%u proc=%s hdrlen=%u",
1358                 __entry->xid, __entry->vers, __entry->credits,
1359                 show_rpcrdma_proc(__entry->proc), __entry->hdrlen)
1360 );
1361
1362 TRACE_EVENT(svcrdma_decode_short,
1363         TP_PROTO(
1364                 unsigned int hdrlen
1365         ),
1366
1367         TP_ARGS(hdrlen),
1368
1369         TP_STRUCT__entry(
1370                 __field(unsigned int, hdrlen)
1371         ),
1372
1373         TP_fast_assign(
1374                 __entry->hdrlen = hdrlen;
1375         ),
1376
1377         TP_printk("hdrlen=%u", __entry->hdrlen)
1378 );
1379
1380 DECLARE_EVENT_CLASS(svcrdma_badreq_event,
1381         TP_PROTO(
1382                 __be32 *p
1383         ),
1384
1385         TP_ARGS(p),
1386
1387         TP_STRUCT__entry(
1388                 __field(u32, xid)
1389                 __field(u32, vers)
1390                 __field(u32, proc)
1391                 __field(u32, credits)
1392         ),
1393
1394         TP_fast_assign(
1395                 __entry->xid = be32_to_cpup(p++);
1396                 __entry->vers = be32_to_cpup(p++);
1397                 __entry->credits = be32_to_cpup(p++);
1398                 __entry->proc = be32_to_cpup(p);
1399         ),
1400
1401         TP_printk("xid=0x%08x vers=%u credits=%u proc=%u",
1402                 __entry->xid, __entry->vers, __entry->credits, __entry->proc)
1403 );
1404
1405 #define DEFINE_BADREQ_EVENT(name)                                       \
1406                 DEFINE_EVENT(svcrdma_badreq_event, svcrdma_decode_##name,\
1407                                 TP_PROTO(                               \
1408                                         __be32 *p                       \
1409                                 ),                                      \
1410                                 TP_ARGS(p))
1411
1412 DEFINE_BADREQ_EVENT(badvers);
1413 DEFINE_BADREQ_EVENT(drop);
1414 DEFINE_BADREQ_EVENT(badproc);
1415 DEFINE_BADREQ_EVENT(parse);
1416
1417 DECLARE_EVENT_CLASS(svcrdma_segment_event,
1418         TP_PROTO(
1419                 u32 handle,
1420                 u32 length,
1421                 u64 offset
1422         ),
1423
1424         TP_ARGS(handle, length, offset),
1425
1426         TP_STRUCT__entry(
1427                 __field(u32, handle)
1428                 __field(u32, length)
1429                 __field(u64, offset)
1430         ),
1431
1432         TP_fast_assign(
1433                 __entry->handle = handle;
1434                 __entry->length = length;
1435                 __entry->offset = offset;
1436         ),
1437
1438         TP_printk("%u@0x%016llx:0x%08x",
1439                 __entry->length, (unsigned long long)__entry->offset,
1440                 __entry->handle
1441         )
1442 );
1443
1444 #define DEFINE_SEGMENT_EVENT(name)                                      \
1445                 DEFINE_EVENT(svcrdma_segment_event, svcrdma_##name,\
1446                                 TP_PROTO(                               \
1447                                         u32 handle,                     \
1448                                         u32 length,                     \
1449                                         u64 offset                      \
1450                                 ),                                      \
1451                                 TP_ARGS(handle, length, offset))
1452
1453 DEFINE_SEGMENT_EVENT(decode_wseg);
1454 DEFINE_SEGMENT_EVENT(encode_rseg);
1455 DEFINE_SEGMENT_EVENT(send_rseg);
1456 DEFINE_SEGMENT_EVENT(encode_wseg);
1457 DEFINE_SEGMENT_EVENT(send_wseg);
1458
1459 DECLARE_EVENT_CLASS(svcrdma_chunk_event,
1460         TP_PROTO(
1461                 u32 length
1462         ),
1463
1464         TP_ARGS(length),
1465
1466         TP_STRUCT__entry(
1467                 __field(u32, length)
1468         ),
1469
1470         TP_fast_assign(
1471                 __entry->length = length;
1472         ),
1473
1474         TP_printk("length=%u",
1475                 __entry->length
1476         )
1477 );
1478
1479 #define DEFINE_CHUNK_EVENT(name)                                        \
1480                 DEFINE_EVENT(svcrdma_chunk_event, svcrdma_##name,       \
1481                                 TP_PROTO(                               \
1482                                         u32 length                      \
1483                                 ),                                      \
1484                                 TP_ARGS(length))
1485
1486 DEFINE_CHUNK_EVENT(send_pzr);
1487 DEFINE_CHUNK_EVENT(encode_write_chunk);
1488 DEFINE_CHUNK_EVENT(send_write_chunk);
1489 DEFINE_CHUNK_EVENT(encode_read_chunk);
1490 DEFINE_CHUNK_EVENT(send_reply_chunk);
1491
1492 TRACE_EVENT(svcrdma_send_read_chunk,
1493         TP_PROTO(
1494                 u32 length,
1495                 u32 position
1496         ),
1497
1498         TP_ARGS(length, position),
1499
1500         TP_STRUCT__entry(
1501                 __field(u32, length)
1502                 __field(u32, position)
1503         ),
1504
1505         TP_fast_assign(
1506                 __entry->length = length;
1507                 __entry->position = position;
1508         ),
1509
1510         TP_printk("length=%u position=%u",
1511                 __entry->length, __entry->position
1512         )
1513 );
1514
1515 DECLARE_EVENT_CLASS(svcrdma_error_event,
1516         TP_PROTO(
1517                 __be32 xid
1518         ),
1519
1520         TP_ARGS(xid),
1521
1522         TP_STRUCT__entry(
1523                 __field(u32, xid)
1524         ),
1525
1526         TP_fast_assign(
1527                 __entry->xid = be32_to_cpu(xid);
1528         ),
1529
1530         TP_printk("xid=0x%08x",
1531                 __entry->xid
1532         )
1533 );
1534
1535 #define DEFINE_ERROR_EVENT(name)                                        \
1536                 DEFINE_EVENT(svcrdma_error_event, svcrdma_err_##name,   \
1537                                 TP_PROTO(                               \
1538                                         __be32 xid                      \
1539                                 ),                                      \
1540                                 TP_ARGS(xid))
1541
1542 DEFINE_ERROR_EVENT(vers);
1543 DEFINE_ERROR_EVENT(chunk);
1544
1545 /**
1546  ** Server-side RDMA API events
1547  **/
1548
1549 DECLARE_EVENT_CLASS(svcrdma_dma_map_class,
1550         TP_PROTO(
1551                 const struct svcxprt_rdma *rdma,
1552                 u64 dma_addr,
1553                 u32 length
1554         ),
1555
1556         TP_ARGS(rdma, dma_addr, length),
1557
1558         TP_STRUCT__entry(
1559                 __field(u64, dma_addr)
1560                 __field(u32, length)
1561                 __string(device, rdma->sc_cm_id->device->name)
1562                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1563         ),
1564
1565         TP_fast_assign(
1566                 __entry->dma_addr = dma_addr;
1567                 __entry->length = length;
1568                 __assign_str(device, rdma->sc_cm_id->device->name);
1569                 __assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1570         ),
1571
1572         TP_printk("addr=%s device=%s dma_addr=%llu length=%u",
1573                 __get_str(addr), __get_str(device),
1574                 __entry->dma_addr, __entry->length
1575         )
1576 );
1577
1578 #define DEFINE_SVC_DMA_EVENT(name)                                      \
1579                 DEFINE_EVENT(svcrdma_dma_map_class, svcrdma_##name,     \
1580                                 TP_PROTO(                               \
1581                                         const struct svcxprt_rdma *rdma,\
1582                                         u64 dma_addr,                   \
1583                                         u32 length                      \
1584                                 ),                                      \
1585                                 TP_ARGS(rdma, dma_addr, length))
1586
1587 DEFINE_SVC_DMA_EVENT(dma_map_page);
1588 DEFINE_SVC_DMA_EVENT(dma_unmap_page);
1589
1590 TRACE_EVENT(svcrdma_dma_map_rwctx,
1591         TP_PROTO(
1592                 const struct svcxprt_rdma *rdma,
1593                 int status
1594         ),
1595
1596         TP_ARGS(rdma, status),
1597
1598         TP_STRUCT__entry(
1599                 __field(int, status)
1600                 __string(device, rdma->sc_cm_id->device->name)
1601                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1602         ),
1603
1604         TP_fast_assign(
1605                 __entry->status = status;
1606                 __assign_str(device, rdma->sc_cm_id->device->name);
1607                 __assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1608         ),
1609
1610         TP_printk("addr=%s device=%s status=%d",
1611                 __get_str(addr), __get_str(device), __entry->status
1612         )
1613 );
1614
1615 TRACE_EVENT(svcrdma_send_pullup,
1616         TP_PROTO(
1617                 unsigned int len
1618         ),
1619
1620         TP_ARGS(len),
1621
1622         TP_STRUCT__entry(
1623                 __field(unsigned int, len)
1624         ),
1625
1626         TP_fast_assign(
1627                 __entry->len = len;
1628         ),
1629
1630         TP_printk("len=%u", __entry->len)
1631 );
1632
1633 TRACE_EVENT(svcrdma_send_failed,
1634         TP_PROTO(
1635                 const struct svc_rqst *rqst,
1636                 int status
1637         ),
1638
1639         TP_ARGS(rqst, status),
1640
1641         TP_STRUCT__entry(
1642                 __field(int, status)
1643                 __field(u32, xid)
1644                 __field(const void *, xprt)
1645                 __string(addr, rqst->rq_xprt->xpt_remotebuf)
1646         ),
1647
1648         TP_fast_assign(
1649                 __entry->status = status;
1650                 __entry->xid = __be32_to_cpu(rqst->rq_xid);
1651                 __entry->xprt = rqst->rq_xprt;
1652                 __assign_str(addr, rqst->rq_xprt->xpt_remotebuf);
1653         ),
1654
1655         TP_printk("xprt=%p addr=%s xid=0x%08x status=%d",
1656                 __entry->xprt, __get_str(addr),
1657                 __entry->xid, __entry->status
1658         )
1659 );
1660
1661 DECLARE_EVENT_CLASS(svcrdma_sendcomp_event,
1662         TP_PROTO(
1663                 const struct ib_wc *wc
1664         ),
1665
1666         TP_ARGS(wc),
1667
1668         TP_STRUCT__entry(
1669                 __field(const void *, cqe)
1670                 __field(unsigned int, status)
1671                 __field(unsigned int, vendor_err)
1672         ),
1673
1674         TP_fast_assign(
1675                 __entry->cqe = wc->wr_cqe;
1676                 __entry->status = wc->status;
1677                 if (wc->status)
1678                         __entry->vendor_err = wc->vendor_err;
1679                 else
1680                         __entry->vendor_err = 0;
1681         ),
1682
1683         TP_printk("cqe=%p status=%s (%u/0x%x)",
1684                 __entry->cqe, rdma_show_wc_status(__entry->status),
1685                 __entry->status, __entry->vendor_err
1686         )
1687 );
1688
1689 #define DEFINE_SENDCOMP_EVENT(name)                                     \
1690                 DEFINE_EVENT(svcrdma_sendcomp_event, svcrdma_wc_##name, \
1691                                 TP_PROTO(                               \
1692                                         const struct ib_wc *wc          \
1693                                 ),                                      \
1694                                 TP_ARGS(wc))
1695
1696 TRACE_EVENT(svcrdma_post_send,
1697         TP_PROTO(
1698                 const struct ib_send_wr *wr,
1699                 int status
1700         ),
1701
1702         TP_ARGS(wr, status),
1703
1704         TP_STRUCT__entry(
1705                 __field(const void *, cqe)
1706                 __field(unsigned int, num_sge)
1707                 __field(u32, inv_rkey)
1708                 __field(int, status)
1709         ),
1710
1711         TP_fast_assign(
1712                 __entry->cqe = wr->wr_cqe;
1713                 __entry->num_sge = wr->num_sge;
1714                 __entry->inv_rkey = (wr->opcode == IB_WR_SEND_WITH_INV) ?
1715                                         wr->ex.invalidate_rkey : 0;
1716                 __entry->status = status;
1717         ),
1718
1719         TP_printk("cqe=%p num_sge=%u inv_rkey=0x%08x status=%d",
1720                 __entry->cqe, __entry->num_sge,
1721                 __entry->inv_rkey, __entry->status
1722         )
1723 );
1724
1725 DEFINE_SENDCOMP_EVENT(send);
1726
1727 TRACE_EVENT(svcrdma_post_recv,
1728         TP_PROTO(
1729                 const struct ib_recv_wr *wr,
1730                 int status
1731         ),
1732
1733         TP_ARGS(wr, status),
1734
1735         TP_STRUCT__entry(
1736                 __field(const void *, cqe)
1737                 __field(int, status)
1738         ),
1739
1740         TP_fast_assign(
1741                 __entry->cqe = wr->wr_cqe;
1742                 __entry->status = status;
1743         ),
1744
1745         TP_printk("cqe=%p status=%d",
1746                 __entry->cqe, __entry->status
1747         )
1748 );
1749
1750 TRACE_EVENT(svcrdma_wc_receive,
1751         TP_PROTO(
1752                 const struct ib_wc *wc
1753         ),
1754
1755         TP_ARGS(wc),
1756
1757         TP_STRUCT__entry(
1758                 __field(const void *, cqe)
1759                 __field(u32, byte_len)
1760                 __field(unsigned int, status)
1761                 __field(u32, vendor_err)
1762         ),
1763
1764         TP_fast_assign(
1765                 __entry->cqe = wc->wr_cqe;
1766                 __entry->status = wc->status;
1767                 if (wc->status) {
1768                         __entry->byte_len = 0;
1769                         __entry->vendor_err = wc->vendor_err;
1770                 } else {
1771                         __entry->byte_len = wc->byte_len;
1772                         __entry->vendor_err = 0;
1773                 }
1774         ),
1775
1776         TP_printk("cqe=%p byte_len=%u status=%s (%u/0x%x)",
1777                 __entry->cqe, __entry->byte_len,
1778                 rdma_show_wc_status(__entry->status),
1779                 __entry->status, __entry->vendor_err
1780         )
1781 );
1782
1783 TRACE_EVENT(svcrdma_post_rw,
1784         TP_PROTO(
1785                 const void *cqe,
1786                 int sqecount,
1787                 int status
1788         ),
1789
1790         TP_ARGS(cqe, sqecount, status),
1791
1792         TP_STRUCT__entry(
1793                 __field(const void *, cqe)
1794                 __field(int, sqecount)
1795                 __field(int, status)
1796         ),
1797
1798         TP_fast_assign(
1799                 __entry->cqe = cqe;
1800                 __entry->sqecount = sqecount;
1801                 __entry->status = status;
1802         ),
1803
1804         TP_printk("cqe=%p sqecount=%d status=%d",
1805                 __entry->cqe, __entry->sqecount, __entry->status
1806         )
1807 );
1808
1809 DEFINE_SENDCOMP_EVENT(read);
1810 DEFINE_SENDCOMP_EVENT(write);
1811
1812 TRACE_EVENT(svcrdma_qp_error,
1813         TP_PROTO(
1814                 const struct ib_event *event,
1815                 const struct sockaddr *sap
1816         ),
1817
1818         TP_ARGS(event, sap),
1819
1820         TP_STRUCT__entry(
1821                 __field(unsigned int, event)
1822                 __string(device, event->device->name)
1823                 __array(__u8, addr, INET6_ADDRSTRLEN + 10)
1824         ),
1825
1826         TP_fast_assign(
1827                 __entry->event = event->event;
1828                 __assign_str(device, event->device->name);
1829                 snprintf(__entry->addr, sizeof(__entry->addr) - 1,
1830                          "%pISpc", sap);
1831         ),
1832
1833         TP_printk("addr=%s dev=%s event=%s (%u)",
1834                 __entry->addr, __get_str(device),
1835                 rdma_show_ib_event(__entry->event), __entry->event
1836         )
1837 );
1838
1839 DECLARE_EVENT_CLASS(svcrdma_sendqueue_event,
1840         TP_PROTO(
1841                 const struct svcxprt_rdma *rdma
1842         ),
1843
1844         TP_ARGS(rdma),
1845
1846         TP_STRUCT__entry(
1847                 __field(int, avail)
1848                 __field(int, depth)
1849                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1850         ),
1851
1852         TP_fast_assign(
1853                 __entry->avail = atomic_read(&rdma->sc_sq_avail);
1854                 __entry->depth = rdma->sc_sq_depth;
1855                 __assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1856         ),
1857
1858         TP_printk("addr=%s sc_sq_avail=%d/%d",
1859                 __get_str(addr), __entry->avail, __entry->depth
1860         )
1861 );
1862
1863 #define DEFINE_SQ_EVENT(name)                                           \
1864                 DEFINE_EVENT(svcrdma_sendqueue_event, svcrdma_sq_##name,\
1865                                 TP_PROTO(                               \
1866                                         const struct svcxprt_rdma *rdma \
1867                                 ),                                      \
1868                                 TP_ARGS(rdma))
1869
1870 DEFINE_SQ_EVENT(full);
1871 DEFINE_SQ_EVENT(retry);
1872
1873 #endif /* _TRACE_RPCRDMA_H */
1874
1875 #include <trace/define_trace.h>