xprtrdma: Use gathered Send for large inline messages

[uclinux-h8/linux.git] / net / sunrpc / xprtrdma / xprt_rdma.h

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index decd134..b2823d9 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -70,9 +70,11 @@ struct rpcrdma_ia {
        struct ib_pd            *ri_pd;
        struct completion       ri_done;
        int                     ri_async_rc;
+       unsigned int            ri_max_segs;
        unsigned int            ri_max_frmr_depth;
        unsigned int            ri_max_inline_write;
        unsigned int            ri_max_inline_read;
+       bool                    ri_reminv_expected;
        struct ib_qp_attr       ri_qp_attr;
        struct ib_qp_init_attr  ri_qp_init_attr;
 };
@@ -87,6 +89,7 @@ struct rpcrdma_ep {
        int                     rep_connected;
        struct ib_qp_init_attr  rep_attr;
        wait_queue_head_t       rep_connect_wait;
+       struct rpcrdma_connect_private  rep_cm_private;
        struct rdma_conn_param  rep_remote_cma;
        struct sockaddr_storage rep_remote_addr;
        struct delayed_work     rep_connect_worker;
@@ -185,10 +188,13 @@ enum {
 struct rpcrdma_rep {
        struct ib_cqe           rr_cqe;
        unsigned int            rr_len;
+       int                     rr_wc_flags;
+       u32                     rr_inv_rkey;
        struct ib_device        *rr_device;
        struct rpcrdma_xprt     *rr_rxprt;
        struct work_struct      rr_work;
        struct list_head        rr_list;
+       struct ib_recv_wr       rr_recv_wr;
        struct rpcrdma_regbuf   *rr_rdmabuf;
 };
 
@@ -279,16 +285,27 @@ struct rpcrdma_mr_seg {           /* chunk descriptors */
        char            *mr_offset;     /* kva if no page, else offset */
 };
 
-#define RPCRDMA_MAX_IOVS       (2)
+/* Reserve enough Send SGEs to send a maximum size inline request:
+ * - RPC-over-RDMA header
+ * - xdr_buf head iovec
+ * - RPCRDMA_MAX_INLINE bytes, possibly unaligned, in pages
+ * - xdr_buf tail iovec
+ */
+enum {
+       RPCRDMA_MAX_SEND_PAGES = PAGE_SIZE + RPCRDMA_MAX_INLINE - 1,
+       RPCRDMA_MAX_PAGE_SGES = (RPCRDMA_MAX_SEND_PAGES >> PAGE_SHIFT) + 1,
+       RPCRDMA_MAX_SEND_SGES = 1 + 1 + RPCRDMA_MAX_PAGE_SGES + 1,
+};
 
 struct rpcrdma_buffer;
 struct rpcrdma_req {
        struct list_head        rl_free;
-       unsigned int            rl_niovs;
+       unsigned int            rl_mapped_sges;
        unsigned int            rl_connect_cookie;
        struct rpcrdma_buffer   *rl_buffer;
-       struct rpcrdma_rep      *rl_reply;/* holder for reply buffer */
-       struct ib_sge           rl_send_iov[RPCRDMA_MAX_IOVS];
+       struct rpcrdma_rep      *rl_reply;
+       struct ib_send_wr       rl_send_wr;
+       struct ib_sge           rl_send_sge[RPCRDMA_MAX_SEND_SGES];
        struct rpcrdma_regbuf   *rl_rdmabuf;    /* xprt header */
        struct rpcrdma_regbuf   *rl_sendbuf;    /* rq_snd_buf */
        struct rpcrdma_regbuf   *rl_recvbuf;    /* rq_rcv_buf */
@@ -382,6 +399,7 @@ struct rpcrdma_stats {
        unsigned long           mrs_recovered;
        unsigned long           mrs_orphaned;
        unsigned long           mrs_allocated;
+       unsigned long           local_inv_needed;
 };
 
 /*
@@ -405,6 +423,7 @@ struct rpcrdma_memreg_ops {
                                      struct rpcrdma_mw *);
        void            (*ro_release_mr)(struct rpcrdma_mw *);
        const char      *ro_displayname;
+       const int       ro_send_w_inv_ok;
 };
 
 extern const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops;
@@ -521,10 +540,20 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *);
 /*
  * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c
  */
+
+enum rpcrdma_chunktype {
+       rpcrdma_noch = 0,
+       rpcrdma_readch,
+       rpcrdma_areadch,
+       rpcrdma_writech,
+       rpcrdma_replych
+};
+
+bool rpcrdma_prepare_send_sges(struct rpcrdma_ia *, struct rpcrdma_req *,
+                              u32, struct xdr_buf *, enum rpcrdma_chunktype);
+void rpcrdma_unmap_sges(struct rpcrdma_ia *, struct rpcrdma_req *);
 int rpcrdma_marshal_req(struct rpc_rqst *);
-void rpcrdma_set_max_header_sizes(struct rpcrdma_ia *,
-                                 struct rpcrdma_create_data_internal *,
-                                 unsigned int);
+void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
 
 /* RPC/RDMA module init - xprtrdma/transport.c
  */