IB/hfi1: Add TID RDMA handlers

This commit adds the TID RDMA READ pointers to the receiving opcode
handlers. It also adds TID RDMA READ header sizes to header size table.
A function to print the RHF EFLAGS errors is created so that it can be
shared by both IB and TID RDMA receiving functions.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Mitko Haralanov <mitko.haralanov@intel.com>
Signed-off-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index a8ad707..2a9d291 100644 (file)
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -1575,25 +1575,32 @@ drop:
        return -EINVAL;
 }
 
-void handle_eflags(struct hfi1_packet *packet)
+static void show_eflags_errs(struct hfi1_packet *packet)
 {
        struct hfi1_ctxtdata *rcd = packet->rcd;
        u32 rte = rhf_rcv_type_err(packet->rhf);
 
+       dd_dev_err(rcd->dd,
+                  "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
+                  rcd->ctxt, packet->rhf,
+                  packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
+                  packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
+                  packet->rhf & RHF_DC_ERR ? "dc " : "",
+                  packet->rhf & RHF_TID_ERR ? "tid " : "",
+                  packet->rhf & RHF_LEN_ERR ? "len " : "",
+                  packet->rhf & RHF_ECC_ERR ? "ecc " : "",
+                  packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
+                  packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
+                  rte);
+}
+
+void handle_eflags(struct hfi1_packet *packet)
+{
+       struct hfi1_ctxtdata *rcd = packet->rcd;
+
        rcv_hdrerr(rcd, rcd->ppd, packet);
        if (rhf_err_flags(packet->rhf))
-               dd_dev_err(rcd->dd,
-                          "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
-                          rcd->ctxt, packet->rhf,
-                          packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
-                          packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
-                          packet->rhf & RHF_DC_ERR ? "dc " : "",
-                          packet->rhf & RHF_TID_ERR ? "tid " : "",
-                          packet->rhf & RHF_LEN_ERR ? "len " : "",
-                          packet->rhf & RHF_ECC_ERR ? "ecc " : "",
-                          packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
-                          packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
-                          rte);
+               show_eflags_errs(packet);
 }
 
 /*
@@ -1699,11 +1706,14 @@ static int kdeth_process_expected(struct hfi1_packet *packet)
        if (unlikely(hfi1_dbg_should_fault_rx(packet)))
                return RHF_RCV_CONTINUE;
 
-       if (unlikely(rhf_err_flags(packet->rhf)))
-               handle_eflags(packet);
+       if (unlikely(rhf_err_flags(packet->rhf))) {
+               struct hfi1_ctxtdata *rcd = packet->rcd;
 
-       dd_dev_err(packet->rcd->dd,
-                  "Unhandled expected packet received. Dropping.\n");
+               if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
+                       return RHF_RCV_CONTINUE;
+       }
+
+       hfi1_kdeth_expected_rcv(packet);
        return RHF_RCV_CONTINUE;
 }
 
@@ -1712,11 +1722,17 @@ static int kdeth_process_eager(struct hfi1_packet *packet)
        hfi1_setup_9B_packet(packet);
        if (unlikely(hfi1_dbg_should_fault_rx(packet)))
                return RHF_RCV_CONTINUE;
-       if (unlikely(rhf_err_flags(packet->rhf)))
-               handle_eflags(packet);
 
-       dd_dev_err(packet->rcd->dd,
-                  "Unhandled eager packet received. Dropping.\n");
+       trace_hfi1_rcvhdr(packet);
+       if (unlikely(rhf_err_flags(packet->rhf))) {
+               struct hfi1_ctxtdata *rcd = packet->rcd;
+
+               show_eflags_errs(packet);
+               if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
+                       return RHF_RCV_CONTINUE;
+       }
+
+       hfi1_kdeth_eager_rcv(packet);
        return RHF_RCV_CONTINUE;
 }
 

diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index 1412ed1..6582184 100644 (file)
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -2120,7 +2120,7 @@ static inline u64 hfi1_pkt_default_send_ctxt_mask(struct hfi1_devdata *dd,
                        SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_TEST_SMASK |
 #endif
                        HFI1_PKT_USER_SC_INTEGRITY;
-       else
+       else if (ctxt_type != SC_KERNEL)
                base_sc_integrity |= HFI1_PKT_KERNEL_SC_INTEGRITY;
 
        /* turn on send-side job key checks if !A0 */

diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 8887a71..2d59fcd 100644 (file)
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -200,6 +200,8 @@ const u8 hdr_len_by_opcode[256] = {
        [IB_OPCODE_RC_FETCH_ADD]                      = 12 + 8 + 28,
        [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE]      = 12 + 8 + 4,
        [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE]      = 12 + 8 + 4,
+       [IB_OPCODE_TID_RDMA_READ_REQ]                 = 12 + 8 + 36,
+       [IB_OPCODE_TID_RDMA_READ_RESP]                = 12 + 8 + 36,
        /* UC */
        [IB_OPCODE_UC_SEND_FIRST]                     = 12 + 8,
        [IB_OPCODE_UC_SEND_MIDDLE]                    = 12 + 8,
@@ -243,6 +245,11 @@ static const opcode_handler opcode_handler_tbl[256] = {
        [IB_OPCODE_RC_FETCH_ADD]                      = &hfi1_rc_rcv,
        [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE]      = &hfi1_rc_rcv,
        [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE]      = &hfi1_rc_rcv,
+
+       /* TID RDMA has separate handlers for different opcodes.*/
+       [IB_OPCODE_TID_RDMA_READ_REQ]        = &hfi1_rc_rcv_tid_rdma_read_req,
+       [IB_OPCODE_TID_RDMA_READ_RESP]       = &hfi1_rc_rcv_tid_rdma_read_resp,
+
        /* UC */
        [IB_OPCODE_UC_SEND_FIRST]                     = &hfi1_uc_rcv,
        [IB_OPCODE_UC_SEND_MIDDLE]                    = &hfi1_uc_rcv,
@@ -336,6 +343,124 @@ static u64 hfi1_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc)
        return pbc;
 }
 
+static opcode_handler tid_qp_ok(int opcode, struct hfi1_packet *packet)
+{
+       if (packet->qp->ibqp.qp_type != IB_QPT_RC ||
+           !(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
+               return NULL;
+       if ((opcode & RVT_OPCODE_QP_MASK) == IB_OPCODE_TID_RDMA)
+               return opcode_handler_tbl[opcode];
+       return NULL;
+}
+
+void hfi1_kdeth_eager_rcv(struct hfi1_packet *packet)
+{
+       struct hfi1_ctxtdata *rcd = packet->rcd;
+       struct ib_header *hdr = packet->hdr;
+       u32 tlen = packet->tlen;
+       struct hfi1_pportdata *ppd = rcd->ppd;
+       struct hfi1_ibport *ibp = &ppd->ibport_data;
+       struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;
+       opcode_handler opcode_handler;
+       unsigned long flags;
+       u32 qp_num;
+       int lnh;
+       u8 opcode;
+
+       /* DW == LRH (2) + BTH (3) + KDETH (9) + CRC (1) */
+       if (unlikely(tlen < 15 * sizeof(u32)))
+               goto drop;
+
+       lnh = be16_to_cpu(hdr->lrh[0]) & 3;
+       if (lnh != HFI1_LRH_BTH)
+               goto drop;
+
+       packet->ohdr = &hdr->u.oth;
+       trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));
+
+       opcode = (be32_to_cpu(packet->ohdr->bth[0]) >> 24);
+       inc_opstats(tlen, &rcd->opstats->stats[opcode]);
+
+       /* verbs_qp can be picked up from any tid_rdma header struct */
+       qp_num = be32_to_cpu(packet->ohdr->u.tid_rdma.r_req.verbs_qp) &
+               RVT_QPN_MASK;
+
+       rcu_read_lock();
+       packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
+       if (!packet->qp)
+               goto drop_rcu;
+       spin_lock_irqsave(&packet->qp->r_lock, flags);
+       opcode_handler = tid_qp_ok(opcode, packet);
+       if (likely(opcode_handler))
+               opcode_handler(packet);
+       else
+               goto drop_unlock;
+       spin_unlock_irqrestore(&packet->qp->r_lock, flags);
+       rcu_read_unlock();
+
+       return;
+drop_unlock:
+       spin_unlock_irqrestore(&packet->qp->r_lock, flags);
+drop_rcu:
+       rcu_read_unlock();
+drop:
+       ibp->rvp.n_pkt_drops++;
+}
+
+void hfi1_kdeth_expected_rcv(struct hfi1_packet *packet)
+{
+       struct hfi1_ctxtdata *rcd = packet->rcd;
+       struct ib_header *hdr = packet->hdr;
+       u32 tlen = packet->tlen;
+       struct hfi1_pportdata *ppd = rcd->ppd;
+       struct hfi1_ibport *ibp = &ppd->ibport_data;
+       struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;
+       opcode_handler opcode_handler;
+       unsigned long flags;
+       u32 qp_num;
+       int lnh;
+       u8 opcode;
+
+       /* DW == LRH (2) + BTH (3) + KDETH (9) + CRC (1) */
+       if (unlikely(tlen < 15 * sizeof(u32)))
+               goto drop;
+
+       lnh = be16_to_cpu(hdr->lrh[0]) & 3;
+       if (lnh != HFI1_LRH_BTH)
+               goto drop;
+
+       packet->ohdr = &hdr->u.oth;
+       trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));
+
+       opcode = (be32_to_cpu(packet->ohdr->bth[0]) >> 24);
+       inc_opstats(tlen, &rcd->opstats->stats[opcode]);
+
+       /* verbs_qp can be picked up from any tid_rdma header struct */
+       qp_num = be32_to_cpu(packet->ohdr->u.tid_rdma.r_rsp.verbs_qp) &
+               RVT_QPN_MASK;
+
+       rcu_read_lock();
+       packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
+       if (!packet->qp)
+               goto drop_rcu;
+       spin_lock_irqsave(&packet->qp->r_lock, flags);
+       opcode_handler = tid_qp_ok(opcode, packet);
+       if (likely(opcode_handler))
+               opcode_handler(packet);
+       else
+               goto drop_unlock;
+       spin_unlock_irqrestore(&packet->qp->r_lock, flags);
+       rcu_read_unlock();
+
+       return;
+drop_unlock:
+       spin_unlock_irqrestore(&packet->qp->r_lock, flags);
+drop_rcu:
+       rcu_read_unlock();
+drop:
+       ibp->rvp.n_pkt_drops++;
+}
+
 static int hfi1_do_pkey_check(struct hfi1_packet *packet)
 {
        struct hfi1_ctxtdata *rcd = packet->rcd;

diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index c1574c0..7642b59 100644 (file)
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -427,6 +427,10 @@ int hfi1_register_ib_device(struct hfi1_devdata *);
 
 void hfi1_unregister_ib_device(struct hfi1_devdata *);
 
+void hfi1_kdeth_eager_rcv(struct hfi1_packet *packet);
+
+void hfi1_kdeth_expected_rcv(struct hfi1_packet *packet);
+
 void hfi1_ib_rcv(struct hfi1_packet *packet);
 
 void hfi1_16B_rcv(struct hfi1_packet *packet);