drivers/infiniband/ulp/isert/ib_isert.c

   1 /*******************************************************************************
   2  * This file contains iSCSI extentions for RDMA (iSER) Verbs
   3  *
   4  * (c) Copyright 2013 Datera, Inc.
   5  *
   6  * Nicholas A. Bellinger <nab@linux-iscsi.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  ****************************************************************************/
  18
  19 #include <linux/string.h>
  20 #include <linux/module.h>
  21 #include <linux/scatterlist.h>
  22 #include <linux/socket.h>
  23 #include <linux/in.h>
  24 #include <linux/in6.h>
  25 #include <rdma/ib_verbs.h>
  26 #include <rdma/rdma_cm.h>
  27 #include <target/target_core_base.h>
  28 #include <target/target_core_fabric.h>
  29 #include <target/iscsi/iscsi_transport.h>
  30 #include <linux/semaphore.h>
  31
  32 #include "isert_proto.h"
  33 #include "ib_isert.h"
  34
  35 #define ISERT_MAX_CONN          8
  36 #define ISER_MAX_RX_CQ_LEN      (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
  37 #define ISER_MAX_TX_CQ_LEN      (ISERT_QP_MAX_REQ_DTOS  * ISERT_MAX_CONN)
  38 #define ISER_MAX_CQ_LEN         (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
  39                                  ISERT_MAX_CONN)
  40
  41 static DEFINE_MUTEX(device_list_mutex);
  42 static LIST_HEAD(device_list);
  43 static struct workqueue_struct *isert_comp_wq;
  44 static struct workqueue_struct *isert_release_wq;
  45
  46 static void
  47 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
  48 static int
  49 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  50                struct isert_rdma_wr *wr);
  51 static void
  52 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
  53 static int
  54 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  55                struct isert_rdma_wr *wr);
  56 static int
  57 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
  58 static int
  59 isert_rdma_post_recvl(struct isert_conn *isert_conn);
  60 static int
  61 isert_rdma_accept(struct isert_conn *isert_conn);
  62 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
  63
  64 static inline bool
  65 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
  66 {
  67         return (conn->pi_support &&
  68                 cmd->prot_op != TARGET_PROT_NORMAL);
  69 }
  70
  71
  72 static void
  73 isert_qp_event_callback(struct ib_event *e, void *context)
  74 {
  75         struct isert_conn *isert_conn = (struct isert_conn *)context;
  76
  77         pr_err("isert_qp_event_callback event: %d\n", e->event);
  78         switch (e->event) {
  79         case IB_EVENT_COMM_EST:
  80                 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST);
  81                 break;
  82         case IB_EVENT_QP_LAST_WQE_REACHED:
  83                 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
  84                 break;
  85         default:
  86                 break;
  87         }
  88 }
  89
  90 static int
  91 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
  92 {
  93         int ret;
  94
  95         ret = ib_query_device(ib_dev, devattr);
  96         if (ret) {
  97                 pr_err("ib_query_device() failed: %d\n", ret);
  98                 return ret;
  99         }
 100         pr_debug("devattr->max_sge: %d\n", devattr->max_sge);
 101         pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
 102
 103         return 0;
 104 }
 105
 106 static int
 107 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
 108 {
 109         struct isert_device *device = isert_conn->conn_device;
 110         struct ib_qp_init_attr attr;
 111         struct isert_comp *comp;
 112         int ret, i, min = 0;
 113
 114         mutex_lock(&device_list_mutex);
 115         for (i = 0; i < device->comps_used; i++)
 116                 if (device->comps[i].active_qps <
 117                     device->comps[min].active_qps)
 118                         min = i;
 119         comp = &device->comps[min];
 120         comp->active_qps++;
 121         pr_info("conn %p, using comp %p min_index: %d\n",
 122                    isert_conn, comp, min);
 123         mutex_unlock(&device_list_mutex);
 124
 125         memset(&attr, 0, sizeof(struct ib_qp_init_attr));
 126         attr.event_handler = isert_qp_event_callback;
 127         attr.qp_context = isert_conn;
 128         attr.send_cq = comp->cq;
 129         attr.recv_cq = comp->cq;
 130         attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
 131         attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
 132         /*
 133          * FIXME: Use devattr.max_sge - 2 for max_send_sge as
 134          * work-around for RDMA_READs with ConnectX-2.
 135          *
 136          * Also, still make sure to have at least two SGEs for
 137          * outgoing control PDU responses.
 138          */
 139         attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2);
 140         isert_conn->max_sge = attr.cap.max_send_sge;
 141
 142         attr.cap.max_recv_sge = 1;
 143         attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 144         attr.qp_type = IB_QPT_RC;
 145         if (device->pi_capable)
 146                 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
 147
 148         pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
 149                  cma_id->device);
 150         pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
 151                  isert_conn->conn_pd->device);
 152
 153         ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr);
 154         if (ret) {
 155                 pr_err("rdma_create_qp failed for cma_id %d\n", ret);
 156                 goto err;
 157         }
 158         isert_conn->conn_qp = cma_id->qp;
 159         pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");
 160
 161         return 0;
 162 err:
 163         mutex_lock(&device_list_mutex);
 164         comp->active_qps--;
 165         mutex_unlock(&device_list_mutex);
 166
 167         return ret;
 168 }
 169
 170 static void
 171 isert_cq_event_callback(struct ib_event *e, void *context)
 172 {
 173         pr_debug("isert_cq_event_callback event: %d\n", e->event);
 174 }
 175
 176 static int
 177 isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
 178 {
 179         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 180         struct iser_rx_desc *rx_desc;
 181         struct ib_sge *rx_sg;
 182         u64 dma_addr;
 183         int i, j;
 184
 185         isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
 186                                 sizeof(struct iser_rx_desc), GFP_KERNEL);
 187         if (!isert_conn->conn_rx_descs)
 188                 goto fail;
 189
 190         rx_desc = isert_conn->conn_rx_descs;
 191
 192         for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
 193                 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
 194                                         ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
 195                 if (ib_dma_mapping_error(ib_dev, dma_addr))
 196                         goto dma_map_fail;
 197
 198                 rx_desc->dma_addr = dma_addr;
 199
 200                 rx_sg = &rx_desc->rx_sg;
 201                 rx_sg->addr = rx_desc->dma_addr;
 202                 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
 203                 rx_sg->lkey = isert_conn->conn_mr->lkey;
 204         }
 205
 206         isert_conn->conn_rx_desc_head = 0;
 207         return 0;
 208
 209 dma_map_fail:
 210         rx_desc = isert_conn->conn_rx_descs;
 211         for (j = 0; j < i; j++, rx_desc++) {
 212                 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
 213                                     ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
 214         }
 215         kfree(isert_conn->conn_rx_descs);
 216         isert_conn->conn_rx_descs = NULL;
 217 fail:
 218         return -ENOMEM;
 219 }
 220
 221 static void
 222 isert_free_rx_descriptors(struct isert_conn *isert_conn)
 223 {
 224         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 225         struct iser_rx_desc *rx_desc;
 226         int i;
 227
 228         if (!isert_conn->conn_rx_descs)
 229                 return;
 230
 231         rx_desc = isert_conn->conn_rx_descs;
 232         for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
 233                 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
 234                                     ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
 235         }
 236
 237         kfree(isert_conn->conn_rx_descs);
 238         isert_conn->conn_rx_descs = NULL;
 239 }
 240
 241 static void isert_cq_work(struct work_struct *);
 242 static void isert_cq_callback(struct ib_cq *, void *);
 243
 244 static int
 245 isert_create_device_ib_res(struct isert_device *device)
 246 {
 247         struct ib_device *ib_dev = device->ib_device;
 248         struct ib_device_attr *dev_attr;
 249         int ret = 0, i;
 250         int max_cqe;
 251
 252         dev_attr = &device->dev_attr;
 253         ret = isert_query_device(ib_dev, dev_attr);
 254         if (ret)
 255                 return ret;
 256
 257         max_cqe = min(ISER_MAX_CQ_LEN, dev_attr->max_cqe);
 258
 259         /* asign function handlers */
 260         if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS &&
 261             dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) {
 262                 device->use_fastreg = 1;
 263                 device->reg_rdma_mem = isert_reg_rdma;
 264                 device->unreg_rdma_mem = isert_unreg_rdma;
 265         } else {
 266                 device->use_fastreg = 0;
 267                 device->reg_rdma_mem = isert_map_rdma;
 268                 device->unreg_rdma_mem = isert_unmap_cmd;
 269         }
 270
 271         /* Check signature cap */
 272         device->pi_capable = dev_attr->device_cap_flags &
 273                              IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
 274
 275         device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
 276                                         device->ib_device->num_comp_vectors));
 277         pr_info("Using %d CQs, %s supports %d vectors support "
 278                    "Fast registration %d pi_capable %d\n",
 279                    device->comps_used, device->ib_device->name,
 280                    device->ib_device->num_comp_vectors, device->use_fastreg,
 281                    device->pi_capable);
 282
 283         device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
 284                                 GFP_KERNEL);
 285         if (!device->comps) {
 286                 pr_err("Unable to allocate completion contexts\n");
 287                 return -ENOMEM;
 288         }
 289
 290         for (i = 0; i < device->comps_used; i++) {
 291                 struct isert_comp *comp = &device->comps[i];
 292
 293                 comp->device = device;
 294                 INIT_WORK(&comp->work, isert_cq_work);
 295                 comp->cq = ib_create_cq(device->ib_device,
 296                                         isert_cq_callback,
 297                                         isert_cq_event_callback,
 298                                         (void *)comp,
 299                                         max_cqe, i);
 300                 if (IS_ERR(comp->cq)) {
 301                         ret = PTR_ERR(comp->cq);
 302                         comp->cq = NULL;
 303                         goto out_cq;
 304                 }
 305
 306                 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP);
 307                 if (ret)
 308                         goto out_cq;
 309         }
 310
 311         return 0;
 312
 313 out_cq:
 314         for (i = 0; i < device->comps_used; i++) {
 315                 struct isert_comp *comp = &device->comps[i];
 316
 317                 if (comp->cq) {
 318                         cancel_work_sync(&comp->work);
 319                         ib_destroy_cq(comp->cq);
 320                 }
 321         }
 322         kfree(device->comps);
 323
 324         return ret;
 325 }
 326
 327 static void
 328 isert_free_device_ib_res(struct isert_device *device)
 329 {
 330         int i;
 331
 332         pr_info("device %p\n", device);
 333
 334         for (i = 0; i < device->comps_used; i++) {
 335                 struct isert_comp *comp = &device->comps[i];
 336
 337                 cancel_work_sync(&comp->work);
 338                 ib_destroy_cq(comp->cq);
 339                 comp->cq = NULL;
 340         }
 341         kfree(device->comps);
 342 }
 343
 344 static void
 345 isert_device_try_release(struct isert_device *device)
 346 {
 347         mutex_lock(&device_list_mutex);
 348         device->refcount--;
 349         if (!device->refcount) {
 350                 isert_free_device_ib_res(device);
 351                 list_del(&device->dev_node);
 352                 kfree(device);
 353         }
 354         mutex_unlock(&device_list_mutex);
 355 }
 356
 357 static struct isert_device *
 358 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
 359 {
 360         struct isert_device *device;
 361         int ret;
 362
 363         mutex_lock(&device_list_mutex);
 364         list_for_each_entry(device, &device_list, dev_node) {
 365                 if (device->ib_device->node_guid == cma_id->device->node_guid) {
 366                         device->refcount++;
 367                         mutex_unlock(&device_list_mutex);
 368                         return device;
 369                 }
 370         }
 371
 372         device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
 373         if (!device) {
 374                 mutex_unlock(&device_list_mutex);
 375                 return ERR_PTR(-ENOMEM);
 376         }
 377
 378         INIT_LIST_HEAD(&device->dev_node);
 379
 380         device->ib_device = cma_id->device;
 381         ret = isert_create_device_ib_res(device);
 382         if (ret) {
 383                 kfree(device);
 384                 mutex_unlock(&device_list_mutex);
 385                 return ERR_PTR(ret);
 386         }
 387
 388         device->refcount++;
 389         list_add_tail(&device->dev_node, &device_list);
 390         mutex_unlock(&device_list_mutex);
 391
 392         return device;
 393 }
 394
 395 static void
 396 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn)
 397 {
 398         struct fast_reg_descriptor *fr_desc, *tmp;
 399         int i = 0;
 400
 401         if (list_empty(&isert_conn->conn_fr_pool))
 402                 return;
 403
 404         pr_debug("Freeing conn %p fastreg pool", isert_conn);
 405
 406         list_for_each_entry_safe(fr_desc, tmp,
 407                                  &isert_conn->conn_fr_pool, list) {
 408                 list_del(&fr_desc->list);
 409                 ib_free_fast_reg_page_list(fr_desc->data_frpl);
 410                 ib_dereg_mr(fr_desc->data_mr);
 411                 if (fr_desc->pi_ctx) {
 412                         ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl);
 413                         ib_dereg_mr(fr_desc->pi_ctx->prot_mr);
 414                         ib_destroy_mr(fr_desc->pi_ctx->sig_mr);
 415                         kfree(fr_desc->pi_ctx);
 416                 }
 417                 kfree(fr_desc);
 418                 ++i;
 419         }
 420
 421         if (i < isert_conn->conn_fr_pool_size)
 422                 pr_warn("Pool still has %d regions registered\n",
 423                         isert_conn->conn_fr_pool_size - i);
 424 }
 425
 426 static int
 427 isert_create_pi_ctx(struct fast_reg_descriptor *desc,
 428                     struct ib_device *device,
 429                     struct ib_pd *pd)
 430 {
 431         struct ib_mr_init_attr mr_init_attr;
 432         struct pi_context *pi_ctx;
 433         int ret;
 434
 435         pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL);
 436         if (!pi_ctx) {
 437                 pr_err("Failed to allocate pi context\n");
 438                 return -ENOMEM;
 439         }
 440
 441         pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(device,
 442                                             ISCSI_ISER_SG_TABLESIZE);
 443         if (IS_ERR(pi_ctx->prot_frpl)) {
 444                 pr_err("Failed to allocate prot frpl err=%ld\n",
 445                           PTR_ERR(pi_ctx->prot_frpl));
 446                 ret = PTR_ERR(pi_ctx->prot_frpl);
 447                 goto err_pi_ctx;
 448         }
 449
 450         pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE);
 451         if (IS_ERR(pi_ctx->prot_mr)) {
 452                 pr_err("Failed to allocate prot frmr err=%ld\n",
 453                           PTR_ERR(pi_ctx->prot_mr));
 454                 ret = PTR_ERR(pi_ctx->prot_mr);
 455                 goto err_prot_frpl;
 456         }
 457         desc->ind |= ISERT_PROT_KEY_VALID;
 458
 459         memset(&mr_init_attr, 0, sizeof(mr_init_attr));
 460         mr_init_attr.max_reg_descriptors = 2;
 461         mr_init_attr.flags |= IB_MR_SIGNATURE_EN;
 462         pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr);
 463         if (IS_ERR(pi_ctx->sig_mr)) {
 464                 pr_err("Failed to allocate signature enabled mr err=%ld\n",
 465                           PTR_ERR(pi_ctx->sig_mr));
 466                 ret = PTR_ERR(pi_ctx->sig_mr);
 467                 goto err_prot_mr;
 468         }
 469
 470         desc->pi_ctx = pi_ctx;
 471         desc->ind |= ISERT_SIG_KEY_VALID;
 472         desc->ind &= ~ISERT_PROTECTED;
 473
 474         return 0;
 475
 476 err_prot_mr:
 477         ib_dereg_mr(desc->pi_ctx->prot_mr);
 478 err_prot_frpl:
 479         ib_free_fast_reg_page_list(desc->pi_ctx->prot_frpl);
 480 err_pi_ctx:
 481         kfree(desc->pi_ctx);
 482
 483         return ret;
 484 }
 485
 486 static int
 487 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd,
 488                      struct fast_reg_descriptor *fr_desc)
 489 {
 490         int ret;
 491
 492         fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device,
 493                                                          ISCSI_ISER_SG_TABLESIZE);
 494         if (IS_ERR(fr_desc->data_frpl)) {
 495                 pr_err("Failed to allocate data frpl err=%ld\n",
 496                        PTR_ERR(fr_desc->data_frpl));
 497                 return PTR_ERR(fr_desc->data_frpl);
 498         }
 499
 500         fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE);
 501         if (IS_ERR(fr_desc->data_mr)) {
 502                 pr_err("Failed to allocate data frmr err=%ld\n",
 503                        PTR_ERR(fr_desc->data_mr));
 504                 ret = PTR_ERR(fr_desc->data_mr);
 505                 goto err_data_frpl;
 506         }
 507         fr_desc->ind |= ISERT_DATA_KEY_VALID;
 508
 509         pr_debug("Created fr_desc %p\n", fr_desc);
 510
 511         return 0;
 512
 513 err_data_frpl:
 514         ib_free_fast_reg_page_list(fr_desc->data_frpl);
 515
 516         return ret;
 517 }
 518
 519 static int
 520 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn)
 521 {
 522         struct fast_reg_descriptor *fr_desc;
 523         struct isert_device *device = isert_conn->conn_device;
 524         struct se_session *se_sess = isert_conn->conn->sess->se_sess;
 525         struct se_node_acl *se_nacl = se_sess->se_node_acl;
 526         int i, ret, tag_num;
 527         /*
 528          * Setup the number of FRMRs based upon the number of tags
 529          * available to session in iscsi_target_locate_portal().
 530          */
 531         tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth);
 532         tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS;
 533
 534         isert_conn->conn_fr_pool_size = 0;
 535         for (i = 0; i < tag_num; i++) {
 536                 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL);
 537                 if (!fr_desc) {
 538                         pr_err("Failed to allocate fast_reg descriptor\n");
 539                         ret = -ENOMEM;
 540                         goto err;
 541                 }
 542
 543                 ret = isert_create_fr_desc(device->ib_device,
 544                                            isert_conn->conn_pd, fr_desc);
 545                 if (ret) {
 546                         pr_err("Failed to create fastreg descriptor err=%d\n",
 547                                ret);
 548                         kfree(fr_desc);
 549                         goto err;
 550                 }
 551
 552                 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool);
 553                 isert_conn->conn_fr_pool_size++;
 554         }
 555
 556         pr_debug("Creating conn %p fastreg pool size=%d",
 557                  isert_conn, isert_conn->conn_fr_pool_size);
 558
 559         return 0;
 560
 561 err:
 562         isert_conn_free_fastreg_pool(isert_conn);
 563         return ret;
 564 }
 565
 566 static int
 567 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 568 {
 569         struct isert_np *isert_np = cma_id->context;
 570         struct iscsi_np *np = isert_np->np;
 571         struct isert_conn *isert_conn;
 572         struct isert_device *device;
 573         struct ib_device *ib_dev = cma_id->device;
 574         int ret = 0;
 575
 576         spin_lock_bh(&np->np_thread_lock);
 577         if (!np->enabled) {
 578                 spin_unlock_bh(&np->np_thread_lock);
 579                 pr_debug("iscsi_np is not enabled, reject connect request\n");
 580                 return rdma_reject(cma_id, NULL, 0);
 581         }
 582         spin_unlock_bh(&np->np_thread_lock);
 583
 584         pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
 585                  cma_id, cma_id->context);
 586
 587         isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
 588         if (!isert_conn) {
 589                 pr_err("Unable to allocate isert_conn\n");
 590                 return -ENOMEM;
 591         }
 592         isert_conn->state = ISER_CONN_INIT;
 593         INIT_LIST_HEAD(&isert_conn->conn_accept_node);
 594         init_completion(&isert_conn->conn_login_comp);
 595         init_completion(&isert_conn->login_req_comp);
 596         init_completion(&isert_conn->conn_wait);
 597         kref_init(&isert_conn->conn_kref);
 598         mutex_init(&isert_conn->conn_mutex);
 599         spin_lock_init(&isert_conn->conn_lock);
 600         INIT_LIST_HEAD(&isert_conn->conn_fr_pool);
 601
 602         isert_conn->conn_cm_id = cma_id;
 603
 604         isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
 605                                         ISER_RX_LOGIN_SIZE, GFP_KERNEL);
 606         if (!isert_conn->login_buf) {
 607                 pr_err("Unable to allocate isert_conn->login_buf\n");
 608                 ret = -ENOMEM;
 609                 goto out;
 610         }
 611
 612         isert_conn->login_req_buf = isert_conn->login_buf;
 613         isert_conn->login_rsp_buf = isert_conn->login_buf +
 614                                     ISCSI_DEF_MAX_RECV_SEG_LEN;
 615         pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
 616                  isert_conn->login_buf, isert_conn->login_req_buf,
 617                  isert_conn->login_rsp_buf);
 618
 619         isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
 620                                 (void *)isert_conn->login_req_buf,
 621                                 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
 622
 623         ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
 624         if (ret) {
 625                 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
 626                        ret);
 627                 isert_conn->login_req_dma = 0;
 628                 goto out_login_buf;
 629         }
 630
 631         isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
 632                                         (void *)isert_conn->login_rsp_buf,
 633                                         ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
 634
 635         ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
 636         if (ret) {
 637                 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
 638                        ret);
 639                 isert_conn->login_rsp_dma = 0;
 640                 goto out_req_dma_map;
 641         }
 642
 643         device = isert_device_find_by_ib_dev(cma_id);
 644         if (IS_ERR(device)) {
 645                 ret = PTR_ERR(device);
 646                 goto out_rsp_dma_map;
 647         }
 648
 649         /* Set max inflight RDMA READ requests */
 650         isert_conn->initiator_depth = min_t(u8,
 651                                 event->param.conn.initiator_depth,
 652                                 device->dev_attr.max_qp_init_rd_atom);
 653         pr_debug("Using initiator_depth: %u\n", isert_conn->initiator_depth);
 654
 655         isert_conn->conn_device = device;
 656         isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device);
 657         if (IS_ERR(isert_conn->conn_pd)) {
 658                 ret = PTR_ERR(isert_conn->conn_pd);
 659                 pr_err("ib_alloc_pd failed for conn %p: ret=%d\n",
 660                        isert_conn, ret);
 661                 goto out_pd;
 662         }
 663
 664         isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd,
 665                                            IB_ACCESS_LOCAL_WRITE);
 666         if (IS_ERR(isert_conn->conn_mr)) {
 667                 ret = PTR_ERR(isert_conn->conn_mr);
 668                 pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n",
 669                        isert_conn, ret);
 670                 goto out_mr;
 671         }
 672
 673         ret = isert_conn_setup_qp(isert_conn, cma_id);
 674         if (ret)
 675                 goto out_conn_dev;
 676
 677         ret = isert_rdma_post_recvl(isert_conn);
 678         if (ret)
 679                 goto out_conn_dev;
 680
 681         ret = isert_rdma_accept(isert_conn);
 682         if (ret)
 683                 goto out_conn_dev;
 684
 685         mutex_lock(&isert_np->np_accept_mutex);
 686         list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list);
 687         mutex_unlock(&isert_np->np_accept_mutex);
 688
 689         pr_debug("isert_connect_request() up np_sem np: %p\n", np);
 690         up(&isert_np->np_sem);
 691         return 0;
 692
 693 out_conn_dev:
 694         ib_dereg_mr(isert_conn->conn_mr);
 695 out_mr:
 696         ib_dealloc_pd(isert_conn->conn_pd);
 697 out_pd:
 698         isert_device_try_release(device);
 699 out_rsp_dma_map:
 700         ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
 701                             ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
 702 out_req_dma_map:
 703         ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
 704                             ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
 705 out_login_buf:
 706         kfree(isert_conn->login_buf);
 707 out:
 708         kfree(isert_conn);
 709         rdma_reject(cma_id, NULL, 0);
 710         return ret;
 711 }
 712
 713 static void
 714 isert_connect_release(struct isert_conn *isert_conn)
 715 {
 716         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 717         struct isert_device *device = isert_conn->conn_device;
 718
 719         pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
 720
 721         if (device && device->use_fastreg)
 722                 isert_conn_free_fastreg_pool(isert_conn);
 723
 724         isert_free_rx_descriptors(isert_conn);
 725         rdma_destroy_id(isert_conn->conn_cm_id);
 726
 727         if (isert_conn->conn_qp) {
 728                 struct isert_comp *comp = isert_conn->conn_qp->recv_cq->cq_context;
 729
 730                 pr_debug("dec completion context %p active_qps\n", comp);
 731                 mutex_lock(&device_list_mutex);
 732                 comp->active_qps--;
 733                 mutex_unlock(&device_list_mutex);
 734
 735                 ib_destroy_qp(isert_conn->conn_qp);
 736         }
 737
 738         ib_dereg_mr(isert_conn->conn_mr);
 739         ib_dealloc_pd(isert_conn->conn_pd);
 740
 741         if (isert_conn->login_buf) {
 742                 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
 743                                     ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
 744                 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
 745                                     ISCSI_DEF_MAX_RECV_SEG_LEN,
 746                                     DMA_FROM_DEVICE);
 747                 kfree(isert_conn->login_buf);
 748         }
 749         kfree(isert_conn);
 750
 751         if (device)
 752                 isert_device_try_release(device);
 753
 754         pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
 755 }
 756
 757 static void
 758 isert_connected_handler(struct rdma_cm_id *cma_id)
 759 {
 760         struct isert_conn *isert_conn = cma_id->qp->qp_context;
 761
 762         pr_info("conn %p\n", isert_conn);
 763
 764         if (!kref_get_unless_zero(&isert_conn->conn_kref)) {
 765                 pr_warn("conn %p connect_release is running\n", isert_conn);
 766                 return;
 767         }
 768
 769         mutex_lock(&isert_conn->conn_mutex);
 770         if (isert_conn->state != ISER_CONN_FULL_FEATURE)
 771                 isert_conn->state = ISER_CONN_UP;
 772         mutex_unlock(&isert_conn->conn_mutex);
 773 }
 774
 775 static void
 776 isert_release_conn_kref(struct kref *kref)
 777 {
 778         struct isert_conn *isert_conn = container_of(kref,
 779                                 struct isert_conn, conn_kref);
 780
 781         pr_debug("Calling isert_connect_release for final kref %s/%d\n",
 782                  current->comm, current->pid);
 783
 784         isert_connect_release(isert_conn);
 785 }
 786
 787 static void
 788 isert_put_conn(struct isert_conn *isert_conn)
 789 {
 790         kref_put(&isert_conn->conn_kref, isert_release_conn_kref);
 791 }
 792
 793 /**
 794  * isert_conn_terminate() - Initiate connection termination
 795  * @isert_conn: isert connection struct
 796  *
 797  * Notes:
 798  * In case the connection state is FULL_FEATURE, move state
 799  * to TEMINATING and start teardown sequence (rdma_disconnect).
 800  * In case the connection state is UP, complete flush as well.
 801  *
 802  * This routine must be called with conn_mutex held. Thus it is
 803  * safe to call multiple times.
 804  */
 805 static void
 806 isert_conn_terminate(struct isert_conn *isert_conn)
 807 {
 808         int err;
 809
 810         switch (isert_conn->state) {
 811         case ISER_CONN_TERMINATING:
 812                 break;
 813         case ISER_CONN_UP:
 814         case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
 815                 pr_info("Terminating conn %p state %d\n",
 816                            isert_conn, isert_conn->state);
 817                 isert_conn->state = ISER_CONN_TERMINATING;
 818                 err = rdma_disconnect(isert_conn->conn_cm_id);
 819                 if (err)
 820                         pr_warn("Failed rdma_disconnect isert_conn %p\n",
 821                                    isert_conn);
 822                 break;
 823         default:
 824                 pr_warn("conn %p teminating in state %d\n",
 825                            isert_conn, isert_conn->state);
 826         }
 827 }
 828
 829 static int
 830 isert_np_cma_handler(struct isert_np *isert_np,
 831                      enum rdma_cm_event_type event)
 832 {
 833         pr_debug("isert np %p, handling event %d\n", isert_np, event);
 834
 835         switch (event) {
 836         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 837                 isert_np->np_cm_id = NULL;
 838                 break;
 839         case RDMA_CM_EVENT_ADDR_CHANGE:
 840                 isert_np->np_cm_id = isert_setup_id(isert_np);
 841                 if (IS_ERR(isert_np->np_cm_id)) {
 842                         pr_err("isert np %p setup id failed: %ld\n",
 843                                  isert_np, PTR_ERR(isert_np->np_cm_id));
 844                         isert_np->np_cm_id = NULL;
 845                 }
 846                 break;
 847         default:
 848                 pr_err("isert np %p Unexpected event %d\n",
 849                           isert_np, event);
 850         }
 851
 852         return -1;
 853 }
 854
 855 static int
 856 isert_disconnected_handler(struct rdma_cm_id *cma_id,
 857                            enum rdma_cm_event_type event)
 858 {
 859         struct isert_np *isert_np = cma_id->context;
 860         struct isert_conn *isert_conn;
 861
 862         if (isert_np->np_cm_id == cma_id)
 863                 return isert_np_cma_handler(cma_id->context, event);
 864
 865         isert_conn = cma_id->qp->qp_context;
 866
 867         mutex_lock(&isert_conn->conn_mutex);
 868         isert_conn_terminate(isert_conn);
 869         mutex_unlock(&isert_conn->conn_mutex);
 870
 871         pr_info("conn %p completing conn_wait\n", isert_conn);
 872         complete(&isert_conn->conn_wait);
 873
 874         return 0;
 875 }
 876
 877 static void
 878 isert_connect_error(struct rdma_cm_id *cma_id)
 879 {
 880         struct isert_conn *isert_conn = cma_id->qp->qp_context;
 881
 882         isert_put_conn(isert_conn);
 883 }
 884
 885 static int
 886 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 887 {
 888         int ret = 0;
 889
 890         pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
 891                  event->event, event->status, cma_id->context, cma_id);
 892
 893         switch (event->event) {
 894         case RDMA_CM_EVENT_CONNECT_REQUEST:
 895                 ret = isert_connect_request(cma_id, event);
 896                 if (ret)
 897                         pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
 898                                 event->event, ret);
 899                 break;
 900         case RDMA_CM_EVENT_ESTABLISHED:
 901                 isert_connected_handler(cma_id);
 902                 break;
 903         case RDMA_CM_EVENT_ADDR_CHANGE:    /* FALLTHRU */
 904         case RDMA_CM_EVENT_DISCONNECTED:   /* FALLTHRU */
 905         case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */
 906         case RDMA_CM_EVENT_TIMEWAIT_EXIT:  /* FALLTHRU */
 907                 ret = isert_disconnected_handler(cma_id, event->event);
 908                 break;
 909         case RDMA_CM_EVENT_REJECTED:       /* FALLTHRU */
 910         case RDMA_CM_EVENT_UNREACHABLE:    /* FALLTHRU */
 911         case RDMA_CM_EVENT_CONNECT_ERROR:
 912                 isert_connect_error(cma_id);
 913                 break;
 914         default:
 915                 pr_err("Unhandled RDMA CMA event: %d\n", event->event);
 916                 break;
 917         }
 918
 919         return ret;
 920 }
 921
 922 static int
 923 isert_post_recv(struct isert_conn *isert_conn, u32 count)
 924 {
 925         struct ib_recv_wr *rx_wr, *rx_wr_failed;
 926         int i, ret;
 927         unsigned int rx_head = isert_conn->conn_rx_desc_head;
 928         struct iser_rx_desc *rx_desc;
 929
 930         for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) {
 931                 rx_desc         = &isert_conn->conn_rx_descs[rx_head];
 932                 rx_wr->wr_id    = (uintptr_t)rx_desc;
 933                 rx_wr->sg_list  = &rx_desc->rx_sg;
 934                 rx_wr->num_sge  = 1;
 935                 rx_wr->next     = rx_wr + 1;
 936                 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
 937         }
 938
 939         rx_wr--;
 940         rx_wr->next = NULL; /* mark end of work requests list */
 941
 942         isert_conn->post_recv_buf_count += count;
 943         ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr,
 944                                 &rx_wr_failed);
 945         if (ret) {
 946                 pr_err("ib_post_recv() failed with ret: %d\n", ret);
 947                 isert_conn->post_recv_buf_count -= count;
 948         } else {
 949                 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count);
 950                 isert_conn->conn_rx_desc_head = rx_head;
 951         }
 952         return ret;
 953 }
 954
 955 static int
 956 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
 957 {
 958         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 959         struct ib_send_wr send_wr, *send_wr_failed;
 960         int ret;
 961
 962         ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
 963                                       ISER_HEADERS_LEN, DMA_TO_DEVICE);
 964
 965         send_wr.next    = NULL;
 966         send_wr.wr_id   = (uintptr_t)tx_desc;
 967         send_wr.sg_list = tx_desc->tx_sg;
 968         send_wr.num_sge = tx_desc->num_sge;
 969         send_wr.opcode  = IB_WR_SEND;
 970         send_wr.send_flags = IB_SEND_SIGNALED;
 971
 972         ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed);
 973         if (ret)
 974                 pr_err("ib_post_send() failed, ret: %d\n", ret);
 975
 976         return ret;
 977 }
 978
 979 static void
 980 isert_create_send_desc(struct isert_conn *isert_conn,
 981                        struct isert_cmd *isert_cmd,
 982                        struct iser_tx_desc *tx_desc)
 983 {
 984         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
 985
 986         ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
 987                                    ISER_HEADERS_LEN, DMA_TO_DEVICE);
 988
 989         memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
 990         tx_desc->iser_header.flags = ISER_VER;
 991
 992         tx_desc->num_sge = 1;
 993         tx_desc->isert_cmd = isert_cmd;
 994
 995         if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) {
 996                 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
 997                 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc);
 998         }
 999 }
1000
1001 static int
1002 isert_init_tx_hdrs(struct isert_conn *isert_conn,
1003                    struct iser_tx_desc *tx_desc)
1004 {
1005         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1006         u64 dma_addr;
1007
1008         dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
1009                         ISER_HEADERS_LEN, DMA_TO_DEVICE);
1010         if (ib_dma_mapping_error(ib_dev, dma_addr)) {
1011                 pr_err("ib_dma_mapping_error() failed\n");
1012                 return -ENOMEM;
1013         }
1014
1015         tx_desc->dma_addr = dma_addr;
1016         tx_desc->tx_sg[0].addr  = tx_desc->dma_addr;
1017         tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
1018         tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
1019
1020         pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
1021                  " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr,
1022                  tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey);
1023
1024         return 0;
1025 }
1026
1027 static void
1028 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1029                    struct ib_send_wr *send_wr)
1030 {
1031         struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;
1032
1033         isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
1034         send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;
1035         send_wr->opcode = IB_WR_SEND;
1036         send_wr->sg_list = &tx_desc->tx_sg[0];
1037         send_wr->num_sge = isert_cmd->tx_desc.num_sge;
1038         send_wr->send_flags = IB_SEND_SIGNALED;
1039 }
1040
1041 static int
1042 isert_rdma_post_recvl(struct isert_conn *isert_conn)
1043 {
1044         struct ib_recv_wr rx_wr, *rx_wr_fail;
1045         struct ib_sge sge;
1046         int ret;
1047
1048         memset(&sge, 0, sizeof(struct ib_sge));
1049         sge.addr = isert_conn->login_req_dma;
1050         sge.length = ISER_RX_LOGIN_SIZE;
1051         sge.lkey = isert_conn->conn_mr->lkey;
1052
1053         pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
1054                 sge.addr, sge.length, sge.lkey);
1055
1056         memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
1057         rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf;
1058         rx_wr.sg_list = &sge;
1059         rx_wr.num_sge = 1;
1060
1061         isert_conn->post_recv_buf_count++;
1062         ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail);
1063         if (ret) {
1064                 pr_err("ib_post_recv() failed: %d\n", ret);
1065                 isert_conn->post_recv_buf_count--;
1066         }
1067
1068         pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
1069         return ret;
1070 }
1071
1072 static int
1073 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
1074                    u32 length)
1075 {
1076         struct isert_conn *isert_conn = conn->context;
1077         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1078         struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc;
1079         int ret;
1080
1081         isert_create_send_desc(isert_conn, NULL, tx_desc);
1082
1083         memcpy(&tx_desc->iscsi_header, &login->rsp[0],
1084                sizeof(struct iscsi_hdr));
1085
1086         isert_init_tx_hdrs(isert_conn, tx_desc);
1087
1088         if (length > 0) {
1089                 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
1090
1091                 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
1092                                            length, DMA_TO_DEVICE);
1093
1094                 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
1095
1096                 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
1097                                               length, DMA_TO_DEVICE);
1098
1099                 tx_dsg->addr    = isert_conn->login_rsp_dma;
1100                 tx_dsg->length  = length;
1101                 tx_dsg->lkey    = isert_conn->conn_mr->lkey;
1102                 tx_desc->num_sge = 2;
1103         }
1104         if (!login->login_failed) {
1105                 if (login->login_complete) {
1106                         if (!conn->sess->sess_ops->SessionType &&
1107                             isert_conn->conn_device->use_fastreg) {
1108                                 ret = isert_conn_create_fastreg_pool(isert_conn);
1109                                 if (ret) {
1110                                         pr_err("Conn: %p failed to create"
1111                                                " fastreg pool\n", isert_conn);
1112                                         return ret;
1113                                 }
1114                         }
1115
1116                         ret = isert_alloc_rx_descriptors(isert_conn);
1117                         if (ret)
1118                                 return ret;
1119
1120                         ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
1121                         if (ret)
1122                                 return ret;
1123
1124                         /* Now we are in FULL_FEATURE phase */
1125                         mutex_lock(&isert_conn->conn_mutex);
1126                         isert_conn->state = ISER_CONN_FULL_FEATURE;
1127                         mutex_unlock(&isert_conn->conn_mutex);
1128                         goto post_send;
1129                 }
1130
1131                 ret = isert_rdma_post_recvl(isert_conn);
1132                 if (ret)
1133                         return ret;
1134         }
1135 post_send:
1136         ret = isert_post_send(isert_conn, tx_desc);
1137         if (ret)
1138                 return ret;
1139
1140         return 0;
1141 }
1142
1143 static void
1144 isert_rx_login_req(struct isert_conn *isert_conn)
1145 {
1146         struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf;
1147         int rx_buflen = isert_conn->login_req_len;
1148         struct iscsi_conn *conn = isert_conn->conn;
1149         struct iscsi_login *login = conn->conn_login;
1150         int size;
1151
1152         pr_info("conn %p\n", isert_conn);
1153
1154         WARN_ON_ONCE(!login);
1155
1156         if (login->first_request) {
1157                 struct iscsi_login_req *login_req =
1158                         (struct iscsi_login_req *)&rx_desc->iscsi_header;
1159                 /*
1160                  * Setup the initial iscsi_login values from the leading
1161                  * login request PDU.
1162                  */
1163                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1164                 login->current_stage =
1165                         (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
1166                          >> 2;
1167                 login->version_min      = login_req->min_version;
1168                 login->version_max      = login_req->max_version;
1169                 memcpy(login->isid, login_req->isid, 6);
1170                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
1171                 login->init_task_tag    = login_req->itt;
1172                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1173                 login->cid              = be16_to_cpu(login_req->cid);
1174                 login->tsih             = be16_to_cpu(login_req->tsih);
1175         }
1176
1177         memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
1178
1179         size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
1180         pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
1181                  size, rx_buflen, MAX_KEY_VALUE_PAIRS);
1182         memcpy(login->req_buf, &rx_desc->data[0], size);
1183
1184         if (login->first_request) {
1185                 complete(&isert_conn->conn_login_comp);
1186                 return;
1187         }
1188         schedule_delayed_work(&conn->login_work, 0);
1189 }
1190
1191 static struct iscsi_cmd
1192 *isert_allocate_cmd(struct iscsi_conn *conn)
1193 {
1194         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1195         struct isert_cmd *isert_cmd;
1196         struct iscsi_cmd *cmd;
1197
1198         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
1199         if (!cmd) {
1200                 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n");
1201                 return NULL;
1202         }
1203         isert_cmd = iscsit_priv_cmd(cmd);
1204         isert_cmd->conn = isert_conn;
1205         isert_cmd->iscsi_cmd = cmd;
1206
1207         return cmd;
1208 }
1209
1210 static int
1211 isert_handle_scsi_cmd(struct isert_conn *isert_conn,
1212                       struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,
1213                       struct iser_rx_desc *rx_desc, unsigned char *buf)
1214 {
1215         struct iscsi_conn *conn = isert_conn->conn;
1216         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1217         struct scatterlist *sg;
1218         int imm_data, imm_data_len, unsol_data, sg_nents, rc;
1219         bool dump_payload = false;
1220
1221         rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1222         if (rc < 0)
1223                 return rc;
1224
1225         imm_data = cmd->immediate_data;
1226         imm_data_len = cmd->first_burst_len;
1227         unsol_data = cmd->unsolicited_data;
1228
1229         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1230         if (rc < 0) {
1231                 return 0;
1232         } else if (rc > 0) {
1233                 dump_payload = true;
1234                 goto sequence_cmd;
1235         }
1236
1237         if (!imm_data)
1238                 return 0;
1239
1240         sg = &cmd->se_cmd.t_data_sg[0];
1241         sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
1242
1243         pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
1244                  sg, sg_nents, &rx_desc->data[0], imm_data_len);
1245
1246         sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);
1247
1248         cmd->write_data_done += imm_data_len;
1249
1250         if (cmd->write_data_done == cmd->se_cmd.data_length) {
1251                 spin_lock_bh(&cmd->istate_lock);
1252                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1253                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1254                 spin_unlock_bh(&cmd->istate_lock);
1255         }
1256
1257 sequence_cmd:
1258         rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
1259
1260         if (!rc && dump_payload == false && unsol_data)
1261                 iscsit_set_unsoliticed_dataout(cmd);
1262         else if (dump_payload && imm_data)
1263                 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
1264
1265         return 0;
1266 }
1267
1268 static int
1269 isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
1270                            struct iser_rx_desc *rx_desc, unsigned char *buf)
1271 {
1272         struct scatterlist *sg_start;
1273         struct iscsi_conn *conn = isert_conn->conn;
1274         struct iscsi_cmd *cmd = NULL;
1275         struct iscsi_data *hdr = (struct iscsi_data *)buf;
1276         u32 unsol_data_len = ntoh24(hdr->dlength);
1277         int rc, sg_nents, sg_off, page_off;
1278
1279         rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1280         if (rc < 0)
1281                 return rc;
1282         else if (!cmd)
1283                 return 0;
1284         /*
1285          * FIXME: Unexpected unsolicited_data out
1286          */
1287         if (!cmd->unsolicited_data) {
1288                 pr_err("Received unexpected solicited data payload\n");
1289                 dump_stack();
1290                 return -1;
1291         }
1292
1293         pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
1294                  unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length);
1295
1296         sg_off = cmd->write_data_done / PAGE_SIZE;
1297         sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1298         sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1299         page_off = cmd->write_data_done % PAGE_SIZE;
1300         /*
1301          * FIXME: Non page-aligned unsolicited_data out
1302          */
1303         if (page_off) {
1304                 pr_err("Received unexpected non-page aligned data payload\n");
1305                 dump_stack();
1306                 return -1;
1307         }
1308         pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
1309                  sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len);
1310
1311         sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
1312                             unsol_data_len);
1313
1314         rc = iscsit_check_dataout_payload(cmd, hdr, false);
1315         if (rc < 0)
1316                 return rc;
1317
1318         return 0;
1319 }
1320
1321 static int
1322 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1323                      struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1324                      unsigned char *buf)
1325 {
1326         struct iscsi_conn *conn = isert_conn->conn;
1327         struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1328         int rc;
1329
1330         rc = iscsit_setup_nop_out(conn, cmd, hdr);
1331         if (rc < 0)
1332                 return rc;
1333         /*
1334          * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
1335          */
1336
1337         return iscsit_process_nop_out(conn, cmd, hdr);
1338 }
1339
1340 static int
1341 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1342                       struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1343                       struct iscsi_text *hdr)
1344 {
1345         struct iscsi_conn *conn = isert_conn->conn;
1346         u32 payload_length = ntoh24(hdr->dlength);
1347         int rc;
1348         unsigned char *text_in;
1349
1350         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1351         if (rc < 0)
1352                 return rc;
1353
1354         text_in = kzalloc(payload_length, GFP_KERNEL);
1355         if (!text_in) {
1356                 pr_err("Unable to allocate text_in of payload_length: %u\n",
1357                        payload_length);
1358                 return -ENOMEM;
1359         }
1360         cmd->text_in_ptr = text_in;
1361
1362         memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length);
1363
1364         return iscsit_process_text_cmd(conn, cmd, hdr);
1365 }
1366
1367 static int
1368 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1369                 uint32_t read_stag, uint64_t read_va,
1370                 uint32_t write_stag, uint64_t write_va)
1371 {
1372         struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1373         struct iscsi_conn *conn = isert_conn->conn;
1374         struct iscsi_session *sess = conn->sess;
1375         struct iscsi_cmd *cmd;
1376         struct isert_cmd *isert_cmd;
1377         int ret = -EINVAL;
1378         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1379
1380         if (sess->sess_ops->SessionType &&
1381            (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) {
1382                 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
1383                        " ignoring\n", opcode);
1384                 return 0;
1385         }
1386
1387         switch (opcode) {
1388         case ISCSI_OP_SCSI_CMD:
1389                 cmd = isert_allocate_cmd(conn);
1390                 if (!cmd)
1391                         break;
1392
1393                 isert_cmd = iscsit_priv_cmd(cmd);
1394                 isert_cmd->read_stag = read_stag;
1395                 isert_cmd->read_va = read_va;
1396                 isert_cmd->write_stag = write_stag;
1397                 isert_cmd->write_va = write_va;
1398
1399                 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
1400                                         rx_desc, (unsigned char *)hdr);
1401                 break;
1402         case ISCSI_OP_NOOP_OUT:
1403                 cmd = isert_allocate_cmd(conn);
1404                 if (!cmd)
1405                         break;
1406
1407                 isert_cmd = iscsit_priv_cmd(cmd);
1408                 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,
1409                                            rx_desc, (unsigned char *)hdr);
1410                 break;
1411         case ISCSI_OP_SCSI_DATA_OUT:
1412                 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1413                                                 (unsigned char *)hdr);
1414                 break;
1415         case ISCSI_OP_SCSI_TMFUNC:
1416                 cmd = isert_allocate_cmd(conn);
1417                 if (!cmd)
1418                         break;
1419
1420                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1421                                                 (unsigned char *)hdr);
1422                 break;
1423         case ISCSI_OP_LOGOUT:
1424                 cmd = isert_allocate_cmd(conn);
1425                 if (!cmd)
1426                         break;
1427
1428                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1429                 if (ret > 0)
1430                         wait_for_completion_timeout(&conn->conn_logout_comp,
1431                                                     SECONDS_FOR_LOGOUT_COMP *
1432                                                     HZ);
1433                 break;
1434         case ISCSI_OP_TEXT:
1435                 cmd = isert_allocate_cmd(conn);
1436                 if (!cmd)
1437                         break;
1438
1439                 isert_cmd = iscsit_priv_cmd(cmd);
1440                 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
1441                                             rx_desc, (struct iscsi_text *)hdr);
1442                 break;
1443         default:
1444                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1445                 dump_stack();
1446                 break;
1447         }
1448
1449         return ret;
1450 }
1451
1452 static void
1453 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
1454 {
1455         struct iser_hdr *iser_hdr = &rx_desc->iser_header;
1456         uint64_t read_va = 0, write_va = 0;
1457         uint32_t read_stag = 0, write_stag = 0;
1458         int rc;
1459
1460         switch (iser_hdr->flags & 0xF0) {
1461         case ISCSI_CTRL:
1462                 if (iser_hdr->flags & ISER_RSV) {
1463                         read_stag = be32_to_cpu(iser_hdr->read_stag);
1464                         read_va = be64_to_cpu(iser_hdr->read_va);
1465                         pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
1466                                  read_stag, (unsigned long long)read_va);
1467                 }
1468                 if (iser_hdr->flags & ISER_WSV) {
1469                         write_stag = be32_to_cpu(iser_hdr->write_stag);
1470                         write_va = be64_to_cpu(iser_hdr->write_va);
1471                         pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
1472                                  write_stag, (unsigned long long)write_va);
1473                 }
1474
1475                 pr_debug("ISER ISCSI_CTRL PDU\n");
1476                 break;
1477         case ISER_HELLO:
1478                 pr_err("iSER Hello message\n");
1479                 break;
1480         default:
1481                 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
1482                 break;
1483         }
1484
1485         rc = isert_rx_opcode(isert_conn, rx_desc,
1486                              read_stag, read_va, write_stag, write_va);
1487 }
1488
1489 static void
1490 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn,
1491                     u32 xfer_len)
1492 {
1493         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1494         struct iscsi_hdr *hdr;
1495         u64 rx_dma;
1496         int rx_buflen, outstanding;
1497
1498         if ((char *)desc == isert_conn->login_req_buf) {
1499                 rx_dma = isert_conn->login_req_dma;
1500                 rx_buflen = ISER_RX_LOGIN_SIZE;
1501                 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1502                          rx_dma, rx_buflen);
1503         } else {
1504                 rx_dma = desc->dma_addr;
1505                 rx_buflen = ISER_RX_PAYLOAD_SIZE;
1506                 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1507                          rx_dma, rx_buflen);
1508         }
1509
1510         ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);
1511
1512         hdr = &desc->iscsi_header;
1513         pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1514                  hdr->opcode, hdr->itt, hdr->flags,
1515                  (int)(xfer_len - ISER_HEADERS_LEN));
1516
1517         if ((char *)desc == isert_conn->login_req_buf) {
1518                 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN;
1519                 if (isert_conn->conn) {
1520                         struct iscsi_login *login = isert_conn->conn->conn_login;
1521
1522                         if (login && !login->first_request)
1523                                 isert_rx_login_req(isert_conn);
1524                 }
1525                 mutex_lock(&isert_conn->conn_mutex);
1526                 complete(&isert_conn->login_req_comp);
1527                 mutex_unlock(&isert_conn->conn_mutex);
1528         } else {
1529                 isert_rx_do_work(desc, isert_conn);
1530         }
1531
1532         ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
1533                                       DMA_FROM_DEVICE);
1534
1535         isert_conn->post_recv_buf_count--;
1536         pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
1537                  isert_conn->post_recv_buf_count);
1538
1539         if ((char *)desc == isert_conn->login_req_buf)
1540                 return;
1541
1542         outstanding = isert_conn->post_recv_buf_count;
1543         if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
1544                 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
1545                                 ISERT_MIN_POSTED_RX);
1546                 err = isert_post_recv(isert_conn, count);
1547                 if (err) {
1548                         pr_err("isert_post_recv() count: %d failed, %d\n",
1549                                count, err);
1550                 }
1551         }
1552 }
1553
1554 static int
1555 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1556                    struct scatterlist *sg, u32 nents, u32 length, u32 offset,
1557                    enum iser_ib_op_code op, struct isert_data_buf *data)
1558 {
1559         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1560
1561         data->dma_dir = op == ISER_IB_RDMA_WRITE ?
1562                               DMA_TO_DEVICE : DMA_FROM_DEVICE;
1563
1564         data->len = length - offset;
1565         data->offset = offset;
1566         data->sg_off = data->offset / PAGE_SIZE;
1567
1568         data->sg = &sg[data->sg_off];
1569         data->nents = min_t(unsigned int, nents - data->sg_off,
1570                                           ISCSI_ISER_SG_TABLESIZE);
1571         data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE *
1572                                         PAGE_SIZE);
1573
1574         data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents,
1575                                         data->dma_dir);
1576         if (unlikely(!data->dma_nents)) {
1577                 pr_err("Cmd: unable to dma map SGs %p\n", sg);
1578                 return -EINVAL;
1579         }
1580
1581         pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n",
1582                  isert_cmd, data->dma_nents, data->sg, data->nents, data->len);
1583
1584         return 0;
1585 }
1586
1587 static void
1588 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data)
1589 {
1590         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1591
1592         ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir);
1593         memset(data, 0, sizeof(*data));
1594 }
1595
1596
1597
1598 static void
1599 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
1600 {
1601         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1602
1603         pr_debug("isert_unmap_cmd: %p\n", isert_cmd);
1604
1605         if (wr->data.sg) {
1606                 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd);
1607                 isert_unmap_data_buf(isert_conn, &wr->data);
1608         }
1609
1610         if (wr->send_wr) {
1611                 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd);
1612                 kfree(wr->send_wr);
1613                 wr->send_wr = NULL;
1614         }
1615
1616         if (wr->ib_sge) {
1617                 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd);
1618                 kfree(wr->ib_sge);
1619                 wr->ib_sge = NULL;
1620         }
1621 }
1622
1623 static void
1624 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
1625 {
1626         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1627         LIST_HEAD(unmap_list);
1628
1629         pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd);
1630
1631         if (wr->fr_desc) {
1632                 pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n",
1633                          isert_cmd, wr->fr_desc);
1634                 if (wr->fr_desc->ind & ISERT_PROTECTED) {
1635                         isert_unmap_data_buf(isert_conn, &wr->prot);
1636                         wr->fr_desc->ind &= ~ISERT_PROTECTED;
1637                 }
1638                 spin_lock_bh(&isert_conn->conn_lock);
1639                 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool);
1640                 spin_unlock_bh(&isert_conn->conn_lock);
1641                 wr->fr_desc = NULL;
1642         }
1643
1644         if (wr->data.sg) {
1645                 pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd);
1646                 isert_unmap_data_buf(isert_conn, &wr->data);
1647         }
1648
1649         wr->ib_sge = NULL;
1650         wr->send_wr = NULL;
1651 }
1652
1653 static void
1654 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
1655 {
1656         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1657         struct isert_conn *isert_conn = isert_cmd->conn;
1658         struct iscsi_conn *conn = isert_conn->conn;
1659         struct isert_device *device = isert_conn->conn_device;
1660
1661         pr_debug("Entering isert_put_cmd: %p\n", isert_cmd);
1662
1663         switch (cmd->iscsi_opcode) {
1664         case ISCSI_OP_SCSI_CMD:
1665                 spin_lock_bh(&conn->cmd_lock);
1666                 if (!list_empty(&cmd->i_conn_node))
1667                         list_del_init(&cmd->i_conn_node);
1668                 spin_unlock_bh(&conn->cmd_lock);
1669
1670                 if (cmd->data_direction == DMA_TO_DEVICE) {
1671                         iscsit_stop_dataout_timer(cmd);
1672                         /*
1673                          * Check for special case during comp_err where
1674                          * WRITE_PENDING has been handed off from core,
1675                          * but requires an extra target_put_sess_cmd()
1676                          * before transport_generic_free_cmd() below.
1677                          */
1678                         if (comp_err &&
1679                             cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
1680                                 struct se_cmd *se_cmd = &cmd->se_cmd;
1681
1682                                 target_put_sess_cmd(se_cmd->se_sess, se_cmd);
1683                         }
1684                 }
1685
1686                 device->unreg_rdma_mem(isert_cmd, isert_conn);
1687                 transport_generic_free_cmd(&cmd->se_cmd, 0);
1688                 break;
1689         case ISCSI_OP_SCSI_TMFUNC:
1690                 spin_lock_bh(&conn->cmd_lock);
1691                 if (!list_empty(&cmd->i_conn_node))
1692                         list_del_init(&cmd->i_conn_node);
1693                 spin_unlock_bh(&conn->cmd_lock);
1694
1695                 transport_generic_free_cmd(&cmd->se_cmd, 0);
1696                 break;
1697         case ISCSI_OP_REJECT:
1698         case ISCSI_OP_NOOP_OUT:
1699         case ISCSI_OP_TEXT:
1700                 spin_lock_bh(&conn->cmd_lock);
1701                 if (!list_empty(&cmd->i_conn_node))
1702                         list_del_init(&cmd->i_conn_node);
1703                 spin_unlock_bh(&conn->cmd_lock);
1704
1705                 /*
1706                  * Handle special case for REJECT when iscsi_add_reject*() has
1707                  * overwritten the original iscsi_opcode assignment, and the
1708                  * associated cmd->se_cmd needs to be released.
1709                  */
1710                 if (cmd->se_cmd.se_tfo != NULL) {
1711                         pr_debug("Calling transport_generic_free_cmd from"
1712                                  " isert_put_cmd for 0x%02x\n",
1713                                  cmd->iscsi_opcode);
1714                         transport_generic_free_cmd(&cmd->se_cmd, 0);
1715                         break;
1716                 }
1717                 /*
1718                  * Fall-through
1719                  */
1720         default:
1721                 iscsit_release_cmd(cmd);
1722                 break;
1723         }
1724 }
1725
1726 static void
1727 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1728 {
1729         if (tx_desc->dma_addr != 0) {
1730                 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
1731                 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1732                                     ISER_HEADERS_LEN, DMA_TO_DEVICE);
1733                 tx_desc->dma_addr = 0;
1734         }
1735 }
1736
1737 static void
1738 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
1739                      struct ib_device *ib_dev, bool comp_err)
1740 {
1741         if (isert_cmd->pdu_buf_dma != 0) {
1742                 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n");
1743                 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,
1744                                     isert_cmd->pdu_buf_len, DMA_TO_DEVICE);
1745                 isert_cmd->pdu_buf_dma = 0;
1746         }
1747
1748         isert_unmap_tx_desc(tx_desc, ib_dev);
1749         isert_put_cmd(isert_cmd, comp_err);
1750 }
1751
1752 static int
1753 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
1754 {
1755         struct ib_mr_status mr_status;
1756         int ret;
1757
1758         ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
1759         if (ret) {
1760                 pr_err("ib_check_mr_status failed, ret %d\n", ret);
1761                 goto fail_mr_status;
1762         }
1763
1764         if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
1765                 u64 sec_offset_err;
1766                 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8;
1767
1768                 switch (mr_status.sig_err.err_type) {
1769                 case IB_SIG_BAD_GUARD:
1770                         se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED;
1771                         break;
1772                 case IB_SIG_BAD_REFTAG:
1773                         se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED;
1774                         break;
1775                 case IB_SIG_BAD_APPTAG:
1776                         se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED;
1777                         break;
1778                 }
1779                 sec_offset_err = mr_status.sig_err.sig_err_offset;
1780                 do_div(sec_offset_err, block_size);
1781                 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba;
1782
1783                 pr_err("isert: PI error found type %d at sector 0x%llx "
1784                        "expected 0x%x vs actual 0x%x\n",
1785                        mr_status.sig_err.err_type,
1786                        (unsigned long long)se_cmd->bad_sector,
1787                        mr_status.sig_err.expected,
1788                        mr_status.sig_err.actual);
1789                 ret = 1;
1790         }
1791
1792 fail_mr_status:
1793         return ret;
1794 }
1795
1796 static void
1797 isert_completion_rdma_write(struct iser_tx_desc *tx_desc,
1798                             struct isert_cmd *isert_cmd)
1799 {
1800         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1801         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1802         struct se_cmd *se_cmd = &cmd->se_cmd;
1803         struct isert_conn *isert_conn = isert_cmd->conn;
1804         struct isert_device *device = isert_conn->conn_device;
1805         int ret = 0;
1806
1807         if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {
1808                 ret = isert_check_pi_status(se_cmd,
1809                                             wr->fr_desc->pi_ctx->sig_mr);
1810                 wr->fr_desc->ind &= ~ISERT_PROTECTED;
1811         }
1812
1813         device->unreg_rdma_mem(isert_cmd, isert_conn);
1814         wr->send_wr_num = 0;
1815         if (ret)
1816                 transport_send_check_condition_and_sense(se_cmd,
1817                                                          se_cmd->pi_err, 0);
1818         else
1819                 isert_put_response(isert_conn->conn, cmd);
1820 }
1821
1822 static void
1823 isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
1824                            struct isert_cmd *isert_cmd)
1825 {
1826         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1827         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1828         struct se_cmd *se_cmd = &cmd->se_cmd;
1829         struct isert_conn *isert_conn = isert_cmd->conn;
1830         struct isert_device *device = isert_conn->conn_device;
1831         int ret = 0;
1832
1833         if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {
1834                 ret = isert_check_pi_status(se_cmd,
1835                                             wr->fr_desc->pi_ctx->sig_mr);
1836                 wr->fr_desc->ind &= ~ISERT_PROTECTED;
1837         }
1838
1839         iscsit_stop_dataout_timer(cmd);
1840         device->unreg_rdma_mem(isert_cmd, isert_conn);
1841         cmd->write_data_done = wr->data.len;
1842         wr->send_wr_num = 0;
1843
1844         pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);
1845         spin_lock_bh(&cmd->istate_lock);
1846         cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1847         cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1848         spin_unlock_bh(&cmd->istate_lock);
1849
1850         if (ret)
1851                 transport_send_check_condition_and_sense(se_cmd,
1852                                                          se_cmd->pi_err, 0);
1853         else
1854                 target_execute_cmd(se_cmd);
1855 }
1856
1857 static void
1858 isert_do_control_comp(struct work_struct *work)
1859 {
1860         struct isert_cmd *isert_cmd = container_of(work,
1861                         struct isert_cmd, comp_work);
1862         struct isert_conn *isert_conn = isert_cmd->conn;
1863         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1864         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1865
1866         switch (cmd->i_state) {
1867         case ISTATE_SEND_TASKMGTRSP:
1868                 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");
1869
1870                 iscsit_tmr_post_handler(cmd, cmd->conn);
1871
1872                 cmd->i_state = ISTATE_SENT_STATUS;
1873                 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false);
1874                 break;
1875         case ISTATE_SEND_REJECT:
1876                 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
1877
1878                 cmd->i_state = ISTATE_SENT_STATUS;
1879                 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false);
1880                 break;
1881         case ISTATE_SEND_LOGOUTRSP:
1882                 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
1883
1884                 iscsit_logout_post_handler(cmd, cmd->conn);
1885                 break;
1886         case ISTATE_SEND_TEXTRSP:
1887                 cmd->i_state = ISTATE_SENT_STATUS;
1888                 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false);
1889                 break;
1890         default:
1891                 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state);
1892                 dump_stack();
1893                 break;
1894         }
1895 }
1896
1897 static void
1898 isert_response_completion(struct iser_tx_desc *tx_desc,
1899                           struct isert_cmd *isert_cmd,
1900                           struct isert_conn *isert_conn,
1901                           struct ib_device *ib_dev)
1902 {
1903         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1904
1905         if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
1906             cmd->i_state == ISTATE_SEND_LOGOUTRSP ||
1907             cmd->i_state == ISTATE_SEND_REJECT ||
1908             cmd->i_state == ISTATE_SEND_TEXTRSP) {
1909                 isert_unmap_tx_desc(tx_desc, ib_dev);
1910
1911                 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1912                 queue_work(isert_comp_wq, &isert_cmd->comp_work);
1913                 return;
1914         }
1915
1916         cmd->i_state = ISTATE_SENT_STATUS;
1917         isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
1918 }
1919
1920 static void
1921 isert_send_completion(struct iser_tx_desc *tx_desc,
1922                       struct isert_conn *isert_conn)
1923 {
1924         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1925         struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
1926         struct isert_rdma_wr *wr;
1927
1928         if (!isert_cmd) {
1929                 isert_unmap_tx_desc(tx_desc, ib_dev);
1930                 return;
1931         }
1932         wr = &isert_cmd->rdma_wr;
1933
1934         switch (wr->iser_ib_op) {
1935         case ISER_IB_RECV:
1936                 pr_err("isert_send_completion: Got ISER_IB_RECV\n");
1937                 dump_stack();
1938                 break;
1939         case ISER_IB_SEND:
1940                 pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
1941                 isert_response_completion(tx_desc, isert_cmd,
1942                                           isert_conn, ib_dev);
1943                 break;
1944         case ISER_IB_RDMA_WRITE:
1945                 pr_debug("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
1946                 isert_completion_rdma_write(tx_desc, isert_cmd);
1947                 break;
1948         case ISER_IB_RDMA_READ:
1949                 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");
1950
1951                 isert_completion_rdma_read(tx_desc, isert_cmd);
1952                 break;
1953         default:
1954                 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op);
1955                 dump_stack();
1956                 break;
1957         }
1958 }
1959
1960 /**
1961  * is_isert_tx_desc() - Indicate if the completion wr_id
1962  *     is a TX descriptor or not.
1963  * @isert_conn: iser connection
1964  * @wr_id: completion WR identifier
1965  *
1966  * Since we cannot rely on wc opcode in FLUSH errors
1967  * we must work around it by checking if the wr_id address
1968  * falls in the iser connection rx_descs buffer. If so
1969  * it is an RX descriptor, otherwize it is a TX.
1970  */
1971 static inline bool
1972 is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id)
1973 {
1974         void *start = isert_conn->conn_rx_descs;
1975         int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->conn_rx_descs);
1976
1977         if (wr_id >= start && wr_id < start + len)
1978                 return false;
1979
1980         return true;
1981 }
1982
1983 static void
1984 isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc)
1985 {
1986         if (wc->wr_id == ISER_BEACON_WRID) {
1987                 struct iscsi_conn *conn = isert_conn->conn;
1988
1989                 if (conn->sess) {
1990                         target_sess_cmd_list_set_waiting(conn->sess->se_sess);
1991                         target_wait_for_sess_cmds(conn->sess->se_sess);
1992                 }
1993
1994                 pr_info("conn %p completing conn_wait_comp_err\n",
1995                            isert_conn);
1996                 complete(&isert_conn->conn_wait_comp_err);
1997         } else
1998         if (is_isert_tx_desc(isert_conn, (void *)wc->wr_id)) {
1999                 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
2000                 struct isert_cmd *isert_cmd;
2001                 struct iser_tx_desc *desc;
2002
2003                 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id;
2004                 isert_cmd = desc->isert_cmd;
2005                 if (!isert_cmd)
2006                         isert_unmap_tx_desc(desc, ib_dev);
2007                 else
2008                         isert_completion_put(desc, isert_cmd, ib_dev, true);
2009         } else {
2010                 isert_conn->post_recv_buf_count--;
2011                 if (!isert_conn->post_recv_buf_count)
2012                         iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
2013         }
2014 }
2015
2016 static void
2017 isert_handle_wc(struct ib_wc *wc)
2018 {
2019         struct isert_conn *isert_conn;
2020         struct iser_tx_desc *tx_desc;
2021         struct iser_rx_desc *rx_desc;
2022
2023         isert_conn = wc->qp->qp_context;
2024         if (likely(wc->status == IB_WC_SUCCESS)) {
2025                 if (wc->opcode == IB_WC_RECV) {
2026                         rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id;
2027                         isert_rx_completion(rx_desc, isert_conn, wc->byte_len);
2028                 } else {
2029                         tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id;
2030                         isert_send_completion(tx_desc, isert_conn);
2031                 }
2032         } else {
2033                 if (wc->status != IB_WC_WR_FLUSH_ERR)
2034                         pr_err("wr id %llx status %d vend_err %x\n",
2035                                   wc->wr_id, wc->status, wc->vendor_err);
2036                 else
2037                         pr_debug("flush error: wr id %llx\n", wc->wr_id);
2038
2039                 if (wc->wr_id != ISER_FASTREG_LI_WRID)
2040                         isert_cq_comp_err(isert_conn, wc);
2041         }
2042 }
2043
2044 static void
2045 isert_cq_work(struct work_struct *work)
2046 {
2047         enum { isert_poll_budget = 65536 };
2048         struct isert_comp *comp = container_of(work, struct isert_comp,
2049                                                work);
2050         struct ib_wc *const wcs = comp->wcs;
2051         int i, n, completed = 0;
2052
2053         while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) {
2054                 for (i = 0; i < n; i++)
2055                         isert_handle_wc(&wcs[i]);
2056
2057                 completed += n;
2058                 if (completed >= isert_poll_budget)
2059                         break;
2060         }
2061
2062         ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP);
2063 }
2064
2065 static void
2066 isert_cq_callback(struct ib_cq *cq, void *context)
2067 {
2068         struct isert_comp *comp = context;
2069
2070         queue_work(isert_comp_wq, &comp->work);
2071 }
2072
2073 static int
2074 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
2075 {
2076         struct ib_send_wr *wr_failed;
2077         int ret;
2078
2079         ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr,
2080                            &wr_failed);
2081         if (ret) {
2082                 pr_err("ib_post_send failed with %d\n", ret);
2083                 return ret;
2084         }
2085         return ret;
2086 }
2087
2088 static int
2089 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2090 {
2091         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2092         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2093         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2094         struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
2095                                 &isert_cmd->tx_desc.iscsi_header;
2096
2097         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2098         iscsit_build_rsp_pdu(cmd, conn, true, hdr);
2099         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2100         /*
2101          * Attach SENSE DATA payload to iSCSI Response PDU
2102          */
2103         if (cmd->se_cmd.sense_buffer &&
2104             ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
2105             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
2106                 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
2107                 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
2108                 u32 padding, pdu_len;
2109
2110                 put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
2111                                    cmd->sense_buffer);
2112                 cmd->se_cmd.scsi_sense_length += sizeof(__be16);
2113
2114                 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
2115                 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
2116                 pdu_len = cmd->se_cmd.scsi_sense_length + padding;
2117
2118                 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2119                                 (void *)cmd->sense_buffer, pdu_len,
2120                                 DMA_TO_DEVICE);
2121
2122                 isert_cmd->pdu_buf_len = pdu_len;
2123                 tx_dsg->addr    = isert_cmd->pdu_buf_dma;
2124                 tx_dsg->length  = pdu_len;
2125                 tx_dsg->lkey    = isert_conn->conn_mr->lkey;
2126                 isert_cmd->tx_desc.num_sge = 2;
2127         }
2128
2129         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2130
2131         pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
2132
2133         return isert_post_response(isert_conn, isert_cmd);
2134 }
2135
2136 static void
2137 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2138 {
2139         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2140         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2141         struct isert_device *device = isert_conn->conn_device;
2142
2143         spin_lock_bh(&conn->cmd_lock);
2144         if (!list_empty(&cmd->i_conn_node))
2145                 list_del_init(&cmd->i_conn_node);
2146         spin_unlock_bh(&conn->cmd_lock);
2147
2148         if (cmd->data_direction == DMA_TO_DEVICE)
2149                 iscsit_stop_dataout_timer(cmd);
2150
2151         device->unreg_rdma_mem(isert_cmd, isert_conn);
2152 }
2153
2154 static enum target_prot_op
2155 isert_get_sup_prot_ops(struct iscsi_conn *conn)
2156 {
2157         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2158         struct isert_device *device = isert_conn->conn_device;
2159
2160         if (conn->tpg->tpg_attrib.t10_pi) {
2161                 if (device->pi_capable) {
2162                         pr_info("conn %p PI offload enabled\n", isert_conn);
2163                         isert_conn->pi_support = true;
2164                         return TARGET_PROT_ALL;
2165                 }
2166         }
2167
2168         pr_info("conn %p PI offload disabled\n", isert_conn);
2169         isert_conn->pi_support = false;
2170
2171         return TARGET_PROT_NORMAL;
2172 }
2173
2174 static int
2175 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2176                 bool nopout_response)
2177 {
2178         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2179         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2180         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2181
2182         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2183         iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
2184                                &isert_cmd->tx_desc.iscsi_header,
2185                                nopout_response);
2186         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2187         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2188
2189         pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
2190
2191         return isert_post_response(isert_conn, isert_cmd);
2192 }
2193
2194 static int
2195 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2196 {
2197         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2198         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2199         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2200
2201         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2202         iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
2203                                 &isert_cmd->tx_desc.iscsi_header);
2204         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2205         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2206
2207         pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
2208
2209         return isert_post_response(isert_conn, isert_cmd);
2210 }
2211
2212 static int
2213 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2214 {
2215         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2216         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2217         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2218
2219         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2220         iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
2221                                   &isert_cmd->tx_desc.iscsi_header);
2222         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2223         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2224
2225         pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
2226
2227         return isert_post_response(isert_conn, isert_cmd);
2228 }
2229
2230 static int
2231 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2232 {
2233         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2234         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2235         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2236         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
2237         struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
2238         struct iscsi_reject *hdr =
2239                 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
2240
2241         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2242         iscsit_build_reject(cmd, conn, hdr);
2243         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2244
2245         hton24(hdr->dlength, ISCSI_HDR_LEN);
2246         isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2247                         (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
2248                         DMA_TO_DEVICE);
2249         isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
2250         tx_dsg->addr    = isert_cmd->pdu_buf_dma;
2251         tx_dsg->length  = ISCSI_HDR_LEN;
2252         tx_dsg->lkey    = isert_conn->conn_mr->lkey;
2253         isert_cmd->tx_desc.num_sge = 2;
2254
2255         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2256
2257         pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
2258
2259         return isert_post_response(isert_conn, isert_cmd);
2260 }
2261
2262 static int
2263 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2264 {
2265         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2266         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2267         struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2268         struct iscsi_text_rsp *hdr =
2269                 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
2270         u32 txt_rsp_len;
2271         int rc;
2272
2273         isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2274         rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);
2275         if (rc < 0)
2276                 return rc;
2277
2278         txt_rsp_len = rc;
2279         isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2280
2281         if (txt_rsp_len) {
2282                 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
2283                 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
2284                 void *txt_rsp_buf = cmd->buf_ptr;
2285
2286                 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2287                                 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
2288
2289                 isert_cmd->pdu_buf_len = txt_rsp_len;
2290                 tx_dsg->addr    = isert_cmd->pdu_buf_dma;
2291                 tx_dsg->length  = txt_rsp_len;
2292                 tx_dsg->lkey    = isert_conn->conn_mr->lkey;
2293                 isert_cmd->tx_desc.num_sge = 2;
2294         }
2295         isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2296
2297         pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
2298
2299         return isert_post_response(isert_conn, isert_cmd);
2300 }
2301
2302 static int
2303 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
2304                     struct ib_sge *ib_sge, struct ib_send_wr *send_wr,
2305                     u32 data_left, u32 offset)
2306 {
2307         struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
2308         struct scatterlist *sg_start, *tmp_sg;
2309         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
2310         u32 sg_off, page_off;
2311         int i = 0, sg_nents;
2312
2313         sg_off = offset / PAGE_SIZE;
2314         sg_start = &cmd->se_cmd.t_data_sg[sg_off];
2315         sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
2316         page_off = offset % PAGE_SIZE;
2317
2318         send_wr->sg_list = ib_sge;
2319         send_wr->num_sge = sg_nents;
2320         send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;
2321         /*
2322          * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
2323          */
2324         for_each_sg(sg_start, tmp_sg, sg_nents, i) {
2325                 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n",
2326                          (unsigned long long)tmp_sg->dma_address,
2327                          tmp_sg->length, page_off);
2328
2329                 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off;
2330                 ib_sge->length = min_t(u32, data_left,
2331                                 ib_sg_dma_len(ib_dev, tmp_sg) - page_off);
2332                 ib_sge->lkey = isert_conn->conn_mr->lkey;
2333
2334                 pr_debug("RDMA ib_sge: addr: 0x%16llx  length: %u lkey: %08x\n",
2335                          ib_sge->addr, ib_sge->length, ib_sge->lkey);
2336                 page_off = 0;
2337                 data_left -= ib_sge->length;
2338                 ib_sge++;
2339                 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge);
2340         }
2341
2342         pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
2343                  send_wr->sg_list, send_wr->num_sge);
2344
2345         return sg_nents;
2346 }
2347
2348 static int
2349 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2350                struct isert_rdma_wr *wr)
2351 {
2352         struct se_cmd *se_cmd = &cmd->se_cmd;
2353         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2354         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2355         struct isert_data_buf *data = &wr->data;
2356         struct ib_send_wr *send_wr;
2357         struct ib_sge *ib_sge;
2358         u32 offset, data_len, data_left, rdma_write_max, va_offset = 0;
2359         int ret = 0, i, ib_sge_cnt;
2360
2361         isert_cmd->tx_desc.isert_cmd = isert_cmd;
2362
2363         offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;
2364         ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,
2365                                  se_cmd->t_data_nents, se_cmd->data_length,
2366                                  offset, wr->iser_ib_op, &wr->data);
2367         if (ret)
2368                 return ret;
2369
2370         data_left = data->len;
2371         offset = data->offset;
2372
2373         ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL);
2374         if (!ib_sge) {
2375                 pr_warn("Unable to allocate ib_sge\n");
2376                 ret = -ENOMEM;
2377                 goto unmap_cmd;
2378         }
2379         wr->ib_sge = ib_sge;
2380
2381         wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge);
2382         wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
2383                                 GFP_KERNEL);
2384         if (!wr->send_wr) {
2385                 pr_debug("Unable to allocate wr->send_wr\n");
2386                 ret = -ENOMEM;
2387                 goto unmap_cmd;
2388         }
2389
2390         wr->isert_cmd = isert_cmd;
2391         rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
2392
2393         for (i = 0; i < wr->send_wr_num; i++) {
2394                 send_wr = &isert_cmd->rdma_wr.send_wr[i];
2395                 data_len = min(data_left, rdma_write_max);
2396
2397                 send_wr->send_flags = 0;
2398                 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {
2399                         send_wr->opcode = IB_WR_RDMA_WRITE;
2400                         send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;
2401                         send_wr->wr.rdma.rkey = isert_cmd->read_stag;
2402                         if (i + 1 == wr->send_wr_num)
2403                                 send_wr->next = &isert_cmd->tx_desc.send_wr;
2404                         else
2405                                 send_wr->next = &wr->send_wr[i + 1];
2406                 } else {
2407                         send_wr->opcode = IB_WR_RDMA_READ;
2408                         send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;
2409                         send_wr->wr.rdma.rkey = isert_cmd->write_stag;
2410                         if (i + 1 == wr->send_wr_num)
2411                                 send_wr->send_flags = IB_SEND_SIGNALED;
2412                         else
2413                                 send_wr->next = &wr->send_wr[i + 1];
2414                 }
2415
2416                 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
2417                                         send_wr, data_len, offset);
2418                 ib_sge += ib_sge_cnt;
2419
2420                 offset += data_len;
2421                 va_offset += data_len;
2422                 data_left -= data_len;
2423         }
2424
2425         return 0;
2426 unmap_cmd:
2427         isert_unmap_data_buf(isert_conn, data);
2428
2429         return ret;
2430 }
2431
2432 static int
2433 isert_map_fr_pagelist(struct ib_device *ib_dev,
2434                       struct scatterlist *sg_start, int sg_nents, u64 *fr_pl)
2435 {
2436         u64 start_addr, end_addr, page, chunk_start = 0;
2437         struct scatterlist *tmp_sg;
2438         int i = 0, new_chunk, last_ent, n_pages;
2439
2440         n_pages = 0;
2441         new_chunk = 1;
2442         last_ent = sg_nents - 1;
2443         for_each_sg(sg_start, tmp_sg, sg_nents, i) {
2444                 start_addr = ib_sg_dma_address(ib_dev, tmp_sg);
2445                 if (new_chunk)
2446                         chunk_start = start_addr;
2447                 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg);
2448
2449                 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n",
2450                          i, (unsigned long long)tmp_sg->dma_address,
2451                          tmp_sg->length);
2452
2453                 if ((end_addr & ~PAGE_MASK) && i < last_ent) {
2454                         new_chunk = 0;
2455                         continue;
2456                 }
2457                 new_chunk = 1;
2458
2459                 page = chunk_start & PAGE_MASK;
2460                 do {
2461                         fr_pl[n_pages++] = page;
2462                         pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n",
2463                                  n_pages - 1, page);
2464                         page += PAGE_SIZE;
2465                 } while (page < end_addr);
2466         }
2467
2468         return n_pages;
2469 }
2470
2471 static int
2472 isert_fast_reg_mr(struct isert_conn *isert_conn,
2473                   struct fast_reg_descriptor *fr_desc,
2474                   struct isert_data_buf *mem,
2475                   enum isert_indicator ind,
2476                   struct ib_sge *sge)
2477 {
2478         struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
2479         struct ib_mr *mr;
2480         struct ib_fast_reg_page_list *frpl;
2481         struct ib_send_wr fr_wr, inv_wr;
2482         struct ib_send_wr *bad_wr, *wr = NULL;
2483         int ret, pagelist_len;
2484         u32 page_off;
2485         u8 key;
2486
2487         if (mem->dma_nents == 1) {
2488                 sge->lkey = isert_conn->conn_mr->lkey;
2489                 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]);
2490                 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]);
2491                 pr_debug("%s:%d sge: addr: 0x%llx  length: %u lkey: %x\n",
2492                          __func__, __LINE__, sge->addr, sge->length,
2493                          sge->lkey);
2494                 return 0;
2495         }
2496
2497         if (ind == ISERT_DATA_KEY_VALID) {
2498                 /* Registering data buffer */
2499                 mr = fr_desc->data_mr;
2500                 frpl = fr_desc->data_frpl;
2501         } else {
2502                 /* Registering protection buffer */
2503                 mr = fr_desc->pi_ctx->prot_mr;
2504                 frpl = fr_desc->pi_ctx->prot_frpl;
2505         }
2506
2507         page_off = mem->offset % PAGE_SIZE;
2508
2509         pr_debug("Use fr_desc %p sg_nents %d offset %u\n",
2510                  fr_desc, mem->nents, mem->offset);
2511
2512         pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents,
2513                                              &frpl->page_list[0]);
2514
2515         if (!(fr_desc->ind & ISERT_DATA_KEY_VALID)) {
2516                 memset(&inv_wr, 0, sizeof(inv_wr));
2517                 inv_wr.wr_id = ISER_FASTREG_LI_WRID;
2518                 inv_wr.opcode = IB_WR_LOCAL_INV;
2519                 inv_wr.ex.invalidate_rkey = mr->rkey;
2520                 wr = &inv_wr;
2521                 /* Bump the key */
2522                 key = (u8)(mr->rkey & 0x000000FF);
2523                 ib_update_fast_reg_key(mr, ++key);
2524         }
2525
2526         /* Prepare FASTREG WR */
2527         memset(&fr_wr, 0, sizeof(fr_wr));
2528         fr_wr.wr_id = ISER_FASTREG_LI_WRID;
2529         fr_wr.opcode = IB_WR_FAST_REG_MR;
2530         fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off;
2531         fr_wr.wr.fast_reg.page_list = frpl;
2532         fr_wr.wr.fast_reg.page_list_len = pagelist_len;
2533         fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
2534         fr_wr.wr.fast_reg.length = mem->len;
2535         fr_wr.wr.fast_reg.rkey = mr->rkey;
2536         fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE;
2537
2538         if (!wr)
2539                 wr = &fr_wr;
2540         else
2541                 wr->next = &fr_wr;
2542
2543         ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr);
2544         if (ret) {
2545                 pr_err("fast registration failed, ret:%d\n", ret);
2546                 return ret;
2547         }
2548         fr_desc->ind &= ~ind;
2549
2550         sge->lkey = mr->lkey;
2551         sge->addr = frpl->page_list[0] + page_off;
2552         sge->length = mem->len;
2553
2554         pr_debug("%s:%d sge: addr: 0x%llx  length: %u lkey: %x\n",
2555                  __func__, __LINE__, sge->addr, sge->length,
2556                  sge->lkey);
2557
2558         return ret;
2559 }
2560
2561 static inline void
2562 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs,
2563                      struct ib_sig_domain *domain)
2564 {
2565         domain->sig_type = IB_SIG_TYPE_T10_DIF;
2566         domain->sig.dif.bg_type = IB_T10DIF_CRC;
2567         domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
2568         domain->sig.dif.ref_tag = se_cmd->reftag_seed;
2569         /*
2570          * At the moment we hard code those, but if in the future
2571          * the target core would like to use it, we will take it
2572          * from se_cmd.
2573          */
2574         domain->sig.dif.apptag_check_mask = 0xffff;
2575         domain->sig.dif.app_escape = true;
2576         domain->sig.dif.ref_escape = true;
2577         if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
2578             se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
2579                 domain->sig.dif.ref_remap = true;
2580 };
2581
2582 static int
2583 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
2584 {
2585         switch (se_cmd->prot_op) {
2586         case TARGET_PROT_DIN_INSERT:
2587         case TARGET_PROT_DOUT_STRIP:
2588                 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
2589                 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
2590                 break;
2591         case TARGET_PROT_DOUT_INSERT:
2592         case TARGET_PROT_DIN_STRIP:
2593                 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
2594                 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
2595                 break;
2596         case TARGET_PROT_DIN_PASS:
2597         case TARGET_PROT_DOUT_PASS:
2598                 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
2599                 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
2600                 break;
2601         default:
2602                 pr_err("Unsupported PI operation %d\n", se_cmd->prot_op);
2603                 return -EINVAL;
2604         }
2605
2606         return 0;
2607 }
2608
2609 static inline u8
2610 isert_set_prot_checks(u8 prot_checks)
2611 {
2612         return (prot_checks & TARGET_DIF_CHECK_GUARD  ? 0xc0 : 0) |
2613                (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) |
2614                (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0);
2615 }
2616
2617 static int
2618 isert_reg_sig_mr(struct isert_conn *isert_conn,
2619                  struct se_cmd *se_cmd,
2620                  struct isert_rdma_wr *rdma_wr,
2621                  struct fast_reg_descriptor *fr_desc)
2622 {
2623         struct ib_send_wr sig_wr, inv_wr;
2624         struct ib_send_wr *bad_wr, *wr = NULL;
2625         struct pi_context *pi_ctx = fr_desc->pi_ctx;
2626         struct ib_sig_attrs sig_attrs;
2627         int ret;
2628         u32 key;
2629
2630         memset(&sig_attrs, 0, sizeof(sig_attrs));
2631         ret = isert_set_sig_attrs(se_cmd, &sig_attrs);
2632         if (ret)
2633                 goto err;
2634
2635         sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks);
2636
2637         if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) {
2638                 memset(&inv_wr, 0, sizeof(inv_wr));
2639                 inv_wr.opcode = IB_WR_LOCAL_INV;
2640                 inv_wr.wr_id = ISER_FASTREG_LI_WRID;
2641                 inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey;
2642                 wr = &inv_wr;
2643                 /* Bump the key */
2644                 key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF);
2645                 ib_update_fast_reg_key(pi_ctx->sig_mr, ++key);
2646         }
2647
2648         memset(&sig_wr, 0, sizeof(sig_wr));
2649         sig_wr.opcode = IB_WR_REG_SIG_MR;
2650         sig_wr.wr_id = ISER_FASTREG_LI_WRID;
2651         sig_wr.sg_list = &rdma_wr->ib_sg[DATA];
2652         sig_wr.num_sge = 1;
2653         sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE;
2654         sig_wr.wr.sig_handover.sig_attrs = &sig_attrs;
2655         sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr;
2656         if (se_cmd->t_prot_sg)
2657                 sig_wr.wr.sig_handover.prot = &rdma_wr->ib_sg[PROT];
2658
2659         if (!wr)
2660                 wr = &sig_wr;
2661         else
2662                 wr->next = &sig_wr;
2663
2664         ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr);
2665         if (ret) {
2666                 pr_err("fast registration failed, ret:%d\n", ret);
2667                 goto err;
2668         }
2669         fr_desc->ind &= ~ISERT_SIG_KEY_VALID;
2670
2671         rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey;
2672         rdma_wr->ib_sg[SIG].addr = 0;
2673         rdma_wr->ib_sg[SIG].length = se_cmd->data_length;
2674         if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP &&
2675             se_cmd->prot_op != TARGET_PROT_DOUT_INSERT)
2676                 /*
2677                  * We have protection guards on the wire
2678                  * so we need to set a larget transfer
2679                  */
2680                 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length;
2681
2682         pr_debug("sig_sge: addr: 0x%llx  length: %u lkey: %x\n",
2683                   rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length,
2684                   rdma_wr->ib_sg[SIG].lkey);
2685 err:
2686         return ret;
2687 }
2688
2689 static int
2690 isert_handle_prot_cmd(struct isert_conn *isert_conn,
2691                       struct isert_cmd *isert_cmd,
2692                       struct isert_rdma_wr *wr)
2693 {
2694         struct isert_device *device = isert_conn->conn_device;
2695         struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd;
2696         int ret;
2697
2698         if (!wr->fr_desc->pi_ctx) {
2699                 ret = isert_create_pi_ctx(wr->fr_desc,
2700                                           device->ib_device,
2701                                           isert_conn->conn_pd);
2702                 if (ret) {
2703                         pr_err("conn %p failed to allocate pi_ctx\n",
2704                                   isert_conn);
2705                         return ret;
2706                 }
2707         }
2708
2709         if (se_cmd->t_prot_sg) {
2710                 ret = isert_map_data_buf(isert_conn, isert_cmd,
2711                                          se_cmd->t_prot_sg,
2712                                          se_cmd->t_prot_nents,
2713                                          se_cmd->prot_length,
2714                                          0, wr->iser_ib_op, &wr->prot);
2715                 if (ret) {
2716                         pr_err("conn %p failed to map protection buffer\n",
2717                                   isert_conn);
2718                         return ret;
2719                 }
2720
2721                 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT]));
2722                 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot,
2723                                         ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]);
2724                 if (ret) {
2725                         pr_err("conn %p failed to fast reg mr\n",
2726                                   isert_conn);
2727                         goto unmap_prot_cmd;
2728                 }
2729         }
2730
2731         ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc);
2732         if (ret) {
2733                 pr_err("conn %p failed to fast reg mr\n",
2734                           isert_conn);
2735                 goto unmap_prot_cmd;
2736         }
2737         wr->fr_desc->ind |= ISERT_PROTECTED;
2738
2739         return 0;
2740
2741 unmap_prot_cmd:
2742         if (se_cmd->t_prot_sg)
2743                 isert_unmap_data_buf(isert_conn, &wr->prot);
2744
2745         return ret;
2746 }
2747
2748 static int
2749 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2750                struct isert_rdma_wr *wr)
2751 {
2752         struct se_cmd *se_cmd = &cmd->se_cmd;
2753         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2754         struct isert_conn *isert_conn = conn->context;
2755         struct fast_reg_descriptor *fr_desc = NULL;
2756         struct ib_send_wr *send_wr;
2757         struct ib_sge *ib_sg;
2758         u32 offset;
2759         int ret = 0;
2760         unsigned long flags;
2761
2762         isert_cmd->tx_desc.isert_cmd = isert_cmd;
2763
2764         offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;
2765         ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,
2766                                  se_cmd->t_data_nents, se_cmd->data_length,
2767                                  offset, wr->iser_ib_op, &wr->data);
2768         if (ret)
2769                 return ret;
2770
2771         if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) {
2772                 spin_lock_irqsave(&isert_conn->conn_lock, flags);
2773                 fr_desc = list_first_entry(&isert_conn->conn_fr_pool,
2774                                            struct fast_reg_descriptor, list);
2775                 list_del(&fr_desc->list);
2776                 spin_unlock_irqrestore(&isert_conn->conn_lock, flags);
2777                 wr->fr_desc = fr_desc;
2778         }
2779
2780         ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data,
2781                                 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]);
2782         if (ret)
2783                 goto unmap_cmd;
2784
2785         if (isert_prot_cmd(isert_conn, se_cmd)) {
2786                 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr);
2787                 if (ret)
2788                         goto unmap_cmd;
2789
2790                 ib_sg = &wr->ib_sg[SIG];
2791         } else {
2792                 ib_sg = &wr->ib_sg[DATA];
2793         }
2794
2795         memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg));
2796         wr->ib_sge = &wr->s_ib_sge;
2797         wr->send_wr_num = 1;
2798         memset(&wr->s_send_wr, 0, sizeof(*send_wr));
2799         wr->send_wr = &wr->s_send_wr;
2800         wr->isert_cmd = isert_cmd;
2801
2802         send_wr = &isert_cmd->rdma_wr.s_send_wr;
2803         send_wr->sg_list = &wr->s_ib_sge;
2804         send_wr->num_sge = 1;
2805         send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;
2806         if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {
2807                 send_wr->opcode = IB_WR_RDMA_WRITE;
2808                 send_wr->wr.rdma.remote_addr = isert_cmd->read_va;
2809                 send_wr->wr.rdma.rkey = isert_cmd->read_stag;
2810                 send_wr->send_flags = !isert_prot_cmd(isert_conn, se_cmd) ?
2811                                       0 : IB_SEND_SIGNALED;
2812         } else {
2813                 send_wr->opcode = IB_WR_RDMA_READ;
2814                 send_wr->wr.rdma.remote_addr = isert_cmd->write_va;
2815                 send_wr->wr.rdma.rkey = isert_cmd->write_stag;
2816                 send_wr->send_flags = IB_SEND_SIGNALED;
2817         }
2818
2819         return 0;
2820
2821 unmap_cmd:
2822         if (fr_desc) {
2823                 spin_lock_irqsave(&isert_conn->conn_lock, flags);
2824                 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool);
2825                 spin_unlock_irqrestore(&isert_conn->conn_lock, flags);
2826         }
2827         isert_unmap_data_buf(isert_conn, &wr->data);
2828
2829         return ret;
2830 }
2831
2832 static int
2833 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2834 {
2835         struct se_cmd *se_cmd = &cmd->se_cmd;
2836         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2837         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
2838         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2839         struct isert_device *device = isert_conn->conn_device;
2840         struct ib_send_wr *wr_failed;
2841         int rc;
2842
2843         pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n",
2844                  isert_cmd, se_cmd->data_length);
2845         wr->iser_ib_op = ISER_IB_RDMA_WRITE;
2846         rc = device->reg_rdma_mem(conn, cmd, wr);
2847         if (rc) {
2848                 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);
2849                 return rc;
2850         }
2851
2852         if (!isert_prot_cmd(isert_conn, se_cmd)) {
2853                 /*
2854                  * Build isert_conn->tx_desc for iSCSI response PDU and attach
2855                  */
2856                 isert_create_send_desc(isert_conn, isert_cmd,
2857                                        &isert_cmd->tx_desc);
2858                 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)
2859                                      &isert_cmd->tx_desc.iscsi_header);
2860                 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2861                 isert_init_send_wr(isert_conn, isert_cmd,
2862                                    &isert_cmd->tx_desc.send_wr);
2863                 isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr;
2864                 wr->send_wr_num += 1;
2865         }
2866
2867         rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
2868         if (rc)
2869                 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
2870
2871         if (!isert_prot_cmd(isert_conn, se_cmd))
2872                 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data "
2873                          "READ\n", isert_cmd);
2874         else
2875                 pr_debug("Cmd: %p posted RDMA_WRITE for iSER Data READ\n",
2876                          isert_cmd);
2877
2878         return 1;
2879 }
2880
2881 static int
2882 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
2883 {
2884         struct se_cmd *se_cmd = &cmd->se_cmd;
2885         struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2886         struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
2887         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2888         struct isert_device *device = isert_conn->conn_device;
2889         struct ib_send_wr *wr_failed;
2890         int rc;
2891
2892         pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
2893                  isert_cmd, se_cmd->data_length, cmd->write_data_done);
2894         wr->iser_ib_op = ISER_IB_RDMA_READ;
2895         rc = device->reg_rdma_mem(conn, cmd, wr);
2896         if (rc) {
2897                 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);
2898                 return rc;
2899         }
2900
2901         rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
2902         if (rc)
2903                 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
2904
2905         pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n",
2906                  isert_cmd);
2907
2908         return 0;
2909 }
2910
2911 static int
2912 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2913 {
2914         int ret;
2915
2916         switch (state) {
2917         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
2918                 ret = isert_put_nopin(cmd, conn, false);
2919                 break;
2920         default:
2921                 pr_err("Unknown immediate state: 0x%02x\n", state);
2922                 ret = -EINVAL;
2923                 break;
2924         }
2925
2926         return ret;
2927 }
2928
2929 static int
2930 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2931 {
2932         int ret;
2933
2934         switch (state) {
2935         case ISTATE_SEND_LOGOUTRSP:
2936                 ret = isert_put_logout_rsp(cmd, conn);
2937                 if (!ret) {
2938                         pr_debug("Returning iSER Logout -EAGAIN\n");
2939                         ret = -EAGAIN;
2940                 }
2941                 break;
2942         case ISTATE_SEND_NOPIN:
2943                 ret = isert_put_nopin(cmd, conn, true);
2944                 break;
2945         case ISTATE_SEND_TASKMGTRSP:
2946                 ret = isert_put_tm_rsp(cmd, conn);
2947                 break;
2948         case ISTATE_SEND_REJECT:
2949                 ret = isert_put_reject(cmd, conn);
2950                 break;
2951         case ISTATE_SEND_TEXTRSP:
2952                 ret = isert_put_text_rsp(cmd, conn);
2953                 break;
2954         case ISTATE_SEND_STATUS:
2955                 /*
2956                  * Special case for sending non GOOD SCSI status from TX thread
2957                  * context during pre se_cmd excecution failure.
2958                  */
2959                 ret = isert_put_response(conn, cmd);
2960                 break;
2961         default:
2962                 pr_err("Unknown response state: 0x%02x\n", state);
2963                 ret = -EINVAL;
2964                 break;
2965         }
2966
2967         return ret;
2968 }
2969
2970 struct rdma_cm_id *
2971 isert_setup_id(struct isert_np *isert_np)
2972 {
2973         struct iscsi_np *np = isert_np->np;
2974         struct rdma_cm_id *id;
2975         struct sockaddr *sa;
2976         int ret;
2977
2978         sa = (struct sockaddr *)&np->np_sockaddr;
2979         pr_debug("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa);
2980
2981         id = rdma_create_id(isert_cma_handler, isert_np,
2982                             RDMA_PS_TCP, IB_QPT_RC);
2983         if (IS_ERR(id)) {
2984                 pr_err("rdma_create_id() failed: %ld\n", PTR_ERR(id));
2985                 ret = PTR_ERR(id);
2986                 goto out;
2987         }
2988         pr_debug("id %p context %p\n", id, id->context);
2989
2990         ret = rdma_bind_addr(id, sa);
2991         if (ret) {
2992                 pr_err("rdma_bind_addr() failed: %d\n", ret);
2993                 goto out_id;
2994         }
2995
2996         ret = rdma_listen(id, ISERT_RDMA_LISTEN_BACKLOG);
2997         if (ret) {
2998                 pr_err("rdma_listen() failed: %d\n", ret);
2999                 goto out_id;
3000         }
3001
3002         return id;
3003 out_id:
3004         rdma_destroy_id(id);
3005 out:
3006         return ERR_PTR(ret);
3007 }
3008
3009 static int
3010 isert_setup_np(struct iscsi_np *np,
3011                struct __kernel_sockaddr_storage *ksockaddr)
3012 {
3013         struct isert_np *isert_np;
3014         struct rdma_cm_id *isert_lid;
3015         int ret;
3016
3017         isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
3018         if (!isert_np) {
3019                 pr_err("Unable to allocate struct isert_np\n");
3020                 return -ENOMEM;
3021         }
3022         sema_init(&isert_np->np_sem, 0);
3023         mutex_init(&isert_np->np_accept_mutex);
3024         INIT_LIST_HEAD(&isert_np->np_accept_list);
3025         init_completion(&isert_np->np_login_comp);
3026         isert_np->np = np;
3027
3028         /*
3029          * Setup the np->np_sockaddr from the passed sockaddr setup
3030          * in iscsi_target_configfs.c code..
3031          */
3032         memcpy(&np->np_sockaddr, ksockaddr,
3033                sizeof(struct __kernel_sockaddr_storage));
3034
3035         isert_lid = isert_setup_id(isert_np);
3036         if (IS_ERR(isert_lid)) {
3037                 ret = PTR_ERR(isert_lid);
3038                 goto out;
3039         }
3040
3041         isert_np->np_cm_id = isert_lid;
3042         np->np_context = isert_np;
3043
3044         return 0;
3045
3046 out:
3047         kfree(isert_np);
3048
3049         return ret;
3050 }
3051
3052 static int
3053 isert_rdma_accept(struct isert_conn *isert_conn)
3054 {
3055         struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
3056         struct rdma_conn_param cp;
3057         int ret;
3058
3059         memset(&cp, 0, sizeof(struct rdma_conn_param));
3060         cp.initiator_depth = isert_conn->initiator_depth;
3061         cp.retry_count = 7;
3062         cp.rnr_retry_count = 7;
3063
3064         pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n");
3065
3066         ret = rdma_accept(cm_id, &cp);
3067         if (ret) {
3068                 pr_err("rdma_accept() failed with: %d\n", ret);
3069                 return ret;
3070         }
3071
3072         pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n");
3073
3074         return 0;
3075 }
3076
3077 static int
3078 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
3079 {
3080         struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
3081         int ret;
3082
3083         pr_info("before login_req comp conn: %p\n", isert_conn);
3084         ret = wait_for_completion_interruptible(&isert_conn->login_req_comp);
3085         if (ret) {
3086                 pr_err("isert_conn %p interrupted before got login req\n",
3087                           isert_conn);
3088                 return ret;
3089         }
3090         reinit_completion(&isert_conn->login_req_comp);
3091
3092         /*
3093          * For login requests after the first PDU, isert_rx_login_req() will
3094          * kick schedule_delayed_work(&conn->login_work) as the packet is
3095          * received, which turns this callback from iscsi_target_do_login_rx()
3096          * into a NOP.
3097          */
3098         if (!login->first_request)
3099                 return 0;
3100
3101         isert_rx_login_req(isert_conn);
3102
3103         pr_info("before conn_login_comp conn: %p\n", conn);
3104         ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp);
3105         if (ret)
3106                 return ret;
3107
3108         pr_info("processing login->req: %p\n", login->req);
3109
3110         return 0;
3111 }
3112
3113 static void
3114 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
3115                     struct isert_conn *isert_conn)
3116 {
3117         struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
3118         struct rdma_route *cm_route = &cm_id->route;
3119         struct sockaddr_in *sock_in;
3120         struct sockaddr_in6 *sock_in6;
3121
3122         conn->login_family = np->np_sockaddr.ss_family;
3123
3124         if (np->np_sockaddr.ss_family == AF_INET6) {
3125                 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr;
3126                 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
3127                          &sock_in6->sin6_addr.in6_u);
3128                 conn->login_port = ntohs(sock_in6->sin6_port);
3129
3130                 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr;
3131                 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
3132                          &sock_in6->sin6_addr.in6_u);
3133                 conn->local_port = ntohs(sock_in6->sin6_port);
3134         } else {
3135                 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr;
3136                 sprintf(conn->login_ip, "%pI4",
3137                         &sock_in->sin_addr.s_addr);
3138                 conn->login_port = ntohs(sock_in->sin_port);
3139
3140                 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr;
3141                 sprintf(conn->local_ip, "%pI4",
3142                         &sock_in->sin_addr.s_addr);
3143                 conn->local_port = ntohs(sock_in->sin_port);
3144         }
3145 }
3146
3147 static int
3148 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
3149 {
3150         struct isert_np *isert_np = (struct isert_np *)np->np_context;
3151         struct isert_conn *isert_conn;
3152         int max_accept = 0, ret;
3153
3154 accept_wait:
3155         ret = down_interruptible(&isert_np->np_sem);
3156         if (ret || max_accept > 5)
3157                 return -ENODEV;
3158
3159         spin_lock_bh(&np->np_thread_lock);
3160         if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
3161                 spin_unlock_bh(&np->np_thread_lock);
3162                 pr_debug("np_thread_state %d for isert_accept_np\n",
3163                          np->np_thread_state);
3164                 /**
3165                  * No point in stalling here when np_thread
3166                  * is in state RESET/SHUTDOWN/EXIT - bail
3167                  **/
3168                 return -ENODEV;
3169         }
3170         spin_unlock_bh(&np->np_thread_lock);
3171
3172         mutex_lock(&isert_np->np_accept_mutex);
3173         if (list_empty(&isert_np->np_accept_list)) {
3174                 mutex_unlock(&isert_np->np_accept_mutex);
3175                 max_accept++;
3176                 goto accept_wait;
3177         }
3178         isert_conn = list_first_entry(&isert_np->np_accept_list,
3179                         struct isert_conn, conn_accept_node);
3180         list_del_init(&isert_conn->conn_accept_node);
3181         mutex_unlock(&isert_np->np_accept_mutex);
3182
3183         conn->context = isert_conn;
3184         isert_conn->conn = conn;
3185         max_accept = 0;
3186
3187         isert_set_conn_info(np, conn, isert_conn);
3188
3189         pr_debug("Processing isert_conn: %p\n", isert_conn);
3190
3191         return 0;
3192 }
3193
3194 static void
3195 isert_free_np(struct iscsi_np *np)
3196 {
3197         struct isert_np *isert_np = (struct isert_np *)np->np_context;
3198         struct isert_conn *isert_conn, *n;
3199
3200         if (isert_np->np_cm_id)
3201                 rdma_destroy_id(isert_np->np_cm_id);
3202
3203         /*
3204          * FIXME: At this point we don't have a good way to insure
3205          * that at this point we don't have hanging connections that
3206          * completed RDMA establishment but didn't start iscsi login
3207          * process. So work-around this by cleaning up what ever piled
3208          * up in np_accept_list.
3209          */
3210         mutex_lock(&isert_np->np_accept_mutex);
3211         if (!list_empty(&isert_np->np_accept_list)) {
3212                 pr_info("Still have isert connections, cleaning up...\n");
3213                 list_for_each_entry_safe(isert_conn, n,
3214                                          &isert_np->np_accept_list,
3215                                          conn_accept_node) {
3216                         pr_info("cleaning isert_conn %p state (%d)\n",
3217                                    isert_conn, isert_conn->state);
3218                         isert_connect_release(isert_conn);
3219                 }
3220         }
3221         mutex_unlock(&isert_np->np_accept_mutex);
3222
3223         np->np_context = NULL;
3224         kfree(isert_np);
3225 }
3226
3227 static void isert_release_work(struct work_struct *work)
3228 {
3229         struct isert_conn *isert_conn = container_of(work,
3230                                                      struct isert_conn,
3231                                                      release_work);
3232
3233         pr_info("Starting release conn %p\n", isert_conn);
3234
3235         wait_for_completion(&isert_conn->conn_wait);
3236
3237         mutex_lock(&isert_conn->conn_mutex);
3238         isert_conn->state = ISER_CONN_DOWN;
3239         mutex_unlock(&isert_conn->conn_mutex);
3240
3241         pr_info("Destroying conn %p\n", isert_conn);
3242         isert_put_conn(isert_conn);
3243 }
3244
3245 static void
3246 isert_wait4flush(struct isert_conn *isert_conn)
3247 {
3248         struct ib_recv_wr *bad_wr;
3249
3250         init_completion(&isert_conn->conn_wait_comp_err);
3251         isert_conn->beacon.wr_id = ISER_BEACON_WRID;
3252         /* post an indication that all flush errors were consumed */
3253         if (ib_post_recv(isert_conn->conn_qp, &isert_conn->beacon, &bad_wr)) {
3254                 pr_err("conn %p failed to post beacon", isert_conn);
3255                 return;
3256         }
3257
3258         wait_for_completion(&isert_conn->conn_wait_comp_err);
3259 }
3260
3261 static void isert_wait_conn(struct iscsi_conn *conn)
3262 {
3263         struct isert_conn *isert_conn = conn->context;
3264
3265         pr_debug("isert_wait_conn: Starting \n");
3266
3267         mutex_lock(&isert_conn->conn_mutex);
3268         /*
3269          * Only wait for conn_wait_comp_err if the isert_conn made it
3270          * into full feature phase..
3271          */
3272         if (isert_conn->state == ISER_CONN_INIT) {
3273                 mutex_unlock(&isert_conn->conn_mutex);
3274                 return;
3275         }
3276         isert_conn_terminate(isert_conn);
3277         mutex_unlock(&isert_conn->conn_mutex);
3278
3279         isert_wait4flush(isert_conn);
3280
3281         INIT_WORK(&isert_conn->release_work, isert_release_work);
3282         queue_work(isert_release_wq, &isert_conn->release_work);
3283 }
3284
3285 static void isert_free_conn(struct iscsi_conn *conn)
3286 {
3287         struct isert_conn *isert_conn = conn->context;
3288
3289         isert_put_conn(isert_conn);
3290 }
3291
3292 static struct iscsit_transport iser_target_transport = {
3293         .name                   = "IB/iSER",
3294         .transport_type         = ISCSI_INFINIBAND,
3295         .priv_size              = sizeof(struct isert_cmd),
3296         .owner                  = THIS_MODULE,
3297         .iscsit_setup_np        = isert_setup_np,
3298         .iscsit_accept_np       = isert_accept_np,
3299         .iscsit_free_np         = isert_free_np,
3300         .iscsit_wait_conn       = isert_wait_conn,
3301         .iscsit_free_conn       = isert_free_conn,
3302         .iscsit_get_login_rx    = isert_get_login_rx,
3303         .iscsit_put_login_tx    = isert_put_login_tx,
3304         .iscsit_immediate_queue = isert_immediate_queue,
3305         .iscsit_response_queue  = isert_response_queue,
3306         .iscsit_get_dataout     = isert_get_dataout,
3307         .iscsit_queue_data_in   = isert_put_datain,
3308         .iscsit_queue_status    = isert_put_response,
3309         .iscsit_aborted_task    = isert_aborted_task,
3310         .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops,
3311 };
3312
3313 static int __init isert_init(void)
3314 {
3315         int ret;
3316
3317         isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0);
3318         if (!isert_comp_wq) {
3319                 pr_err("Unable to allocate isert_comp_wq\n");
3320                 ret = -ENOMEM;
3321                 return -ENOMEM;
3322         }
3323
3324         isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND,
3325                                         WQ_UNBOUND_MAX_ACTIVE);
3326         if (!isert_release_wq) {
3327                 pr_err("Unable to allocate isert_release_wq\n");
3328                 ret = -ENOMEM;
3329                 goto destroy_comp_wq;
3330         }
3331
3332         iscsit_register_transport(&iser_target_transport);
3333         pr_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
3334
3335         return 0;
3336
3337 destroy_comp_wq:
3338         destroy_workqueue(isert_comp_wq);
3339
3340         return ret;
3341 }
3342
3343 static void __exit isert_exit(void)
3344 {
3345         flush_scheduled_work();
3346         destroy_workqueue(isert_release_wq);
3347         destroy_workqueue(isert_comp_wq);
3348         iscsit_unregister_transport(&iser_target_transport);
3349         pr_debug("iSER_TARGET[0] - Released iser_target_transport\n");
3350 }
3351
3352 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
3353 MODULE_VERSION("0.1");
3354 MODULE_AUTHOR("nab@Linux-iSCSI.org");
3355 MODULE_LICENSE("GPL");
3356
3357 module_init(isert_init);
3358 module_exit(isert_exit);