nvmet: add metadata support for block devices

Allocate the metadata SGL buffers and set metadata fields for the
request. Then create a block IO request for the metadata from the
protection SG list.

Signed-off-by: Israel Rukshin <israelr@mellanox.com>
Signed-off-by: Max Gurtovoy <maxg@mellanox.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>

diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index f9172d3..edf54d9 100644 (file)
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -900,8 +900,11 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
        req->sq = sq;
        req->ops = ops;
        req->sg = NULL;
+       req->metadata_sg = NULL;
        req->sg_cnt = 0;
+       req->metadata_sg_cnt = 0;
        req->transfer_len = 0;
+       req->metadata_len = 0;
        req->cqe->status = 0;
        req->cqe->sq_head = 0;
        req->ns = NULL;
@@ -986,50 +989,90 @@ bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len)
        return true;
 }
 
-int nvmet_req_alloc_sgl(struct nvmet_req *req)
+static unsigned int nvmet_data_transfer_len(struct nvmet_req *req)
 {
-       struct pci_dev *p2p_dev = NULL;
+       return req->transfer_len - req->metadata_len;
+}
 
-       if (IS_ENABLED(CONFIG_PCI_P2PDMA)) {
-               if (req->sq->ctrl && req->ns)
-                       p2p_dev = radix_tree_lookup(&req->sq->ctrl->p2p_ns_map,
-                                                   req->ns->nsid);
+static int nvmet_req_alloc_p2pmem_sgls(struct nvmet_req *req)
+{
+       req->sg = pci_p2pmem_alloc_sgl(req->p2p_dev, &req->sg_cnt,
+                       nvmet_data_transfer_len(req));
+       if (!req->sg)
+               goto out_err;
 
-               req->p2p_dev = NULL;
-               if (req->sq->qid && p2p_dev) {
-                       req->sg = pci_p2pmem_alloc_sgl(p2p_dev, &req->sg_cnt,
-                                                      req->transfer_len);
-                       if (req->sg) {
-                               req->p2p_dev = p2p_dev;
-                               return 0;
-                       }
-               }
+       if (req->metadata_len) {
+               req->metadata_sg = pci_p2pmem_alloc_sgl(req->p2p_dev,
+                               &req->metadata_sg_cnt, req->metadata_len);
+               if (!req->metadata_sg)
+                       goto out_free_sg;
+       }
+       return 0;
+out_free_sg:
+       pci_p2pmem_free_sgl(req->p2p_dev, req->sg);
+out_err:
+       return -ENOMEM;
+}
 
-               /*
-                * If no P2P memory was available we fallback to using
-                * regular memory
-                */
+static bool nvmet_req_find_p2p_dev(struct nvmet_req *req)
+{
+       if (!IS_ENABLED(CONFIG_PCI_P2PDMA))
+               return false;
+
+       if (req->sq->ctrl && req->sq->qid && req->ns) {
+               req->p2p_dev = radix_tree_lookup(&req->sq->ctrl->p2p_ns_map,
+                                                req->ns->nsid);
+               if (req->p2p_dev)
+                       return true;
        }
 
-       req->sg = sgl_alloc(req->transfer_len, GFP_KERNEL, &req->sg_cnt);
+       req->p2p_dev = NULL;
+       return false;
+}
+
+int nvmet_req_alloc_sgls(struct nvmet_req *req)
+{
+       if (nvmet_req_find_p2p_dev(req) && !nvmet_req_alloc_p2pmem_sgls(req))
+               return 0;
+
+       req->sg = sgl_alloc(nvmet_data_transfer_len(req), GFP_KERNEL,
+                           &req->sg_cnt);
        if (unlikely(!req->sg))
-               return -ENOMEM;
+               goto out;
+
+       if (req->metadata_len) {
+               req->metadata_sg = sgl_alloc(req->metadata_len, GFP_KERNEL,
+                                            &req->metadata_sg_cnt);
+               if (unlikely(!req->metadata_sg))
+                       goto out_free;
+       }
 
        return 0;
+out_free:
+       sgl_free(req->sg);
+out:
+       return -ENOMEM;
 }
-EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgl);
+EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgls);
 
-void nvmet_req_free_sgl(struct nvmet_req *req)
+void nvmet_req_free_sgls(struct nvmet_req *req)
 {
-       if (req->p2p_dev)
+       if (req->p2p_dev) {
                pci_p2pmem_free_sgl(req->p2p_dev, req->sg);
-       else
+               if (req->metadata_sg)
+                       pci_p2pmem_free_sgl(req->p2p_dev, req->metadata_sg);
+       } else {
                sgl_free(req->sg);
+               if (req->metadata_sg)
+                       sgl_free(req->metadata_sg);
+       }
 
        req->sg = NULL;
+       req->metadata_sg = NULL;
        req->sg_cnt = 0;
+       req->metadata_sg_cnt = 0;
 }
-EXPORT_SYMBOL_GPL(nvmet_req_free_sgl);
+EXPORT_SYMBOL_GPL(nvmet_req_free_sgls);
 
 static inline bool nvmet_cc_en(u32 cc)
 {

diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c
index e065c24..07055f7 100644 (file)
--- a/drivers/nvme/target/io-cmd-bdev.c
+++ b/drivers/nvme/target/io-cmd-bdev.c
@@ -169,6 +169,61 @@ static void nvmet_bio_done(struct bio *bio)
                bio_put(bio);
 }
 
+#ifdef CONFIG_BLK_DEV_INTEGRITY
+static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,
+                               struct sg_mapping_iter *miter)
+{
+       struct blk_integrity *bi;
+       struct bio_integrity_payload *bip;
+       struct block_device *bdev = req->ns->bdev;
+       int rc;
+       size_t resid, len;
+
+       bi = bdev_get_integrity(bdev);
+       if (unlikely(!bi)) {
+               pr_err("Unable to locate bio_integrity\n");
+               return -ENODEV;
+       }
+
+       bip = bio_integrity_alloc(bio, GFP_NOIO,
+               min_t(unsigned int, req->metadata_sg_cnt, BIO_MAX_PAGES));
+       if (IS_ERR(bip)) {
+               pr_err("Unable to allocate bio_integrity_payload\n");
+               return PTR_ERR(bip);
+       }
+
+       bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
+       /* virtual start sector must be in integrity interval units */
+       bip_set_seed(bip, bio->bi_iter.bi_sector >>
+                    (bi->interval_exp - SECTOR_SHIFT));
+
+       resid = bip->bip_iter.bi_size;
+       while (resid > 0 && sg_miter_next(miter)) {
+               len = min_t(size_t, miter->length, resid);
+               rc = bio_integrity_add_page(bio, miter->page, len,
+                                           offset_in_page(miter->addr));
+               if (unlikely(rc != len)) {
+                       pr_err("bio_integrity_add_page() failed; %d\n", rc);
+                       sg_miter_stop(miter);
+                       return -ENOMEM;
+               }
+
+               resid -= len;
+               if (len < miter->length)
+                       miter->consumed -= miter->length - len;
+       }
+       sg_miter_stop(miter);
+
+       return 0;
+}
+#else
+static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,
+                               struct sg_mapping_iter *miter)
+{
+       return -EINVAL;
+}
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
+
 static void nvmet_bdev_execute_rw(struct nvmet_req *req)
 {
        int sg_cnt = req->sg_cnt;
@@ -176,9 +231,12 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
        struct scatterlist *sg;
        struct blk_plug plug;
        sector_t sector;
-       int op, i;
+       int op, i, rc;
+       struct sg_mapping_iter prot_miter;
+       unsigned int iter_flags;
+       unsigned int total_len = nvmet_rw_data_len(req) + req->metadata_len;
 
-       if (!nvmet_check_transfer_len(req, nvmet_rw_data_len(req)))
+       if (!nvmet_check_transfer_len(req, total_len))
                return;
 
        if (!req->sg_cnt) {
@@ -190,8 +248,10 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
                op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
                if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
                        op |= REQ_FUA;
+               iter_flags = SG_MITER_TO_SG;
        } else {
                op = REQ_OP_READ;
+               iter_flags = SG_MITER_FROM_SG;
        }
 
        if (is_pci_p2pdma_page(sg_page(req->sg)))
@@ -213,11 +273,24 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
        bio->bi_opf = op;
 
        blk_start_plug(&plug);
+       if (req->metadata_len)
+               sg_miter_start(&prot_miter, req->metadata_sg,
+                              req->metadata_sg_cnt, iter_flags);
+
        for_each_sg(req->sg, sg, req->sg_cnt, i) {
                while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
                                != sg->length) {
                        struct bio *prev = bio;
 
+                       if (req->metadata_len) {
+                               rc = nvmet_bdev_alloc_bip(req, bio,
+                                                         &prot_miter);
+                               if (unlikely(rc)) {
+                                       bio_io_error(bio);
+                                       return;
+                               }
+                       }
+
                        bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));
                        bio_set_dev(bio, req->ns->bdev);
                        bio->bi_iter.bi_sector = sector;
@@ -231,6 +304,14 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
                sg_cnt--;
        }
 
+       if (req->metadata_len) {
+               rc = nvmet_bdev_alloc_bip(req, bio, &prot_miter);
+               if (unlikely(rc)) {
+                       bio_io_error(bio);
+                       return;
+               }
+       }
+
        submit_bio(bio);
        blk_finish_plug(&plug);
 }
@@ -358,6 +439,8 @@ u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req)
        case nvme_cmd_read:
        case nvme_cmd_write:
                req->execute = nvmet_bdev_execute_rw;
+               if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns))
+                       req->metadata_len = nvmet_rw_metadata_len(req);
                return 0;
        case nvme_cmd_flush:
                req->execute = nvmet_bdev_execute_flush;

diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 46b5d2b..8096912 100644 (file)
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -309,6 +309,7 @@ struct nvmet_req {
        struct nvmet_cq         *cq;
        struct nvmet_ns         *ns;
        struct scatterlist      *sg;
+       struct scatterlist      *metadata_sg;
        struct bio_vec          inline_bvec[NVMET_MAX_INLINE_BIOVEC];
        union {
                struct {
@@ -322,8 +323,10 @@ struct nvmet_req {
                } f;
        };
        int                     sg_cnt;
+       int                     metadata_sg_cnt;
        /* data length as parsed from the SGL descriptor: */
        size_t                  transfer_len;
+       size_t                  metadata_len;
 
        struct nvmet_port       *port;
 
@@ -394,8 +397,8 @@ void nvmet_req_uninit(struct nvmet_req *req);
 bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len);
 bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len);
 void nvmet_req_complete(struct nvmet_req *req, u16 status);
-int nvmet_req_alloc_sgl(struct nvmet_req *req);
-void nvmet_req_free_sgl(struct nvmet_req *req);
+int nvmet_req_alloc_sgls(struct nvmet_req *req);
+void nvmet_req_free_sgls(struct nvmet_req *req);
 
 void nvmet_execute_keep_alive(struct nvmet_req *req);
 

diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index 7a90b10..178dbff 100644 (file)
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -546,7 +546,7 @@ static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp)
        }
 
        if (rsp->req.sg != rsp->cmd->inline_sg)
-               nvmet_req_free_sgl(&rsp->req);
+               nvmet_req_free_sgls(&rsp->req);
 
        if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list)))
                nvmet_rdma_process_wr_wait_list(queue);
@@ -708,7 +708,7 @@ static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp,
        if (!rsp->req.transfer_len)
                return 0;
 
-       ret = nvmet_req_alloc_sgl(&rsp->req);
+       ret = nvmet_req_alloc_sgls(&rsp->req);
        if (unlikely(ret < 0))
                goto error_out;