drivers/nvme/host/multipath.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2017-2018 Christoph Hellwig.
   4  */
   5
   6 #include <linux/moduleparam.h>
   7 #include <trace/events/block.h>
   8 #include "nvme.h"
   9
  10 static bool multipath = true;
  11 module_param(multipath, bool, 0444);
  12 MODULE_PARM_DESC(multipath,
  13         "turn on native support for multiple controllers per subsystem");
  14
  15 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
  16 {
  17         struct nvme_ns_head *h;
  18
  19         lockdep_assert_held(&subsys->lock);
  20         list_for_each_entry(h, &subsys->nsheads, entry)
  21                 if (h->disk)
  22                         blk_mq_unfreeze_queue(h->disk->queue);
  23 }
  24
  25 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
  26 {
  27         struct nvme_ns_head *h;
  28
  29         lockdep_assert_held(&subsys->lock);
  30         list_for_each_entry(h, &subsys->nsheads, entry)
  31                 if (h->disk)
  32                         blk_mq_freeze_queue_wait(h->disk->queue);
  33 }
  34
  35 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
  36 {
  37         struct nvme_ns_head *h;
  38
  39         lockdep_assert_held(&subsys->lock);
  40         list_for_each_entry(h, &subsys->nsheads, entry)
  41                 if (h->disk)
  42                         blk_freeze_queue_start(h->disk->queue);
  43 }
  44
  45 /*
  46  * If multipathing is enabled we need to always use the subsystem instance
  47  * number for numbering our devices to avoid conflicts between subsystems that
  48  * have multiple controllers and thus use the multipath-aware subsystem node
  49  * and those that have a single controller and use the controller node
  50  * directly.
  51  */
  52 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
  53                         struct nvme_ctrl *ctrl, int *flags)
  54 {
  55         if (!multipath) {
  56                 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
  57         } else if (ns->head->disk) {
  58                 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
  59                                 ctrl->instance, ns->head->instance);
  60                 *flags = GENHD_FL_HIDDEN;
  61         } else {
  62                 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
  63                                 ns->head->instance);
  64         }
  65 }
  66
  67 bool nvme_failover_req(struct request *req)
  68 {
  69         struct nvme_ns *ns = req->q->queuedata;
  70         u16 status = nvme_req(req)->status;
  71         unsigned long flags;
  72
  73         switch (status & 0x7ff) {
  74         case NVME_SC_ANA_TRANSITION:
  75         case NVME_SC_ANA_INACCESSIBLE:
  76         case NVME_SC_ANA_PERSISTENT_LOSS:
  77                 /*
  78                  * If we got back an ANA error we know the controller is alive,
  79                  * but not ready to serve this namespaces.  The spec suggests
  80                  * we should update our general state here, but due to the fact
  81                  * that the admin and I/O queues are not serialized that is
  82                  * fundamentally racy.  So instead just clear the current path,
  83                  * mark the the path as pending and kick of a re-read of the ANA
  84                  * log page ASAP.
  85                  */
  86                 nvme_mpath_clear_current_path(ns);
  87                 if (ns->ctrl->ana_log_buf) {
  88                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  89                         queue_work(nvme_wq, &ns->ctrl->ana_work);
  90                 }
  91                 break;
  92         case NVME_SC_HOST_PATH_ERROR:
  93         case NVME_SC_HOST_ABORTED_CMD:
  94                 /*
  95                  * Temporary transport disruption in talking to the controller.
  96                  * Try to send on a new path.
  97                  */
  98                 nvme_mpath_clear_current_path(ns);
  99                 break;
 100         default:
 101                 /* This was a non-ANA error so follow the normal error path. */
 102                 return false;
 103         }
 104
 105         spin_lock_irqsave(&ns->head->requeue_lock, flags);
 106         blk_steal_bios(&ns->head->requeue_list, req);
 107         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
 108         blk_mq_end_request(req, 0);
 109
 110         kblockd_schedule_work(&ns->head->requeue_work);
 111         return true;
 112 }
 113
 114 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 115 {
 116         struct nvme_ns *ns;
 117
 118         down_read(&ctrl->namespaces_rwsem);
 119         list_for_each_entry(ns, &ctrl->namespaces, list) {
 120                 if (ns->head->disk)
 121                         kblockd_schedule_work(&ns->head->requeue_work);
 122         }
 123         up_read(&ctrl->namespaces_rwsem);
 124 }
 125
 126 static const char *nvme_ana_state_names[] = {
 127         [0]                             = "invalid state",
 128         [NVME_ANA_OPTIMIZED]            = "optimized",
 129         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 130         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 131         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 132         [NVME_ANA_CHANGE]               = "change",
 133 };
 134
 135 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 136 {
 137         struct nvme_ns_head *head = ns->head;
 138         bool changed = false;
 139         int node;
 140
 141         if (!head)
 142                 goto out;
 143
 144         for_each_node(node) {
 145                 if (ns == rcu_access_pointer(head->current_path[node])) {
 146                         rcu_assign_pointer(head->current_path[node], NULL);
 147                         changed = true;
 148                 }
 149         }
 150 out:
 151         return changed;
 152 }
 153
 154 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 155 {
 156         struct nvme_ns *ns;
 157
 158         mutex_lock(&ctrl->scan_lock);
 159         down_read(&ctrl->namespaces_rwsem);
 160         list_for_each_entry(ns, &ctrl->namespaces, list)
 161                 if (nvme_mpath_clear_current_path(ns))
 162                         kblockd_schedule_work(&ns->head->requeue_work);
 163         up_read(&ctrl->namespaces_rwsem);
 164         mutex_unlock(&ctrl->scan_lock);
 165 }
 166
 167 static bool nvme_path_is_disabled(struct nvme_ns *ns)
 168 {
 169         return ns->ctrl->state != NVME_CTRL_LIVE ||
 170                 test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
 171                 test_bit(NVME_NS_REMOVING, &ns->flags);
 172 }
 173
 174 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 175 {
 176         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 177         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 178
 179         list_for_each_entry_rcu(ns, &head->list, siblings) {
 180                 if (nvme_path_is_disabled(ns))
 181                         continue;
 182
 183                 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
 184                         distance = node_distance(node, ns->ctrl->numa_node);
 185                 else
 186                         distance = LOCAL_DISTANCE;
 187
 188                 switch (ns->ana_state) {
 189                 case NVME_ANA_OPTIMIZED:
 190                         if (distance < found_distance) {
 191                                 found_distance = distance;
 192                                 found = ns;
 193                         }
 194                         break;
 195                 case NVME_ANA_NONOPTIMIZED:
 196                         if (distance < fallback_distance) {
 197                                 fallback_distance = distance;
 198                                 fallback = ns;
 199                         }
 200                         break;
 201                 default:
 202                         break;
 203                 }
 204         }
 205
 206         if (!found)
 207                 found = fallback;
 208         if (found)
 209                 rcu_assign_pointer(head->current_path[node], found);
 210         return found;
 211 }
 212
 213 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,
 214                 struct nvme_ns *ns)
 215 {
 216         ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,
 217                         siblings);
 218         if (ns)
 219                 return ns;
 220         return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);
 221 }
 222
 223 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 224                 int node, struct nvme_ns *old)
 225 {
 226         struct nvme_ns *ns, *found, *fallback = NULL;
 227
 228         if (list_is_singular(&head->list)) {
 229                 if (nvme_path_is_disabled(old))
 230                         return NULL;
 231                 return old;
 232         }
 233
 234         for (ns = nvme_next_ns(head, old);
 235              ns != old;
 236              ns = nvme_next_ns(head, ns)) {
 237                 if (nvme_path_is_disabled(ns))
 238                         continue;
 239
 240                 if (ns->ana_state == NVME_ANA_OPTIMIZED) {
 241                         found = ns;
 242                         goto out;
 243                 }
 244                 if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
 245                         fallback = ns;
 246         }
 247
 248         if (!fallback)
 249                 return NULL;
 250         found = fallback;
 251 out:
 252         rcu_assign_pointer(head->current_path[node], found);
 253         return found;
 254 }
 255
 256 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 257 {
 258         return ns->ctrl->state == NVME_CTRL_LIVE &&
 259                 ns->ana_state == NVME_ANA_OPTIMIZED;
 260 }
 261
 262 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 263 {
 264         int node = numa_node_id();
 265         struct nvme_ns *ns;
 266
 267         ns = srcu_dereference(head->current_path[node], &head->srcu);
 268         if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR && ns)
 269                 ns = nvme_round_robin_path(head, node, ns);
 270         if (unlikely(!ns || !nvme_path_is_optimized(ns)))
 271                 ns = __nvme_find_path(head, node);
 272         return ns;
 273 }
 274
 275 static bool nvme_available_path(struct nvme_ns_head *head)
 276 {
 277         struct nvme_ns *ns;
 278
 279         list_for_each_entry_rcu(ns, &head->list, siblings) {
 280                 switch (ns->ctrl->state) {
 281                 case NVME_CTRL_LIVE:
 282                 case NVME_CTRL_RESETTING:
 283                 case NVME_CTRL_CONNECTING:
 284                         /* fallthru */
 285                         return true;
 286                 default:
 287                         break;
 288                 }
 289         }
 290         return false;
 291 }
 292
 293 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
 294                 struct bio *bio)
 295 {
 296         struct nvme_ns_head *head = q->queuedata;
 297         struct device *dev = disk_to_dev(head->disk);
 298         struct nvme_ns *ns;
 299         blk_qc_t ret = BLK_QC_T_NONE;
 300         int srcu_idx;
 301
 302         /*
 303          * The namespace might be going away and the bio might
 304          * be moved to a different queue via blk_steal_bios(),
 305          * so we need to use the bio_split pool from the original
 306          * queue to allocate the bvecs from.
 307          */
 308         blk_queue_split(q, &bio);
 309
 310         srcu_idx = srcu_read_lock(&head->srcu);
 311         ns = nvme_find_path(head);
 312         if (likely(ns)) {
 313                 bio->bi_disk = ns->disk;
 314                 bio->bi_opf |= REQ_NVME_MPATH;
 315                 trace_block_bio_remap(bio->bi_disk->queue, bio,
 316                                       disk_devt(ns->head->disk),
 317                                       bio->bi_iter.bi_sector);
 318                 ret = direct_make_request(bio);
 319         } else if (nvme_available_path(head)) {
 320                 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
 321
 322                 spin_lock_irq(&head->requeue_lock);
 323                 bio_list_add(&head->requeue_list, bio);
 324                 spin_unlock_irq(&head->requeue_lock);
 325         } else {
 326                 dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 327
 328                 bio->bi_status = BLK_STS_IOERR;
 329                 bio_endio(bio);
 330         }
 331
 332         srcu_read_unlock(&head->srcu, srcu_idx);
 333         return ret;
 334 }
 335
 336 static void nvme_requeue_work(struct work_struct *work)
 337 {
 338         struct nvme_ns_head *head =
 339                 container_of(work, struct nvme_ns_head, requeue_work);
 340         struct bio *bio, *next;
 341
 342         spin_lock_irq(&head->requeue_lock);
 343         next = bio_list_get(&head->requeue_list);
 344         spin_unlock_irq(&head->requeue_lock);
 345
 346         while ((bio = next) != NULL) {
 347                 next = bio->bi_next;
 348                 bio->bi_next = NULL;
 349
 350                 /*
 351                  * Reset disk to the mpath node and resubmit to select a new
 352                  * path.
 353                  */
 354                 bio->bi_disk = head->disk;
 355                 generic_make_request(bio);
 356         }
 357 }
 358
 359 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 360 {
 361         struct request_queue *q;
 362         bool vwc = false;
 363
 364         mutex_init(&head->lock);
 365         bio_list_init(&head->requeue_list);
 366         spin_lock_init(&head->requeue_lock);
 367         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 368
 369         /*
 370          * Add a multipath node if the subsystems supports multiple controllers.
 371          * We also do this for private namespaces as the namespace sharing data could
 372          * change after a rescan.
 373          */
 374         if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath)
 375                 return 0;
 376
 377         q = blk_alloc_queue(nvme_ns_head_make_request, ctrl->numa_node);
 378         if (!q)
 379                 goto out;
 380         q->queuedata = head;
 381         blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
 382         /* set to a default value for 512 until disk is validated */
 383         blk_queue_logical_block_size(q, 512);
 384         blk_set_stacking_limits(&q->limits);
 385
 386         /* we need to propagate up the VMC settings */
 387         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 388                 vwc = true;
 389         blk_queue_write_cache(q, vwc, vwc);
 390
 391         head->disk = alloc_disk(0);
 392         if (!head->disk)
 393                 goto out_cleanup_queue;
 394         head->disk->fops = &nvme_ns_head_ops;
 395         head->disk->private_data = head;
 396         head->disk->queue = q;
 397         head->disk->flags = GENHD_FL_EXT_DEVT;
 398         sprintf(head->disk->disk_name, "nvme%dn%d",
 399                         ctrl->subsys->instance, head->instance);
 400         return 0;
 401
 402 out_cleanup_queue:
 403         blk_cleanup_queue(q);
 404 out:
 405         return -ENOMEM;
 406 }
 407
 408 static void nvme_mpath_set_live(struct nvme_ns *ns)
 409 {
 410         struct nvme_ns_head *head = ns->head;
 411
 412         lockdep_assert_held(&ns->head->lock);
 413
 414         if (!head->disk)
 415                 return;
 416
 417         if (!(head->disk->flags & GENHD_FL_UP))
 418                 device_add_disk(&head->subsys->dev, head->disk,
 419                                 nvme_ns_id_attr_groups);
 420
 421         if (nvme_path_is_optimized(ns)) {
 422                 int node, srcu_idx;
 423
 424                 srcu_idx = srcu_read_lock(&head->srcu);
 425                 for_each_node(node)
 426                         __nvme_find_path(head, node);
 427                 srcu_read_unlock(&head->srcu, srcu_idx);
 428         }
 429
 430         synchronize_srcu(&ns->head->srcu);
 431         kblockd_schedule_work(&ns->head->requeue_work);
 432 }
 433
 434 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 435                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 436                         void *))
 437 {
 438         void *base = ctrl->ana_log_buf;
 439         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 440         int error, i;
 441
 442         lockdep_assert_held(&ctrl->ana_lock);
 443
 444         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 445                 struct nvme_ana_group_desc *desc = base + offset;
 446                 u32 nr_nsids;
 447                 size_t nsid_buf_size;
 448
 449                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 450                         return -EINVAL;
 451
 452                 nr_nsids = le32_to_cpu(desc->nnsids);
 453                 nsid_buf_size = nr_nsids * sizeof(__le32);
 454
 455                 if (WARN_ON_ONCE(desc->grpid == 0))
 456                         return -EINVAL;
 457                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 458                         return -EINVAL;
 459                 if (WARN_ON_ONCE(desc->state == 0))
 460                         return -EINVAL;
 461                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 462                         return -EINVAL;
 463
 464                 offset += sizeof(*desc);
 465                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 466                         return -EINVAL;
 467
 468                 error = cb(ctrl, desc, data);
 469                 if (error)
 470                         return error;
 471
 472                 offset += nsid_buf_size;
 473         }
 474
 475         return 0;
 476 }
 477
 478 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 479 {
 480         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 481 }
 482
 483 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 484                 struct nvme_ns *ns)
 485 {
 486         mutex_lock(&ns->head->lock);
 487         ns->ana_grpid = le32_to_cpu(desc->grpid);
 488         ns->ana_state = desc->state;
 489         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 490
 491         if (nvme_state_is_live(ns->ana_state))
 492                 nvme_mpath_set_live(ns);
 493         mutex_unlock(&ns->head->lock);
 494 }
 495
 496 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 497                 struct nvme_ana_group_desc *desc, void *data)
 498 {
 499         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 500         unsigned *nr_change_groups = data;
 501         struct nvme_ns *ns;
 502
 503         dev_dbg(ctrl->device, "ANA group %d: %s.\n",
 504                         le32_to_cpu(desc->grpid),
 505                         nvme_ana_state_names[desc->state]);
 506
 507         if (desc->state == NVME_ANA_CHANGE)
 508                 (*nr_change_groups)++;
 509
 510         if (!nr_nsids)
 511                 return 0;
 512
 513         down_write(&ctrl->namespaces_rwsem);
 514         list_for_each_entry(ns, &ctrl->namespaces, list) {
 515                 unsigned nsid = le32_to_cpu(desc->nsids[n]);
 516
 517                 if (ns->head->ns_id < nsid)
 518                         continue;
 519                 if (ns->head->ns_id == nsid)
 520                         nvme_update_ns_ana_state(desc, ns);
 521                 if (++n == nr_nsids)
 522                         break;
 523         }
 524         up_write(&ctrl->namespaces_rwsem);
 525         return 0;
 526 }
 527
 528 static int nvme_read_ana_log(struct nvme_ctrl *ctrl)
 529 {
 530         u32 nr_change_groups = 0;
 531         int error;
 532
 533         mutex_lock(&ctrl->ana_lock);
 534         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0,
 535                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 536         if (error) {
 537                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 538                 goto out_unlock;
 539         }
 540
 541         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 542                         nvme_update_ana_state);
 543         if (error)
 544                 goto out_unlock;
 545
 546         /*
 547          * In theory we should have an ANATT timer per group as they might enter
 548          * the change state at different times.  But that is a lot of overhead
 549          * just to protect against a target that keeps entering new changes
 550          * states while never finishing previous ones.  But we'll still
 551          * eventually time out once all groups are in change state, so this
 552          * isn't a big deal.
 553          *
 554          * We also double the ANATT value to provide some slack for transports
 555          * or AEN processing overhead.
 556          */
 557         if (nr_change_groups)
 558                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 559         else
 560                 del_timer_sync(&ctrl->anatt_timer);
 561 out_unlock:
 562         mutex_unlock(&ctrl->ana_lock);
 563         return error;
 564 }
 565
 566 static void nvme_ana_work(struct work_struct *work)
 567 {
 568         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 569
 570         nvme_read_ana_log(ctrl);
 571 }
 572
 573 static void nvme_anatt_timeout(struct timer_list *t)
 574 {
 575         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 576
 577         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 578         nvme_reset_ctrl(ctrl);
 579 }
 580
 581 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 582 {
 583         if (!nvme_ctrl_use_ana(ctrl))
 584                 return;
 585         del_timer_sync(&ctrl->anatt_timer);
 586         cancel_work_sync(&ctrl->ana_work);
 587 }
 588
 589 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store)  \
 590         struct device_attribute subsys_attr_##_name =   \
 591                 __ATTR(_name, _mode, _show, _store)
 592
 593 static const char *nvme_iopolicy_names[] = {
 594         [NVME_IOPOLICY_NUMA]    = "numa",
 595         [NVME_IOPOLICY_RR]      = "round-robin",
 596 };
 597
 598 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 599                 struct device_attribute *attr, char *buf)
 600 {
 601         struct nvme_subsystem *subsys =
 602                 container_of(dev, struct nvme_subsystem, dev);
 603
 604         return sprintf(buf, "%s\n",
 605                         nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 606 }
 607
 608 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 609                 struct device_attribute *attr, const char *buf, size_t count)
 610 {
 611         struct nvme_subsystem *subsys =
 612                 container_of(dev, struct nvme_subsystem, dev);
 613         int i;
 614
 615         for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 616                 if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
 617                         WRITE_ONCE(subsys->iopolicy, i);
 618                         return count;
 619                 }
 620         }
 621
 622         return -EINVAL;
 623 }
 624 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
 625                       nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
 626
 627 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 628                 char *buf)
 629 {
 630         return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 631 }
 632 DEVICE_ATTR_RO(ana_grpid);
 633
 634 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 635                 char *buf)
 636 {
 637         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 638
 639         return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 640 }
 641 DEVICE_ATTR_RO(ana_state);
 642
 643 static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
 644                 struct nvme_ana_group_desc *desc, void *data)
 645 {
 646         struct nvme_ns *ns = data;
 647
 648         if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
 649                 nvme_update_ns_ana_state(desc, ns);
 650                 return -ENXIO; /* just break out of the loop */
 651         }
 652
 653         return 0;
 654 }
 655
 656 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
 657 {
 658         if (nvme_ctrl_use_ana(ns->ctrl)) {
 659                 mutex_lock(&ns->ctrl->ana_lock);
 660                 ns->ana_grpid = le32_to_cpu(id->anagrpid);
 661                 nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
 662                 mutex_unlock(&ns->ctrl->ana_lock);
 663         } else {
 664                 mutex_lock(&ns->head->lock);
 665                 ns->ana_state = NVME_ANA_OPTIMIZED;
 666                 nvme_mpath_set_live(ns);
 667                 mutex_unlock(&ns->head->lock);
 668         }
 669 }
 670
 671 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 672 {
 673         if (!head->disk)
 674                 return;
 675         if (head->disk->flags & GENHD_FL_UP)
 676                 del_gendisk(head->disk);
 677         blk_set_queue_dying(head->disk->queue);
 678         /* make sure all pending bios are cleaned up */
 679         kblockd_schedule_work(&head->requeue_work);
 680         flush_work(&head->requeue_work);
 681         blk_cleanup_queue(head->disk->queue);
 682         put_disk(head->disk);
 683 }
 684
 685 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 686 {
 687         int error;
 688
 689         /* check if multipath is enabled and we have the capability */
 690         if (!multipath || !ctrl->subsys || !(ctrl->subsys->cmic & (1 << 3)))
 691                 return 0;
 692
 693         ctrl->anacap = id->anacap;
 694         ctrl->anatt = id->anatt;
 695         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 696         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 697
 698         mutex_init(&ctrl->ana_lock);
 699         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 700         ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 701                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
 702         ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
 703
 704         if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
 705                 dev_err(ctrl->device,
 706                         "ANA log page size (%zd) larger than MDTS (%d).\n",
 707                         ctrl->ana_log_size,
 708                         ctrl->max_hw_sectors << SECTOR_SHIFT);
 709                 dev_err(ctrl->device, "disabling ANA support.\n");
 710                 return 0;
 711         }
 712
 713         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 714         kfree(ctrl->ana_log_buf);
 715         ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
 716         if (!ctrl->ana_log_buf) {
 717                 error = -ENOMEM;
 718                 goto out;
 719         }
 720
 721         error = nvme_read_ana_log(ctrl);
 722         if (error)
 723                 goto out_free_ana_log_buf;
 724         return 0;
 725 out_free_ana_log_buf:
 726         kfree(ctrl->ana_log_buf);
 727         ctrl->ana_log_buf = NULL;
 728 out:
 729         return error;
 730 }
 731
 732 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 733 {
 734         kfree(ctrl->ana_log_buf);
 735         ctrl->ana_log_buf = NULL;
 736 }
 737