drivers/acpi/nfit/core.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/list_sort.h>
  14 #include <linux/libnvdimm.h>
  15 #include <linux/module.h>
  16 #include <linux/mutex.h>
  17 #include <linux/ndctl.h>
  18 #include <linux/sysfs.h>
  19 #include <linux/delay.h>
  20 #include <linux/list.h>
  21 #include <linux/acpi.h>
  22 #include <linux/sort.h>
  23 #include <linux/io.h>
  24 #include <linux/nd.h>
  25 #include <asm/cacheflush.h>
  26 #include <acpi/nfit.h>
  27 #include "intel.h"
  28 #include "nfit.h"
  29
  30 /*
  31  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
  32  * irrelevant.
  33  */
  34 #include <linux/io-64-nonatomic-hi-lo.h>
  35
  36 static bool force_enable_dimms;
  37 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
  38 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
  39
  40 static bool disable_vendor_specific;
  41 module_param(disable_vendor_specific, bool, S_IRUGO);
  42 MODULE_PARM_DESC(disable_vendor_specific,
  43                 "Limit commands to the publicly specified set");
  44
  45 static unsigned long override_dsm_mask;
  46 module_param(override_dsm_mask, ulong, S_IRUGO);
  47 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions");
  48
  49 static int default_dsm_family = -1;
  50 module_param(default_dsm_family, int, S_IRUGO);
  51 MODULE_PARM_DESC(default_dsm_family,
  52                 "Try this DSM type first when identifying NVDIMM family");
  53
  54 static bool no_init_ars;
  55 module_param(no_init_ars, bool, 0644);
  56 MODULE_PARM_DESC(no_init_ars, "Skip ARS run at nfit init time");
  57
  58 LIST_HEAD(acpi_descs);
  59 DEFINE_MUTEX(acpi_desc_lock);
  60
  61 static struct workqueue_struct *nfit_wq;
  62
  63 struct nfit_table_prev {
  64         struct list_head spas;
  65         struct list_head memdevs;
  66         struct list_head dcrs;
  67         struct list_head bdws;
  68         struct list_head idts;
  69         struct list_head flushes;
  70 };
  71
  72 static guid_t nfit_uuid[NFIT_UUID_MAX];
  73
  74 const guid_t *to_nfit_uuid(enum nfit_uuids id)
  75 {
  76         return &nfit_uuid[id];
  77 }
  78 EXPORT_SYMBOL(to_nfit_uuid);
  79
  80 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
  81 {
  82         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
  83
  84         /*
  85          * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
  86          * acpi_device.
  87          */
  88         if (!nd_desc->provider_name
  89                         || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
  90                 return NULL;
  91
  92         return to_acpi_device(acpi_desc->dev);
  93 }
  94
  95 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status)
  96 {
  97         struct nd_cmd_clear_error *clear_err;
  98         struct nd_cmd_ars_status *ars_status;
  99         u16 flags;
 100
 101         switch (cmd) {
 102         case ND_CMD_ARS_CAP:
 103                 if ((status & 0xffff) == NFIT_ARS_CAP_NONE)
 104                         return -ENOTTY;
 105
 106                 /* Command failed */
 107                 if (status & 0xffff)
 108                         return -EIO;
 109
 110                 /* No supported scan types for this range */
 111                 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
 112                 if ((status >> 16 & flags) == 0)
 113                         return -ENOTTY;
 114                 return 0;
 115         case ND_CMD_ARS_START:
 116                 /* ARS is in progress */
 117                 if ((status & 0xffff) == NFIT_ARS_START_BUSY)
 118                         return -EBUSY;
 119
 120                 /* Command failed */
 121                 if (status & 0xffff)
 122                         return -EIO;
 123                 return 0;
 124         case ND_CMD_ARS_STATUS:
 125                 ars_status = buf;
 126                 /* Command failed */
 127                 if (status & 0xffff)
 128                         return -EIO;
 129                 /* Check extended status (Upper two bytes) */
 130                 if (status == NFIT_ARS_STATUS_DONE)
 131                         return 0;
 132
 133                 /* ARS is in progress */
 134                 if (status == NFIT_ARS_STATUS_BUSY)
 135                         return -EBUSY;
 136
 137                 /* No ARS performed for the current boot */
 138                 if (status == NFIT_ARS_STATUS_NONE)
 139                         return -EAGAIN;
 140
 141                 /*
 142                  * ARS interrupted, either we overflowed or some other
 143                  * agent wants the scan to stop.  If we didn't overflow
 144                  * then just continue with the returned results.
 145                  */
 146                 if (status == NFIT_ARS_STATUS_INTR) {
 147                         if (ars_status->out_length >= 40 && (ars_status->flags
 148                                                 & NFIT_ARS_F_OVERFLOW))
 149                                 return -ENOSPC;
 150                         return 0;
 151                 }
 152
 153                 /* Unknown status */
 154                 if (status >> 16)
 155                         return -EIO;
 156                 return 0;
 157         case ND_CMD_CLEAR_ERROR:
 158                 clear_err = buf;
 159                 if (status & 0xffff)
 160                         return -EIO;
 161                 if (!clear_err->cleared)
 162                         return -EIO;
 163                 if (clear_err->length > clear_err->cleared)
 164                         return clear_err->cleared;
 165                 return 0;
 166         default:
 167                 break;
 168         }
 169
 170         /* all other non-zero status results in an error */
 171         if (status)
 172                 return -EIO;
 173         return 0;
 174 }
 175
 176 #define ACPI_LABELS_LOCKED 3
 177
 178 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
 179                 u32 status)
 180 {
 181         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 182
 183         switch (cmd) {
 184         case ND_CMD_GET_CONFIG_SIZE:
 185                 /*
 186                  * In the _LSI, _LSR, _LSW case the locked status is
 187                  * communicated via the read/write commands
 188                  */
 189                 if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
 190                         break;
 191
 192                 if (status >> 16 & ND_CONFIG_LOCKED)
 193                         return -EACCES;
 194                 break;
 195         case ND_CMD_GET_CONFIG_DATA:
 196                 if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
 197                                 && status == ACPI_LABELS_LOCKED)
 198                         return -EACCES;
 199                 break;
 200         case ND_CMD_SET_CONFIG_DATA:
 201                 if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags)
 202                                 && status == ACPI_LABELS_LOCKED)
 203                         return -EACCES;
 204                 break;
 205         default:
 206                 break;
 207         }
 208
 209         /* all other non-zero status results in an error */
 210         if (status)
 211                 return -EIO;
 212         return 0;
 213 }
 214
 215 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
 216                 u32 status)
 217 {
 218         if (!nvdimm)
 219                 return xlat_bus_status(buf, cmd, status);
 220         return xlat_nvdimm_status(nvdimm, buf, cmd, status);
 221 }
 222
 223 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */
 224 static union acpi_object *pkg_to_buf(union acpi_object *pkg)
 225 {
 226         int i;
 227         void *dst;
 228         size_t size = 0;
 229         union acpi_object *buf = NULL;
 230
 231         if (pkg->type != ACPI_TYPE_PACKAGE) {
 232                 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 233                                 pkg->type);
 234                 goto err;
 235         }
 236
 237         for (i = 0; i < pkg->package.count; i++) {
 238                 union acpi_object *obj = &pkg->package.elements[i];
 239
 240                 if (obj->type == ACPI_TYPE_INTEGER)
 241                         size += 4;
 242                 else if (obj->type == ACPI_TYPE_BUFFER)
 243                         size += obj->buffer.length;
 244                 else {
 245                         WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 246                                         obj->type);
 247                         goto err;
 248                 }
 249         }
 250
 251         buf = ACPI_ALLOCATE(sizeof(*buf) + size);
 252         if (!buf)
 253                 goto err;
 254
 255         dst = buf + 1;
 256         buf->type = ACPI_TYPE_BUFFER;
 257         buf->buffer.length = size;
 258         buf->buffer.pointer = dst;
 259         for (i = 0; i < pkg->package.count; i++) {
 260                 union acpi_object *obj = &pkg->package.elements[i];
 261
 262                 if (obj->type == ACPI_TYPE_INTEGER) {
 263                         memcpy(dst, &obj->integer.value, 4);
 264                         dst += 4;
 265                 } else if (obj->type == ACPI_TYPE_BUFFER) {
 266                         memcpy(dst, obj->buffer.pointer, obj->buffer.length);
 267                         dst += obj->buffer.length;
 268                 }
 269         }
 270 err:
 271         ACPI_FREE(pkg);
 272         return buf;
 273 }
 274
 275 static union acpi_object *int_to_buf(union acpi_object *integer)
 276 {
 277         union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4);
 278         void *dst = NULL;
 279
 280         if (!buf)
 281                 goto err;
 282
 283         if (integer->type != ACPI_TYPE_INTEGER) {
 284                 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 285                                 integer->type);
 286                 goto err;
 287         }
 288
 289         dst = buf + 1;
 290         buf->type = ACPI_TYPE_BUFFER;
 291         buf->buffer.length = 4;
 292         buf->buffer.pointer = dst;
 293         memcpy(dst, &integer->integer.value, 4);
 294 err:
 295         ACPI_FREE(integer);
 296         return buf;
 297 }
 298
 299 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset,
 300                 u32 len, void *data)
 301 {
 302         acpi_status rc;
 303         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 304         struct acpi_object_list input = {
 305                 .count = 3,
 306                 .pointer = (union acpi_object []) {
 307                         [0] = {
 308                                 .integer.type = ACPI_TYPE_INTEGER,
 309                                 .integer.value = offset,
 310                         },
 311                         [1] = {
 312                                 .integer.type = ACPI_TYPE_INTEGER,
 313                                 .integer.value = len,
 314                         },
 315                         [2] = {
 316                                 .buffer.type = ACPI_TYPE_BUFFER,
 317                                 .buffer.pointer = data,
 318                                 .buffer.length = len,
 319                         },
 320                 },
 321         };
 322
 323         rc = acpi_evaluate_object(handle, "_LSW", &input, &buf);
 324         if (ACPI_FAILURE(rc))
 325                 return NULL;
 326         return int_to_buf(buf.pointer);
 327 }
 328
 329 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset,
 330                 u32 len)
 331 {
 332         acpi_status rc;
 333         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 334         struct acpi_object_list input = {
 335                 .count = 2,
 336                 .pointer = (union acpi_object []) {
 337                         [0] = {
 338                                 .integer.type = ACPI_TYPE_INTEGER,
 339                                 .integer.value = offset,
 340                         },
 341                         [1] = {
 342                                 .integer.type = ACPI_TYPE_INTEGER,
 343                                 .integer.value = len,
 344                         },
 345                 },
 346         };
 347
 348         rc = acpi_evaluate_object(handle, "_LSR", &input, &buf);
 349         if (ACPI_FAILURE(rc))
 350                 return NULL;
 351         return pkg_to_buf(buf.pointer);
 352 }
 353
 354 static union acpi_object *acpi_label_info(acpi_handle handle)
 355 {
 356         acpi_status rc;
 357         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 358
 359         rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf);
 360         if (ACPI_FAILURE(rc))
 361                 return NULL;
 362         return pkg_to_buf(buf.pointer);
 363 }
 364
 365 static u8 nfit_dsm_revid(unsigned family, unsigned func)
 366 {
 367         static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = {
 368                 [NVDIMM_FAMILY_INTEL] = {
 369                         [NVDIMM_INTEL_GET_MODES] = 2,
 370                         [NVDIMM_INTEL_GET_FWINFO] = 2,
 371                         [NVDIMM_INTEL_START_FWUPDATE] = 2,
 372                         [NVDIMM_INTEL_SEND_FWUPDATE] = 2,
 373                         [NVDIMM_INTEL_FINISH_FWUPDATE] = 2,
 374                         [NVDIMM_INTEL_QUERY_FWUPDATE] = 2,
 375                         [NVDIMM_INTEL_SET_THRESHOLD] = 2,
 376                         [NVDIMM_INTEL_INJECT_ERROR] = 2,
 377                         [NVDIMM_INTEL_GET_SECURITY_STATE] = 2,
 378                         [NVDIMM_INTEL_SET_PASSPHRASE] = 2,
 379                         [NVDIMM_INTEL_DISABLE_PASSPHRASE] = 2,
 380                         [NVDIMM_INTEL_UNLOCK_UNIT] = 2,
 381                         [NVDIMM_INTEL_FREEZE_LOCK] = 2,
 382                         [NVDIMM_INTEL_SECURE_ERASE] = 2,
 383                         [NVDIMM_INTEL_OVERWRITE] = 2,
 384                         [NVDIMM_INTEL_QUERY_OVERWRITE] = 2,
 385                         [NVDIMM_INTEL_SET_MASTER_PASSPHRASE] = 2,
 386                         [NVDIMM_INTEL_MASTER_SECURE_ERASE] = 2,
 387                 },
 388         };
 389         u8 id;
 390
 391         if (family > NVDIMM_FAMILY_MAX)
 392                 return 0;
 393         if (func > 31)
 394                 return 0;
 395         id = revid_table[family][func];
 396         if (id == 0)
 397                 return 1; /* default */
 398         return id;
 399 }
 400
 401 static bool payload_dumpable(struct nvdimm *nvdimm, unsigned int func)
 402 {
 403         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 404
 405         if (nfit_mem && nfit_mem->family == NVDIMM_FAMILY_INTEL
 406                         && func >= NVDIMM_INTEL_GET_SECURITY_STATE
 407                         && func <= NVDIMM_INTEL_MASTER_SECURE_ERASE)
 408                 return IS_ENABLED(CONFIG_NFIT_SECURITY_DEBUG);
 409         return true;
 410 }
 411
 412 static int cmd_to_func(struct nfit_mem *nfit_mem, unsigned int cmd,
 413                 struct nd_cmd_pkg *call_pkg)
 414 {
 415         if (call_pkg) {
 416                 int i;
 417
 418                 if (nfit_mem && nfit_mem->family != call_pkg->nd_family)
 419                         return -ENOTTY;
 420
 421                 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++)
 422                         if (call_pkg->nd_reserved2[i])
 423                                 return -EINVAL;
 424                 return call_pkg->nd_command;
 425         }
 426
 427         /* In the !call_pkg case, bus commands == bus functions */
 428         if (!nfit_mem)
 429                 return cmd;
 430
 431         /* Linux ND commands == NVDIMM_FAMILY_INTEL function numbers */
 432         if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
 433                 return cmd;
 434
 435         /*
 436          * Force function number validation to fail since 0 is never
 437          * published as a valid function in dsm_mask.
 438          */
 439         return 0;
 440 }
 441
 442 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
 443                 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc)
 444 {
 445         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
 446         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 447         union acpi_object in_obj, in_buf, *out_obj;
 448         const struct nd_cmd_desc *desc = NULL;
 449         struct device *dev = acpi_desc->dev;
 450         struct nd_cmd_pkg *call_pkg = NULL;
 451         const char *cmd_name, *dimm_name;
 452         unsigned long cmd_mask, dsm_mask;
 453         u32 offset, fw_status = 0;
 454         acpi_handle handle;
 455         const guid_t *guid;
 456         int func, rc, i;
 457
 458         if (cmd_rc)
 459                 *cmd_rc = -EINVAL;
 460
 461         if (cmd == ND_CMD_CALL)
 462                 call_pkg = buf;
 463         func = cmd_to_func(nfit_mem, cmd, call_pkg);
 464         if (func < 0)
 465                 return func;
 466
 467         if (nvdimm) {
 468                 struct acpi_device *adev = nfit_mem->adev;
 469
 470                 if (!adev)
 471                         return -ENOTTY;
 472
 473                 dimm_name = nvdimm_name(nvdimm);
 474                 cmd_name = nvdimm_cmd_name(cmd);
 475                 cmd_mask = nvdimm_cmd_mask(nvdimm);
 476                 dsm_mask = nfit_mem->dsm_mask;
 477                 desc = nd_cmd_dimm_desc(cmd);
 478                 guid = to_nfit_uuid(nfit_mem->family);
 479                 handle = adev->handle;
 480         } else {
 481                 struct acpi_device *adev = to_acpi_dev(acpi_desc);
 482
 483                 cmd_name = nvdimm_bus_cmd_name(cmd);
 484                 cmd_mask = nd_desc->cmd_mask;
 485                 dsm_mask = nd_desc->bus_dsm_mask;
 486                 desc = nd_cmd_bus_desc(cmd);
 487                 guid = to_nfit_uuid(NFIT_DEV_BUS);
 488                 handle = adev->handle;
 489                 dimm_name = "bus";
 490         }
 491
 492         if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
 493                 return -ENOTTY;
 494
 495         /*
 496          * Check for a valid command.  For ND_CMD_CALL, we also have to
 497          * make sure that the DSM function is supported.
 498          */
 499         if (cmd == ND_CMD_CALL && !test_bit(func, &dsm_mask))
 500                 return -ENOTTY;
 501         else if (!test_bit(cmd, &cmd_mask))
 502                 return -ENOTTY;
 503
 504         in_obj.type = ACPI_TYPE_PACKAGE;
 505         in_obj.package.count = 1;
 506         in_obj.package.elements = &in_buf;
 507         in_buf.type = ACPI_TYPE_BUFFER;
 508         in_buf.buffer.pointer = buf;
 509         in_buf.buffer.length = 0;
 510
 511         /* libnvdimm has already validated the input envelope */
 512         for (i = 0; i < desc->in_num; i++)
 513                 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
 514                                 i, buf);
 515
 516         if (call_pkg) {
 517                 /* skip over package wrapper */
 518                 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
 519                 in_buf.buffer.length = call_pkg->nd_size_in;
 520         }
 521
 522         dev_dbg(dev, "%s cmd: %d: func: %d input length: %d\n",
 523                 dimm_name, cmd, func, in_buf.buffer.length);
 524         if (payload_dumpable(nvdimm, func))
 525                 print_hex_dump_debug("nvdimm in  ", DUMP_PREFIX_OFFSET, 4, 4,
 526                                 in_buf.buffer.pointer,
 527                                 min_t(u32, 256, in_buf.buffer.length), true);
 528
 529         /* call the BIOS, prefer the named methods over _DSM if available */
 530         if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE
 531                         && test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
 532                 out_obj = acpi_label_info(handle);
 533         else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA
 534                         && test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
 535                 struct nd_cmd_get_config_data_hdr *p = buf;
 536
 537                 out_obj = acpi_label_read(handle, p->in_offset, p->in_length);
 538         } else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA
 539                         && test_bit(NFIT_MEM_LSW, &nfit_mem->flags)) {
 540                 struct nd_cmd_set_config_hdr *p = buf;
 541
 542                 out_obj = acpi_label_write(handle, p->in_offset, p->in_length,
 543                                 p->in_buf);
 544         } else {
 545                 u8 revid;
 546
 547                 if (nvdimm)
 548                         revid = nfit_dsm_revid(nfit_mem->family, func);
 549                 else
 550                         revid = 1;
 551                 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
 552         }
 553
 554         if (!out_obj) {
 555                 dev_dbg(dev, "%s _DSM failed cmd: %s\n", dimm_name, cmd_name);
 556                 return -EINVAL;
 557         }
 558
 559         if (call_pkg) {
 560                 call_pkg->nd_fw_size = out_obj->buffer.length;
 561                 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
 562                         out_obj->buffer.pointer,
 563                         min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
 564
 565                 ACPI_FREE(out_obj);
 566                 /*
 567                  * Need to support FW function w/o known size in advance.
 568                  * Caller can determine required size based upon nd_fw_size.
 569                  * If we return an error (like elsewhere) then caller wouldn't
 570                  * be able to rely upon data returned to make calculation.
 571                  */
 572                 if (cmd_rc)
 573                         *cmd_rc = 0;
 574                 return 0;
 575         }
 576
 577         if (out_obj->package.type != ACPI_TYPE_BUFFER) {
 578                 dev_dbg(dev, "%s unexpected output object type cmd: %s type: %d\n",
 579                                 dimm_name, cmd_name, out_obj->type);
 580                 rc = -EINVAL;
 581                 goto out;
 582         }
 583
 584         dev_dbg(dev, "%s cmd: %s output length: %d\n", dimm_name,
 585                         cmd_name, out_obj->buffer.length);
 586         print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4,
 587                         out_obj->buffer.pointer,
 588                         min_t(u32, 128, out_obj->buffer.length), true);
 589
 590         for (i = 0, offset = 0; i < desc->out_num; i++) {
 591                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
 592                                 (u32 *) out_obj->buffer.pointer,
 593                                 out_obj->buffer.length - offset);
 594
 595                 if (offset + out_size > out_obj->buffer.length) {
 596                         dev_dbg(dev, "%s output object underflow cmd: %s field: %d\n",
 597                                         dimm_name, cmd_name, i);
 598                         break;
 599                 }
 600
 601                 if (in_buf.buffer.length + offset + out_size > buf_len) {
 602                         dev_dbg(dev, "%s output overrun cmd: %s field: %d\n",
 603                                         dimm_name, cmd_name, i);
 604                         rc = -ENXIO;
 605                         goto out;
 606                 }
 607                 memcpy(buf + in_buf.buffer.length + offset,
 608                                 out_obj->buffer.pointer + offset, out_size);
 609                 offset += out_size;
 610         }
 611
 612         /*
 613          * Set fw_status for all the commands with a known format to be
 614          * later interpreted by xlat_status().
 615          */
 616         if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP
 617                                         && cmd <= ND_CMD_CLEAR_ERROR)
 618                                 || (nvdimm && cmd >= ND_CMD_SMART
 619                                         && cmd <= ND_CMD_VENDOR)))
 620                 fw_status = *(u32 *) out_obj->buffer.pointer;
 621
 622         if (offset + in_buf.buffer.length < buf_len) {
 623                 if (i >= 1) {
 624                         /*
 625                          * status valid, return the number of bytes left
 626                          * unfilled in the output buffer
 627                          */
 628                         rc = buf_len - offset - in_buf.buffer.length;
 629                         if (cmd_rc)
 630                                 *cmd_rc = xlat_status(nvdimm, buf, cmd,
 631                                                 fw_status);
 632                 } else {
 633                         dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
 634                                         __func__, dimm_name, cmd_name, buf_len,
 635                                         offset);
 636                         rc = -ENXIO;
 637                 }
 638         } else {
 639                 rc = 0;
 640                 if (cmd_rc)
 641                         *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status);
 642         }
 643
 644  out:
 645         ACPI_FREE(out_obj);
 646
 647         return rc;
 648 }
 649 EXPORT_SYMBOL_GPL(acpi_nfit_ctl);
 650
 651 static const char *spa_type_name(u16 type)
 652 {
 653         static const char *to_name[] = {
 654                 [NFIT_SPA_VOLATILE] = "volatile",
 655                 [NFIT_SPA_PM] = "pmem",
 656                 [NFIT_SPA_DCR] = "dimm-control-region",
 657                 [NFIT_SPA_BDW] = "block-data-window",
 658                 [NFIT_SPA_VDISK] = "volatile-disk",
 659                 [NFIT_SPA_VCD] = "volatile-cd",
 660                 [NFIT_SPA_PDISK] = "persistent-disk",
 661                 [NFIT_SPA_PCD] = "persistent-cd",
 662
 663         };
 664
 665         if (type > NFIT_SPA_PCD)
 666                 return "unknown";
 667
 668         return to_name[type];
 669 }
 670
 671 int nfit_spa_type(struct acpi_nfit_system_address *spa)
 672 {
 673         int i;
 674
 675         for (i = 0; i < NFIT_UUID_MAX; i++)
 676                 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid))
 677                         return i;
 678         return -1;
 679 }
 680
 681 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
 682                 struct nfit_table_prev *prev,
 683                 struct acpi_nfit_system_address *spa)
 684 {
 685         struct device *dev = acpi_desc->dev;
 686         struct nfit_spa *nfit_spa;
 687
 688         if (spa->header.length != sizeof(*spa))
 689                 return false;
 690
 691         list_for_each_entry(nfit_spa, &prev->spas, list) {
 692                 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) {
 693                         list_move_tail(&nfit_spa->list, &acpi_desc->spas);
 694                         return true;
 695                 }
 696         }
 697
 698         nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa),
 699                         GFP_KERNEL);
 700         if (!nfit_spa)
 701                 return false;
 702         INIT_LIST_HEAD(&nfit_spa->list);
 703         memcpy(nfit_spa->spa, spa, sizeof(*spa));
 704         list_add_tail(&nfit_spa->list, &acpi_desc->spas);
 705         dev_dbg(dev, "spa index: %d type: %s\n",
 706                         spa->range_index,
 707                         spa_type_name(nfit_spa_type(spa)));
 708         return true;
 709 }
 710
 711 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
 712                 struct nfit_table_prev *prev,
 713                 struct acpi_nfit_memory_map *memdev)
 714 {
 715         struct device *dev = acpi_desc->dev;
 716         struct nfit_memdev *nfit_memdev;
 717
 718         if (memdev->header.length != sizeof(*memdev))
 719                 return false;
 720
 721         list_for_each_entry(nfit_memdev, &prev->memdevs, list)
 722                 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) {
 723                         list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 724                         return true;
 725                 }
 726
 727         nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev),
 728                         GFP_KERNEL);
 729         if (!nfit_memdev)
 730                 return false;
 731         INIT_LIST_HEAD(&nfit_memdev->list);
 732         memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev));
 733         list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 734         dev_dbg(dev, "memdev handle: %#x spa: %d dcr: %d flags: %#x\n",
 735                         memdev->device_handle, memdev->range_index,
 736                         memdev->region_index, memdev->flags);
 737         return true;
 738 }
 739
 740 int nfit_get_smbios_id(u32 device_handle, u16 *flags)
 741 {
 742         struct acpi_nfit_memory_map *memdev;
 743         struct acpi_nfit_desc *acpi_desc;
 744         struct nfit_mem *nfit_mem;
 745         u16 physical_id;
 746
 747         mutex_lock(&acpi_desc_lock);
 748         list_for_each_entry(acpi_desc, &acpi_descs, list) {
 749                 mutex_lock(&acpi_desc->init_mutex);
 750                 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
 751                         memdev = __to_nfit_memdev(nfit_mem);
 752                         if (memdev->device_handle == device_handle) {
 753                                 *flags = memdev->flags;
 754                                 physical_id = memdev->physical_id;
 755                                 mutex_unlock(&acpi_desc->init_mutex);
 756                                 mutex_unlock(&acpi_desc_lock);
 757                                 return physical_id;
 758                         }
 759                 }
 760                 mutex_unlock(&acpi_desc->init_mutex);
 761         }
 762         mutex_unlock(&acpi_desc_lock);
 763
 764         return -ENODEV;
 765 }
 766 EXPORT_SYMBOL_GPL(nfit_get_smbios_id);
 767
 768 /*
 769  * An implementation may provide a truncated control region if no block windows
 770  * are defined.
 771  */
 772 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr)
 773 {
 774         if (dcr->header.length < offsetof(struct acpi_nfit_control_region,
 775                                 window_size))
 776                 return 0;
 777         if (dcr->windows)
 778                 return sizeof(*dcr);
 779         return offsetof(struct acpi_nfit_control_region, window_size);
 780 }
 781
 782 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
 783                 struct nfit_table_prev *prev,
 784                 struct acpi_nfit_control_region *dcr)
 785 {
 786         struct device *dev = acpi_desc->dev;
 787         struct nfit_dcr *nfit_dcr;
 788
 789         if (!sizeof_dcr(dcr))
 790                 return false;
 791
 792         list_for_each_entry(nfit_dcr, &prev->dcrs, list)
 793                 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) {
 794                         list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 795                         return true;
 796                 }
 797
 798         nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr),
 799                         GFP_KERNEL);
 800         if (!nfit_dcr)
 801                 return false;
 802         INIT_LIST_HEAD(&nfit_dcr->list);
 803         memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr));
 804         list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 805         dev_dbg(dev, "dcr index: %d windows: %d\n",
 806                         dcr->region_index, dcr->windows);
 807         return true;
 808 }
 809
 810 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
 811                 struct nfit_table_prev *prev,
 812                 struct acpi_nfit_data_region *bdw)
 813 {
 814         struct device *dev = acpi_desc->dev;
 815         struct nfit_bdw *nfit_bdw;
 816
 817         if (bdw->header.length != sizeof(*bdw))
 818                 return false;
 819         list_for_each_entry(nfit_bdw, &prev->bdws, list)
 820                 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) {
 821                         list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
 822                         return true;
 823                 }
 824
 825         nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw),
 826                         GFP_KERNEL);
 827         if (!nfit_bdw)
 828                 return false;
 829         INIT_LIST_HEAD(&nfit_bdw->list);
 830         memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw));
 831         list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
 832         dev_dbg(dev, "bdw dcr: %d windows: %d\n",
 833                         bdw->region_index, bdw->windows);
 834         return true;
 835 }
 836
 837 static size_t sizeof_idt(struct acpi_nfit_interleave *idt)
 838 {
 839         if (idt->header.length < sizeof(*idt))
 840                 return 0;
 841         return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1);
 842 }
 843
 844 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
 845                 struct nfit_table_prev *prev,
 846                 struct acpi_nfit_interleave *idt)
 847 {
 848         struct device *dev = acpi_desc->dev;
 849         struct nfit_idt *nfit_idt;
 850
 851         if (!sizeof_idt(idt))
 852                 return false;
 853
 854         list_for_each_entry(nfit_idt, &prev->idts, list) {
 855                 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt))
 856                         continue;
 857
 858                 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) {
 859                         list_move_tail(&nfit_idt->list, &acpi_desc->idts);
 860                         return true;
 861                 }
 862         }
 863
 864         nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt),
 865                         GFP_KERNEL);
 866         if (!nfit_idt)
 867                 return false;
 868         INIT_LIST_HEAD(&nfit_idt->list);
 869         memcpy(nfit_idt->idt, idt, sizeof_idt(idt));
 870         list_add_tail(&nfit_idt->list, &acpi_desc->idts);
 871         dev_dbg(dev, "idt index: %d num_lines: %d\n",
 872                         idt->interleave_index, idt->line_count);
 873         return true;
 874 }
 875
 876 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush)
 877 {
 878         if (flush->header.length < sizeof(*flush))
 879                 return 0;
 880         return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1);
 881 }
 882
 883 static bool add_flush(struct acpi_nfit_desc *acpi_desc,
 884                 struct nfit_table_prev *prev,
 885                 struct acpi_nfit_flush_address *flush)
 886 {
 887         struct device *dev = acpi_desc->dev;
 888         struct nfit_flush *nfit_flush;
 889
 890         if (!sizeof_flush(flush))
 891                 return false;
 892
 893         list_for_each_entry(nfit_flush, &prev->flushes, list) {
 894                 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush))
 895                         continue;
 896
 897                 if (memcmp(nfit_flush->flush, flush,
 898                                         sizeof_flush(flush)) == 0) {
 899                         list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
 900                         return true;
 901                 }
 902         }
 903
 904         nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush)
 905                         + sizeof_flush(flush), GFP_KERNEL);
 906         if (!nfit_flush)
 907                 return false;
 908         INIT_LIST_HEAD(&nfit_flush->list);
 909         memcpy(nfit_flush->flush, flush, sizeof_flush(flush));
 910         list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
 911         dev_dbg(dev, "nfit_flush handle: %d hint_count: %d\n",
 912                         flush->device_handle, flush->hint_count);
 913         return true;
 914 }
 915
 916 static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc,
 917                 struct acpi_nfit_capabilities *pcap)
 918 {
 919         struct device *dev = acpi_desc->dev;
 920         u32 mask;
 921
 922         mask = (1 << (pcap->highest_capability + 1)) - 1;
 923         acpi_desc->platform_cap = pcap->capabilities & mask;
 924         dev_dbg(dev, "cap: %#x\n", acpi_desc->platform_cap);
 925         return true;
 926 }
 927
 928 static void *add_table(struct acpi_nfit_desc *acpi_desc,
 929                 struct nfit_table_prev *prev, void *table, const void *end)
 930 {
 931         struct device *dev = acpi_desc->dev;
 932         struct acpi_nfit_header *hdr;
 933         void *err = ERR_PTR(-ENOMEM);
 934
 935         if (table >= end)
 936                 return NULL;
 937
 938         hdr = table;
 939         if (!hdr->length) {
 940                 dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
 941                         hdr->type);
 942                 return NULL;
 943         }
 944
 945         switch (hdr->type) {
 946         case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
 947                 if (!add_spa(acpi_desc, prev, table))
 948                         return err;
 949                 break;
 950         case ACPI_NFIT_TYPE_MEMORY_MAP:
 951                 if (!add_memdev(acpi_desc, prev, table))
 952                         return err;
 953                 break;
 954         case ACPI_NFIT_TYPE_CONTROL_REGION:
 955                 if (!add_dcr(acpi_desc, prev, table))
 956                         return err;
 957                 break;
 958         case ACPI_NFIT_TYPE_DATA_REGION:
 959                 if (!add_bdw(acpi_desc, prev, table))
 960                         return err;
 961                 break;
 962         case ACPI_NFIT_TYPE_INTERLEAVE:
 963                 if (!add_idt(acpi_desc, prev, table))
 964                         return err;
 965                 break;
 966         case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
 967                 if (!add_flush(acpi_desc, prev, table))
 968                         return err;
 969                 break;
 970         case ACPI_NFIT_TYPE_SMBIOS:
 971                 dev_dbg(dev, "smbios\n");
 972                 break;
 973         case ACPI_NFIT_TYPE_CAPABILITIES:
 974                 if (!add_platform_cap(acpi_desc, table))
 975                         return err;
 976                 break;
 977         default:
 978                 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
 979                 break;
 980         }
 981
 982         return table + hdr->length;
 983 }
 984
 985 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
 986                 struct nfit_mem *nfit_mem)
 987 {
 988         u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 989         u16 dcr = nfit_mem->dcr->region_index;
 990         struct nfit_spa *nfit_spa;
 991
 992         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 993                 u16 range_index = nfit_spa->spa->range_index;
 994                 int type = nfit_spa_type(nfit_spa->spa);
 995                 struct nfit_memdev *nfit_memdev;
 996
 997                 if (type != NFIT_SPA_BDW)
 998                         continue;
 999
1000                 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1001                         if (nfit_memdev->memdev->range_index != range_index)
1002                                 continue;
1003                         if (nfit_memdev->memdev->device_handle != device_handle)
1004                                 continue;
1005                         if (nfit_memdev->memdev->region_index != dcr)
1006                                 continue;
1007
1008                         nfit_mem->spa_bdw = nfit_spa->spa;
1009                         return;
1010                 }
1011         }
1012
1013         dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
1014                         nfit_mem->spa_dcr->range_index);
1015         nfit_mem->bdw = NULL;
1016 }
1017
1018 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
1019                 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
1020 {
1021         u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
1022         struct nfit_memdev *nfit_memdev;
1023         struct nfit_bdw *nfit_bdw;
1024         struct nfit_idt *nfit_idt;
1025         u16 idt_idx, range_index;
1026
1027         list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
1028                 if (nfit_bdw->bdw->region_index != dcr)
1029                         continue;
1030                 nfit_mem->bdw = nfit_bdw->bdw;
1031                 break;
1032         }
1033
1034         if (!nfit_mem->bdw)
1035                 return;
1036
1037         nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
1038
1039         if (!nfit_mem->spa_bdw)
1040                 return;
1041
1042         range_index = nfit_mem->spa_bdw->range_index;
1043         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1044                 if (nfit_memdev->memdev->range_index != range_index ||
1045                                 nfit_memdev->memdev->region_index != dcr)
1046                         continue;
1047                 nfit_mem->memdev_bdw = nfit_memdev->memdev;
1048                 idt_idx = nfit_memdev->memdev->interleave_index;
1049                 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1050                         if (nfit_idt->idt->interleave_index != idt_idx)
1051                                 continue;
1052                         nfit_mem->idt_bdw = nfit_idt->idt;
1053                         break;
1054                 }
1055                 break;
1056         }
1057 }
1058
1059 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc,
1060                 struct acpi_nfit_system_address *spa)
1061 {
1062         struct nfit_mem *nfit_mem, *found;
1063         struct nfit_memdev *nfit_memdev;
1064         int type = spa ? nfit_spa_type(spa) : 0;
1065
1066         switch (type) {
1067         case NFIT_SPA_DCR:
1068         case NFIT_SPA_PM:
1069                 break;
1070         default:
1071                 if (spa)
1072                         return 0;
1073         }
1074
1075         /*
1076          * This loop runs in two modes, when a dimm is mapped the loop
1077          * adds memdev associations to an existing dimm, or creates a
1078          * dimm. In the unmapped dimm case this loop sweeps for memdev
1079          * instances with an invalid / zero range_index and adds those
1080          * dimms without spa associations.
1081          */
1082         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1083                 struct nfit_flush *nfit_flush;
1084                 struct nfit_dcr *nfit_dcr;
1085                 u32 device_handle;
1086                 u16 dcr;
1087
1088                 if (spa && nfit_memdev->memdev->range_index != spa->range_index)
1089                         continue;
1090                 if (!spa && nfit_memdev->memdev->range_index)
1091                         continue;
1092                 found = NULL;
1093                 dcr = nfit_memdev->memdev->region_index;
1094                 device_handle = nfit_memdev->memdev->device_handle;
1095                 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1096                         if (__to_nfit_memdev(nfit_mem)->device_handle
1097                                         == device_handle) {
1098                                 found = nfit_mem;
1099                                 break;
1100                         }
1101
1102                 if (found)
1103                         nfit_mem = found;
1104                 else {
1105                         nfit_mem = devm_kzalloc(acpi_desc->dev,
1106                                         sizeof(*nfit_mem), GFP_KERNEL);
1107                         if (!nfit_mem)
1108                                 return -ENOMEM;
1109                         INIT_LIST_HEAD(&nfit_mem->list);
1110                         nfit_mem->acpi_desc = acpi_desc;
1111                         list_add(&nfit_mem->list, &acpi_desc->dimms);
1112                 }
1113
1114                 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1115                         if (nfit_dcr->dcr->region_index != dcr)
1116                                 continue;
1117                         /*
1118                          * Record the control region for the dimm.  For
1119                          * the ACPI 6.1 case, where there are separate
1120                          * control regions for the pmem vs blk
1121                          * interfaces, be sure to record the extended
1122                          * blk details.
1123                          */
1124                         if (!nfit_mem->dcr)
1125                                 nfit_mem->dcr = nfit_dcr->dcr;
1126                         else if (nfit_mem->dcr->windows == 0
1127                                         && nfit_dcr->dcr->windows)
1128                                 nfit_mem->dcr = nfit_dcr->dcr;
1129                         break;
1130                 }
1131
1132                 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
1133                         struct acpi_nfit_flush_address *flush;
1134                         u16 i;
1135
1136                         if (nfit_flush->flush->device_handle != device_handle)
1137                                 continue;
1138                         nfit_mem->nfit_flush = nfit_flush;
1139                         flush = nfit_flush->flush;
1140                         nfit_mem->flush_wpq = devm_kcalloc(acpi_desc->dev,
1141                                         flush->hint_count,
1142                                         sizeof(struct resource),
1143                                         GFP_KERNEL);
1144                         if (!nfit_mem->flush_wpq)
1145                                 return -ENOMEM;
1146                         for (i = 0; i < flush->hint_count; i++) {
1147                                 struct resource *res = &nfit_mem->flush_wpq[i];
1148
1149                                 res->start = flush->hint_address[i];
1150                                 res->end = res->start + 8 - 1;
1151                         }
1152                         break;
1153                 }
1154
1155                 if (dcr && !nfit_mem->dcr) {
1156                         dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
1157                                         spa->range_index, dcr);
1158                         return -ENODEV;
1159                 }
1160
1161                 if (type == NFIT_SPA_DCR) {
1162                         struct nfit_idt *nfit_idt;
1163                         u16 idt_idx;
1164
1165                         /* multiple dimms may share a SPA when interleaved */
1166                         nfit_mem->spa_dcr = spa;
1167                         nfit_mem->memdev_dcr = nfit_memdev->memdev;
1168                         idt_idx = nfit_memdev->memdev->interleave_index;
1169                         list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1170                                 if (nfit_idt->idt->interleave_index != idt_idx)
1171                                         continue;
1172                                 nfit_mem->idt_dcr = nfit_idt->idt;
1173                                 break;
1174                         }
1175                         nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
1176                 } else if (type == NFIT_SPA_PM) {
1177                         /*
1178                          * A single dimm may belong to multiple SPA-PM
1179                          * ranges, record at least one in addition to
1180                          * any SPA-DCR range.
1181                          */
1182                         nfit_mem->memdev_pmem = nfit_memdev->memdev;
1183                 } else
1184                         nfit_mem->memdev_dcr = nfit_memdev->memdev;
1185         }
1186
1187         return 0;
1188 }
1189
1190 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
1191 {
1192         struct nfit_mem *a = container_of(_a, typeof(*a), list);
1193         struct nfit_mem *b = container_of(_b, typeof(*b), list);
1194         u32 handleA, handleB;
1195
1196         handleA = __to_nfit_memdev(a)->device_handle;
1197         handleB = __to_nfit_memdev(b)->device_handle;
1198         if (handleA < handleB)
1199                 return -1;
1200         else if (handleA > handleB)
1201                 return 1;
1202         return 0;
1203 }
1204
1205 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
1206 {
1207         struct nfit_spa *nfit_spa;
1208         int rc;
1209
1210
1211         /*
1212          * For each SPA-DCR or SPA-PMEM address range find its
1213          * corresponding MEMDEV(s).  From each MEMDEV find the
1214          * corresponding DCR.  Then, if we're operating on a SPA-DCR,
1215          * try to find a SPA-BDW and a corresponding BDW that references
1216          * the DCR.  Throw it all into an nfit_mem object.  Note, that
1217          * BDWs are optional.
1218          */
1219         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1220                 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa);
1221                 if (rc)
1222                         return rc;
1223         }
1224
1225         /*
1226          * If a DIMM has failed to be mapped into SPA there will be no
1227          * SPA entries above. Find and register all the unmapped DIMMs
1228          * for reporting and recovery purposes.
1229          */
1230         rc = __nfit_mem_init(acpi_desc, NULL);
1231         if (rc)
1232                 return rc;
1233
1234         list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
1235
1236         return 0;
1237 }
1238
1239 static ssize_t bus_dsm_mask_show(struct device *dev,
1240                 struct device_attribute *attr, char *buf)
1241 {
1242         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1243         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1244
1245         return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask);
1246 }
1247 static struct device_attribute dev_attr_bus_dsm_mask =
1248                 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL);
1249
1250 static ssize_t revision_show(struct device *dev,
1251                 struct device_attribute *attr, char *buf)
1252 {
1253         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1254         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1255         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1256
1257         return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
1258 }
1259 static DEVICE_ATTR_RO(revision);
1260
1261 static ssize_t hw_error_scrub_show(struct device *dev,
1262                 struct device_attribute *attr, char *buf)
1263 {
1264         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1265         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1266         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1267
1268         return sprintf(buf, "%d\n", acpi_desc->scrub_mode);
1269 }
1270
1271 /*
1272  * The 'hw_error_scrub' attribute can have the following values written to it:
1273  * '0': Switch to the default mode where an exception will only insert
1274  *      the address of the memory error into the poison and badblocks lists.
1275  * '1': Enable a full scrub to happen if an exception for a memory error is
1276  *      received.
1277  */
1278 static ssize_t hw_error_scrub_store(struct device *dev,
1279                 struct device_attribute *attr, const char *buf, size_t size)
1280 {
1281         struct nvdimm_bus_descriptor *nd_desc;
1282         ssize_t rc;
1283         long val;
1284
1285         rc = kstrtol(buf, 0, &val);
1286         if (rc)
1287                 return rc;
1288
1289         device_lock(dev);
1290         nd_desc = dev_get_drvdata(dev);
1291         if (nd_desc) {
1292                 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1293
1294                 switch (val) {
1295                 case HW_ERROR_SCRUB_ON:
1296                         acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON;
1297                         break;
1298                 case HW_ERROR_SCRUB_OFF:
1299                         acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF;
1300                         break;
1301                 default:
1302                         rc = -EINVAL;
1303                         break;
1304                 }
1305         }
1306         device_unlock(dev);
1307         if (rc)
1308                 return rc;
1309         return size;
1310 }
1311 static DEVICE_ATTR_RW(hw_error_scrub);
1312
1313 /*
1314  * This shows the number of full Address Range Scrubs that have been
1315  * completed since driver load time. Userspace can wait on this using
1316  * select/poll etc. A '+' at the end indicates an ARS is in progress
1317  */
1318 static ssize_t scrub_show(struct device *dev,
1319                 struct device_attribute *attr, char *buf)
1320 {
1321         struct nvdimm_bus_descriptor *nd_desc;
1322         ssize_t rc = -ENXIO;
1323
1324         device_lock(dev);
1325         nd_desc = dev_get_drvdata(dev);
1326         if (nd_desc) {
1327                 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1328
1329                 mutex_lock(&acpi_desc->init_mutex);
1330                 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count,
1331                                 acpi_desc->scrub_busy
1332                                 && !acpi_desc->cancel ? "+\n" : "\n");
1333                 mutex_unlock(&acpi_desc->init_mutex);
1334         }
1335         device_unlock(dev);
1336         return rc;
1337 }
1338
1339 static ssize_t scrub_store(struct device *dev,
1340                 struct device_attribute *attr, const char *buf, size_t size)
1341 {
1342         struct nvdimm_bus_descriptor *nd_desc;
1343         ssize_t rc;
1344         long val;
1345
1346         rc = kstrtol(buf, 0, &val);
1347         if (rc)
1348                 return rc;
1349         if (val != 1)
1350                 return -EINVAL;
1351
1352         device_lock(dev);
1353         nd_desc = dev_get_drvdata(dev);
1354         if (nd_desc) {
1355                 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1356
1357                 rc = acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
1358         }
1359         device_unlock(dev);
1360         if (rc)
1361                 return rc;
1362         return size;
1363 }
1364 static DEVICE_ATTR_RW(scrub);
1365
1366 static bool ars_supported(struct nvdimm_bus *nvdimm_bus)
1367 {
1368         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1369         const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START
1370                 | 1 << ND_CMD_ARS_STATUS;
1371
1372         return (nd_desc->cmd_mask & mask) == mask;
1373 }
1374
1375 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n)
1376 {
1377         struct device *dev = container_of(kobj, struct device, kobj);
1378         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1379
1380         if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus))
1381                 return 0;
1382         return a->mode;
1383 }
1384
1385 static struct attribute *acpi_nfit_attributes[] = {
1386         &dev_attr_revision.attr,
1387         &dev_attr_scrub.attr,
1388         &dev_attr_hw_error_scrub.attr,
1389         &dev_attr_bus_dsm_mask.attr,
1390         NULL,
1391 };
1392
1393 static const struct attribute_group acpi_nfit_attribute_group = {
1394         .name = "nfit",
1395         .attrs = acpi_nfit_attributes,
1396         .is_visible = nfit_visible,
1397 };
1398
1399 static const struct attribute_group *acpi_nfit_attribute_groups[] = {
1400         &nvdimm_bus_attribute_group,
1401         &acpi_nfit_attribute_group,
1402         NULL,
1403 };
1404
1405 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
1406 {
1407         struct nvdimm *nvdimm = to_nvdimm(dev);
1408         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1409
1410         return __to_nfit_memdev(nfit_mem);
1411 }
1412
1413 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
1414 {
1415         struct nvdimm *nvdimm = to_nvdimm(dev);
1416         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1417
1418         return nfit_mem->dcr;
1419 }
1420
1421 static ssize_t handle_show(struct device *dev,
1422                 struct device_attribute *attr, char *buf)
1423 {
1424         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1425
1426         return sprintf(buf, "%#x\n", memdev->device_handle);
1427 }
1428 static DEVICE_ATTR_RO(handle);
1429
1430 static ssize_t phys_id_show(struct device *dev,
1431                 struct device_attribute *attr, char *buf)
1432 {
1433         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1434
1435         return sprintf(buf, "%#x\n", memdev->physical_id);
1436 }
1437 static DEVICE_ATTR_RO(phys_id);
1438
1439 static ssize_t vendor_show(struct device *dev,
1440                 struct device_attribute *attr, char *buf)
1441 {
1442         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1443
1444         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id));
1445 }
1446 static DEVICE_ATTR_RO(vendor);
1447
1448 static ssize_t rev_id_show(struct device *dev,
1449                 struct device_attribute *attr, char *buf)
1450 {
1451         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1452
1453         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id));
1454 }
1455 static DEVICE_ATTR_RO(rev_id);
1456
1457 static ssize_t device_show(struct device *dev,
1458                 struct device_attribute *attr, char *buf)
1459 {
1460         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1461
1462         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id));
1463 }
1464 static DEVICE_ATTR_RO(device);
1465
1466 static ssize_t subsystem_vendor_show(struct device *dev,
1467                 struct device_attribute *attr, char *buf)
1468 {
1469         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1470
1471         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id));
1472 }
1473 static DEVICE_ATTR_RO(subsystem_vendor);
1474
1475 static ssize_t subsystem_rev_id_show(struct device *dev,
1476                 struct device_attribute *attr, char *buf)
1477 {
1478         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1479
1480         return sprintf(buf, "0x%04x\n",
1481                         be16_to_cpu(dcr->subsystem_revision_id));
1482 }
1483 static DEVICE_ATTR_RO(subsystem_rev_id);
1484
1485 static ssize_t subsystem_device_show(struct device *dev,
1486                 struct device_attribute *attr, char *buf)
1487 {
1488         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1489
1490         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id));
1491 }
1492 static DEVICE_ATTR_RO(subsystem_device);
1493
1494 static int num_nvdimm_formats(struct nvdimm *nvdimm)
1495 {
1496         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1497         int formats = 0;
1498
1499         if (nfit_mem->memdev_pmem)
1500                 formats++;
1501         if (nfit_mem->memdev_bdw)
1502                 formats++;
1503         return formats;
1504 }
1505
1506 static ssize_t format_show(struct device *dev,
1507                 struct device_attribute *attr, char *buf)
1508 {
1509         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1510
1511         return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code));
1512 }
1513 static DEVICE_ATTR_RO(format);
1514
1515 static ssize_t format1_show(struct device *dev,
1516                 struct device_attribute *attr, char *buf)
1517 {
1518         u32 handle;
1519         ssize_t rc = -ENXIO;
1520         struct nfit_mem *nfit_mem;
1521         struct nfit_memdev *nfit_memdev;
1522         struct acpi_nfit_desc *acpi_desc;
1523         struct nvdimm *nvdimm = to_nvdimm(dev);
1524         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1525
1526         nfit_mem = nvdimm_provider_data(nvdimm);
1527         acpi_desc = nfit_mem->acpi_desc;
1528         handle = to_nfit_memdev(dev)->device_handle;
1529
1530         /* assumes DIMMs have at most 2 published interface codes */
1531         mutex_lock(&acpi_desc->init_mutex);
1532         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1533                 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1534                 struct nfit_dcr *nfit_dcr;
1535
1536                 if (memdev->device_handle != handle)
1537                         continue;
1538
1539                 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1540                         if (nfit_dcr->dcr->region_index != memdev->region_index)
1541                                 continue;
1542                         if (nfit_dcr->dcr->code == dcr->code)
1543                                 continue;
1544                         rc = sprintf(buf, "0x%04x\n",
1545                                         le16_to_cpu(nfit_dcr->dcr->code));
1546                         break;
1547                 }
1548                 if (rc != ENXIO)
1549                         break;
1550         }
1551         mutex_unlock(&acpi_desc->init_mutex);
1552         return rc;
1553 }
1554 static DEVICE_ATTR_RO(format1);
1555
1556 static ssize_t formats_show(struct device *dev,
1557                 struct device_attribute *attr, char *buf)
1558 {
1559         struct nvdimm *nvdimm = to_nvdimm(dev);
1560
1561         return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm));
1562 }
1563 static DEVICE_ATTR_RO(formats);
1564
1565 static ssize_t serial_show(struct device *dev,
1566                 struct device_attribute *attr, char *buf)
1567 {
1568         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1569
1570         return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number));
1571 }
1572 static DEVICE_ATTR_RO(serial);
1573
1574 static ssize_t family_show(struct device *dev,
1575                 struct device_attribute *attr, char *buf)
1576 {
1577         struct nvdimm *nvdimm = to_nvdimm(dev);
1578         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1579
1580         if (nfit_mem->family < 0)
1581                 return -ENXIO;
1582         return sprintf(buf, "%d\n", nfit_mem->family);
1583 }
1584 static DEVICE_ATTR_RO(family);
1585
1586 static ssize_t dsm_mask_show(struct device *dev,
1587                 struct device_attribute *attr, char *buf)
1588 {
1589         struct nvdimm *nvdimm = to_nvdimm(dev);
1590         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1591
1592         if (nfit_mem->family < 0)
1593                 return -ENXIO;
1594         return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask);
1595 }
1596 static DEVICE_ATTR_RO(dsm_mask);
1597
1598 static ssize_t flags_show(struct device *dev,
1599                 struct device_attribute *attr, char *buf)
1600 {
1601         struct nvdimm *nvdimm = to_nvdimm(dev);
1602         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1603         u16 flags = __to_nfit_memdev(nfit_mem)->flags;
1604
1605         if (test_bit(NFIT_MEM_DIRTY, &nfit_mem->flags))
1606                 flags |= ACPI_NFIT_MEM_FLUSH_FAILED;
1607
1608         return sprintf(buf, "%s%s%s%s%s%s%s\n",
1609                 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
1610                 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
1611                 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
1612                 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
1613                 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "",
1614                 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "",
1615                 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : "");
1616 }
1617 static DEVICE_ATTR_RO(flags);
1618
1619 static ssize_t id_show(struct device *dev,
1620                 struct device_attribute *attr, char *buf)
1621 {
1622         struct nvdimm *nvdimm = to_nvdimm(dev);
1623         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1624
1625         return sprintf(buf, "%s\n", nfit_mem->id);
1626 }
1627 static DEVICE_ATTR_RO(id);
1628
1629 static ssize_t dirty_shutdown_show(struct device *dev,
1630                 struct device_attribute *attr, char *buf)
1631 {
1632         struct nvdimm *nvdimm = to_nvdimm(dev);
1633         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1634
1635         return sprintf(buf, "%d\n", nfit_mem->dirty_shutdown);
1636 }
1637 static DEVICE_ATTR_RO(dirty_shutdown);
1638
1639 static struct attribute *acpi_nfit_dimm_attributes[] = {
1640         &dev_attr_handle.attr,
1641         &dev_attr_phys_id.attr,
1642         &dev_attr_vendor.attr,
1643         &dev_attr_device.attr,
1644         &dev_attr_rev_id.attr,
1645         &dev_attr_subsystem_vendor.attr,
1646         &dev_attr_subsystem_device.attr,
1647         &dev_attr_subsystem_rev_id.attr,
1648         &dev_attr_format.attr,
1649         &dev_attr_formats.attr,
1650         &dev_attr_format1.attr,
1651         &dev_attr_serial.attr,
1652         &dev_attr_flags.attr,
1653         &dev_attr_id.attr,
1654         &dev_attr_family.attr,
1655         &dev_attr_dsm_mask.attr,
1656         &dev_attr_dirty_shutdown.attr,
1657         NULL,
1658 };
1659
1660 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
1661                 struct attribute *a, int n)
1662 {
1663         struct device *dev = container_of(kobj, struct device, kobj);
1664         struct nvdimm *nvdimm = to_nvdimm(dev);
1665         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1666
1667         if (!to_nfit_dcr(dev)) {
1668                 /* Without a dcr only the memdev attributes can be surfaced */
1669                 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr
1670                                 || a == &dev_attr_flags.attr
1671                                 || a == &dev_attr_family.attr
1672                                 || a == &dev_attr_dsm_mask.attr)
1673                         return a->mode;
1674                 return 0;
1675         }
1676
1677         if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1)
1678                 return 0;
1679
1680         if (!test_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags)
1681                         && a == &dev_attr_dirty_shutdown.attr)
1682                 return 0;
1683
1684         return a->mode;
1685 }
1686
1687 static const struct attribute_group acpi_nfit_dimm_attribute_group = {
1688         .name = "nfit",
1689         .attrs = acpi_nfit_dimm_attributes,
1690         .is_visible = acpi_nfit_dimm_attr_visible,
1691 };
1692
1693 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
1694         &nvdimm_attribute_group,
1695         &nd_device_attribute_group,
1696         &acpi_nfit_dimm_attribute_group,
1697         NULL,
1698 };
1699
1700 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
1701                 u32 device_handle)
1702 {
1703         struct nfit_mem *nfit_mem;
1704
1705         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1706                 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
1707                         return nfit_mem->nvdimm;
1708
1709         return NULL;
1710 }
1711
1712 void __acpi_nvdimm_notify(struct device *dev, u32 event)
1713 {
1714         struct nfit_mem *nfit_mem;
1715         struct acpi_nfit_desc *acpi_desc;
1716
1717         dev_dbg(dev->parent, "%s: event: %d\n", dev_name(dev),
1718                         event);
1719
1720         if (event != NFIT_NOTIFY_DIMM_HEALTH) {
1721                 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev),
1722                                 event);
1723                 return;
1724         }
1725
1726         acpi_desc = dev_get_drvdata(dev->parent);
1727         if (!acpi_desc)
1728                 return;
1729
1730         /*
1731          * If we successfully retrieved acpi_desc, then we know nfit_mem data
1732          * is still valid.
1733          */
1734         nfit_mem = dev_get_drvdata(dev);
1735         if (nfit_mem && nfit_mem->flags_attr)
1736                 sysfs_notify_dirent(nfit_mem->flags_attr);
1737 }
1738 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify);
1739
1740 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
1741 {
1742         struct acpi_device *adev = data;
1743         struct device *dev = &adev->dev;
1744
1745         device_lock(dev->parent);
1746         __acpi_nvdimm_notify(dev, event);
1747         device_unlock(dev->parent);
1748 }
1749
1750 static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method)
1751 {
1752         acpi_handle handle;
1753         acpi_status status;
1754
1755         status = acpi_get_handle(adev->handle, method, &handle);
1756
1757         if (ACPI_SUCCESS(status))
1758                 return true;
1759         return false;
1760 }
1761
1762 __weak void nfit_intel_shutdown_status(struct nfit_mem *nfit_mem)
1763 {
1764         struct nd_intel_smart smart = { 0 };
1765         union acpi_object in_buf = {
1766                 .type = ACPI_TYPE_BUFFER,
1767                 .buffer.pointer = (char *) &smart,
1768                 .buffer.length = sizeof(smart),
1769         };
1770         union acpi_object in_obj = {
1771                 .type = ACPI_TYPE_PACKAGE,
1772                 .package.count = 1,
1773                 .package.elements = &in_buf,
1774         };
1775         const u8 func = ND_INTEL_SMART;
1776         const guid_t *guid = to_nfit_uuid(nfit_mem->family);
1777         u8 revid = nfit_dsm_revid(nfit_mem->family, func);
1778         struct acpi_device *adev = nfit_mem->adev;
1779         acpi_handle handle = adev->handle;
1780         union acpi_object *out_obj;
1781
1782         if ((nfit_mem->dsm_mask & (1 << func)) == 0)
1783                 return;
1784
1785         out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
1786         if (!out_obj)
1787                 return;
1788
1789         if (smart.flags & ND_INTEL_SMART_SHUTDOWN_VALID) {
1790                 if (smart.shutdown_state)
1791                         set_bit(NFIT_MEM_DIRTY, &nfit_mem->flags);
1792         }
1793
1794         if (smart.flags & ND_INTEL_SMART_SHUTDOWN_COUNT_VALID) {
1795                 set_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags);
1796                 nfit_mem->dirty_shutdown = smart.shutdown_count;
1797         }
1798         ACPI_FREE(out_obj);
1799 }
1800
1801 static void populate_shutdown_status(struct nfit_mem *nfit_mem)
1802 {
1803         /*
1804          * For DIMMs that provide a dynamic facility to retrieve a
1805          * dirty-shutdown status and/or a dirty-shutdown count, cache
1806          * these values in nfit_mem.
1807          */
1808         if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
1809                 nfit_intel_shutdown_status(nfit_mem);
1810 }
1811
1812 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
1813                 struct nfit_mem *nfit_mem, u32 device_handle)
1814 {
1815         struct acpi_device *adev, *adev_dimm;
1816         struct device *dev = acpi_desc->dev;
1817         unsigned long dsm_mask, label_mask;
1818         const guid_t *guid;
1819         int i;
1820         int family = -1;
1821         struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
1822
1823         /* nfit test assumes 1:1 relationship between commands and dsms */
1824         nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
1825         nfit_mem->family = NVDIMM_FAMILY_INTEL;
1826
1827         if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID)
1828                 sprintf(nfit_mem->id, "%04x-%02x-%04x-%08x",
1829                                 be16_to_cpu(dcr->vendor_id),
1830                                 dcr->manufacturing_location,
1831                                 be16_to_cpu(dcr->manufacturing_date),
1832                                 be32_to_cpu(dcr->serial_number));
1833         else
1834                 sprintf(nfit_mem->id, "%04x-%08x",
1835                                 be16_to_cpu(dcr->vendor_id),
1836                                 be32_to_cpu(dcr->serial_number));
1837
1838         adev = to_acpi_dev(acpi_desc);
1839         if (!adev) {
1840                 /* unit test case */
1841                 populate_shutdown_status(nfit_mem);
1842                 return 0;
1843         }
1844
1845         adev_dimm = acpi_find_child_device(adev, device_handle, false);
1846         nfit_mem->adev = adev_dimm;
1847         if (!adev_dimm) {
1848                 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
1849                                 device_handle);
1850                 return force_enable_dimms ? 0 : -ENODEV;
1851         }
1852
1853         if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle,
1854                 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) {
1855                 dev_err(dev, "%s: notification registration failed\n",
1856                                 dev_name(&adev_dimm->dev));
1857                 return -ENXIO;
1858         }
1859         /*
1860          * Record nfit_mem for the notification path to track back to
1861          * the nfit sysfs attributes for this dimm device object.
1862          */
1863         dev_set_drvdata(&adev_dimm->dev, nfit_mem);
1864
1865         /*
1866          * Until standardization materializes we need to consider 4
1867          * different command sets.  Note, that checking for function0 (bit0)
1868          * tells us if any commands are reachable through this GUID.
1869          */
1870         for (i = 0; i <= NVDIMM_FAMILY_MAX; i++)
1871                 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1))
1872                         if (family < 0 || i == default_dsm_family)
1873                                 family = i;
1874
1875         /* limit the supported commands to those that are publicly documented */
1876         nfit_mem->family = family;
1877         if (override_dsm_mask && !disable_vendor_specific)
1878                 dsm_mask = override_dsm_mask;
1879         else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
1880                 dsm_mask = NVDIMM_INTEL_CMDMASK;
1881                 if (disable_vendor_specific)
1882                         dsm_mask &= ~(1 << ND_CMD_VENDOR);
1883         } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) {
1884                 dsm_mask = 0x1c3c76;
1885         } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) {
1886                 dsm_mask = 0x1fe;
1887                 if (disable_vendor_specific)
1888                         dsm_mask &= ~(1 << 8);
1889         } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) {
1890                 dsm_mask = 0xffffffff;
1891         } else {
1892                 dev_dbg(dev, "unknown dimm command family\n");
1893                 nfit_mem->family = -1;
1894                 /* DSMs are optional, continue loading the driver... */
1895                 return 0;
1896         }
1897
1898         /*
1899          * Function 0 is the command interrogation function, don't
1900          * export it to potential userspace use, and enable it to be
1901          * used as an error value in acpi_nfit_ctl().
1902          */
1903         dsm_mask &= ~1UL;
1904
1905         guid = to_nfit_uuid(nfit_mem->family);
1906         for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
1907                 if (acpi_check_dsm(adev_dimm->handle, guid,
1908                                         nfit_dsm_revid(nfit_mem->family, i),
1909                                         1ULL << i))
1910                         set_bit(i, &nfit_mem->dsm_mask);
1911
1912         /*
1913          * Prefer the NVDIMM_FAMILY_INTEL label read commands if present
1914          * due to their better semantics handling locked capacity.
1915          */
1916         label_mask = 1 << ND_CMD_GET_CONFIG_SIZE | 1 << ND_CMD_GET_CONFIG_DATA
1917                 | 1 << ND_CMD_SET_CONFIG_DATA;
1918         if (family == NVDIMM_FAMILY_INTEL
1919                         && (dsm_mask & label_mask) == label_mask)
1920                 return 0;
1921
1922         if (acpi_nvdimm_has_method(adev_dimm, "_LSI")
1923                         && acpi_nvdimm_has_method(adev_dimm, "_LSR")) {
1924                 dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev));
1925                 set_bit(NFIT_MEM_LSR, &nfit_mem->flags);
1926         }
1927
1928         if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
1929                         && acpi_nvdimm_has_method(adev_dimm, "_LSW")) {
1930                 dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev));
1931                 set_bit(NFIT_MEM_LSW, &nfit_mem->flags);
1932         }
1933
1934         populate_shutdown_status(nfit_mem);
1935
1936         return 0;
1937 }
1938
1939 static void shutdown_dimm_notify(void *data)
1940 {
1941         struct acpi_nfit_desc *acpi_desc = data;
1942         struct nfit_mem *nfit_mem;
1943
1944         mutex_lock(&acpi_desc->init_mutex);
1945         /*
1946          * Clear out the nfit_mem->flags_attr and shut down dimm event
1947          * notifications.
1948          */
1949         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1950                 struct acpi_device *adev_dimm = nfit_mem->adev;
1951
1952                 if (nfit_mem->flags_attr) {
1953                         sysfs_put(nfit_mem->flags_attr);
1954                         nfit_mem->flags_attr = NULL;
1955                 }
1956                 if (adev_dimm) {
1957                         acpi_remove_notify_handler(adev_dimm->handle,
1958                                         ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify);
1959                         dev_set_drvdata(&adev_dimm->dev, NULL);
1960                 }
1961         }
1962         mutex_unlock(&acpi_desc->init_mutex);
1963 }
1964
1965 static const struct nvdimm_security_ops *acpi_nfit_get_security_ops(int family)
1966 {
1967         switch (family) {
1968         case NVDIMM_FAMILY_INTEL:
1969                 return intel_security_ops;
1970         default:
1971                 return NULL;
1972         }
1973 }
1974
1975 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
1976 {
1977         struct nfit_mem *nfit_mem;
1978         int dimm_count = 0, rc;
1979         struct nvdimm *nvdimm;
1980
1981         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1982                 struct acpi_nfit_flush_address *flush;
1983                 unsigned long flags = 0, cmd_mask;
1984                 struct nfit_memdev *nfit_memdev;
1985                 u32 device_handle;
1986                 u16 mem_flags;
1987
1988                 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
1989                 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
1990                 if (nvdimm) {
1991                         dimm_count++;
1992                         continue;
1993                 }
1994
1995                 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
1996                         set_bit(NDD_ALIASING, &flags);
1997
1998                 /* collate flags across all memdevs for this dimm */
1999                 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2000                         struct acpi_nfit_memory_map *dimm_memdev;
2001
2002                         dimm_memdev = __to_nfit_memdev(nfit_mem);
2003                         if (dimm_memdev->device_handle
2004                                         != nfit_memdev->memdev->device_handle)
2005                                 continue;
2006                         dimm_memdev->flags |= nfit_memdev->memdev->flags;
2007                 }
2008
2009                 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
2010                 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
2011                         set_bit(NDD_UNARMED, &flags);
2012
2013                 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
2014                 if (rc)
2015                         continue;
2016
2017                 /*
2018                  * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
2019                  * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
2020                  * userspace interface.
2021                  */
2022                 cmd_mask = 1UL << ND_CMD_CALL;
2023                 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
2024                         /*
2025                          * These commands have a 1:1 correspondence
2026                          * between DSM payload and libnvdimm ioctl
2027                          * payload format.
2028                          */
2029                         cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK;
2030                 }
2031
2032                 if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
2033                         set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
2034                         set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
2035                 }
2036                 if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags))
2037                         set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
2038
2039                 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush
2040                         : NULL;
2041                 nvdimm = __nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
2042                                 acpi_nfit_dimm_attribute_groups,
2043                                 flags, cmd_mask, flush ? flush->hint_count : 0,
2044                                 nfit_mem->flush_wpq, &nfit_mem->id[0],
2045                                 acpi_nfit_get_security_ops(nfit_mem->family));
2046                 if (!nvdimm)
2047                         return -ENOMEM;
2048
2049                 nfit_mem->nvdimm = nvdimm;
2050                 dimm_count++;
2051
2052                 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
2053                         continue;
2054
2055                 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n",
2056                                 nvdimm_name(nvdimm),
2057                   mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
2058                   mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
2059                   mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
2060                   mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "",
2061                   mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : "");
2062
2063         }
2064
2065         rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
2066         if (rc)
2067                 return rc;
2068
2069         /*
2070          * Now that dimms are successfully registered, and async registration
2071          * is flushed, attempt to enable event notification.
2072          */
2073         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2074                 struct kernfs_node *nfit_kernfs;
2075
2076                 nvdimm = nfit_mem->nvdimm;
2077                 if (!nvdimm)
2078                         continue;
2079
2080                 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
2081                 if (nfit_kernfs)
2082                         nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
2083                                         "flags");
2084                 sysfs_put(nfit_kernfs);
2085                 if (!nfit_mem->flags_attr)
2086                         dev_warn(acpi_desc->dev, "%s: notifications disabled\n",
2087                                         nvdimm_name(nvdimm));
2088         }
2089
2090         return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify,
2091                         acpi_desc);
2092 }
2093
2094 /*
2095  * These constants are private because there are no kernel consumers of
2096  * these commands.
2097  */
2098 enum nfit_aux_cmds {
2099         NFIT_CMD_TRANSLATE_SPA = 5,
2100         NFIT_CMD_ARS_INJECT_SET = 7,
2101         NFIT_CMD_ARS_INJECT_CLEAR = 8,
2102         NFIT_CMD_ARS_INJECT_GET = 9,
2103 };
2104
2105 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
2106 {
2107         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2108         const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS);
2109         struct acpi_device *adev;
2110         unsigned long dsm_mask;
2111         int i;
2112
2113         nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
2114         nd_desc->bus_dsm_mask = acpi_desc->bus_nfit_cmd_force_en;
2115         adev = to_acpi_dev(acpi_desc);
2116         if (!adev)
2117                 return;
2118
2119         for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
2120                 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2121                         set_bit(i, &nd_desc->cmd_mask);
2122         set_bit(ND_CMD_CALL, &nd_desc->cmd_mask);
2123
2124         dsm_mask =
2125                 (1 << ND_CMD_ARS_CAP) |
2126                 (1 << ND_CMD_ARS_START) |
2127                 (1 << ND_CMD_ARS_STATUS) |
2128                 (1 << ND_CMD_CLEAR_ERROR) |
2129                 (1 << NFIT_CMD_TRANSLATE_SPA) |
2130                 (1 << NFIT_CMD_ARS_INJECT_SET) |
2131                 (1 << NFIT_CMD_ARS_INJECT_CLEAR) |
2132                 (1 << NFIT_CMD_ARS_INJECT_GET);
2133         for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
2134                 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2135                         set_bit(i, &nd_desc->bus_dsm_mask);
2136 }
2137
2138 static ssize_t range_index_show(struct device *dev,
2139                 struct device_attribute *attr, char *buf)
2140 {
2141         struct nd_region *nd_region = to_nd_region(dev);
2142         struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
2143
2144         return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
2145 }
2146 static DEVICE_ATTR_RO(range_index);
2147
2148 static struct attribute *acpi_nfit_region_attributes[] = {
2149         &dev_attr_range_index.attr,
2150         NULL,
2151 };
2152
2153 static const struct attribute_group acpi_nfit_region_attribute_group = {
2154         .name = "nfit",
2155         .attrs = acpi_nfit_region_attributes,
2156 };
2157
2158 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
2159         &nd_region_attribute_group,
2160         &nd_mapping_attribute_group,
2161         &nd_device_attribute_group,
2162         &nd_numa_attribute_group,
2163         &acpi_nfit_region_attribute_group,
2164         NULL,
2165 };
2166
2167 /* enough info to uniquely specify an interleave set */
2168 struct nfit_set_info {
2169         struct nfit_set_info_map {
2170                 u64 region_offset;
2171                 u32 serial_number;
2172                 u32 pad;
2173         } mapping[0];
2174 };
2175
2176 struct nfit_set_info2 {
2177         struct nfit_set_info_map2 {
2178                 u64 region_offset;
2179                 u32 serial_number;
2180                 u16 vendor_id;
2181                 u16 manufacturing_date;
2182                 u8  manufacturing_location;
2183                 u8  reserved[31];
2184         } mapping[0];
2185 };
2186
2187 static size_t sizeof_nfit_set_info(int num_mappings)
2188 {
2189         return sizeof(struct nfit_set_info)
2190                 + num_mappings * sizeof(struct nfit_set_info_map);
2191 }
2192
2193 static size_t sizeof_nfit_set_info2(int num_mappings)
2194 {
2195         return sizeof(struct nfit_set_info2)
2196                 + num_mappings * sizeof(struct nfit_set_info_map2);
2197 }
2198
2199 static int cmp_map_compat(const void *m0, const void *m1)
2200 {
2201         const struct nfit_set_info_map *map0 = m0;
2202         const struct nfit_set_info_map *map1 = m1;
2203
2204         return memcmp(&map0->region_offset, &map1->region_offset,
2205                         sizeof(u64));
2206 }
2207
2208 static int cmp_map(const void *m0, const void *m1)
2209 {
2210         const struct nfit_set_info_map *map0 = m0;
2211         const struct nfit_set_info_map *map1 = m1;
2212
2213         if (map0->region_offset < map1->region_offset)
2214                 return -1;
2215         else if (map0->region_offset > map1->region_offset)
2216                 return 1;
2217         return 0;
2218 }
2219
2220 static int cmp_map2(const void *m0, const void *m1)
2221 {
2222         const struct nfit_set_info_map2 *map0 = m0;
2223         const struct nfit_set_info_map2 *map1 = m1;
2224
2225         if (map0->region_offset < map1->region_offset)
2226                 return -1;
2227         else if (map0->region_offset > map1->region_offset)
2228                 return 1;
2229         return 0;
2230 }
2231
2232 /* Retrieve the nth entry referencing this spa */
2233 static struct acpi_nfit_memory_map *memdev_from_spa(
2234                 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
2235 {
2236         struct nfit_memdev *nfit_memdev;
2237
2238         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
2239                 if (nfit_memdev->memdev->range_index == range_index)
2240                         if (n-- == 0)
2241                                 return nfit_memdev->memdev;
2242         return NULL;
2243 }
2244
2245 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
2246                 struct nd_region_desc *ndr_desc,
2247                 struct acpi_nfit_system_address *spa)
2248 {
2249         struct device *dev = acpi_desc->dev;
2250         struct nd_interleave_set *nd_set;
2251         u16 nr = ndr_desc->num_mappings;
2252         struct nfit_set_info2 *info2;
2253         struct nfit_set_info *info;
2254         int i;
2255
2256         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
2257         if (!nd_set)
2258                 return -ENOMEM;
2259         guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid);
2260
2261         info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
2262         if (!info)
2263                 return -ENOMEM;
2264
2265         info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL);
2266         if (!info2)
2267                 return -ENOMEM;
2268
2269         for (i = 0; i < nr; i++) {
2270                 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
2271                 struct nfit_set_info_map *map = &info->mapping[i];
2272                 struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2273                 struct nvdimm *nvdimm = mapping->nvdimm;
2274                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2275                 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
2276                                 spa->range_index, i);
2277                 struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2278
2279                 if (!memdev || !nfit_mem->dcr) {
2280                         dev_err(dev, "%s: failed to find DCR\n", __func__);
2281                         return -ENODEV;
2282                 }
2283
2284                 map->region_offset = memdev->region_offset;
2285                 map->serial_number = dcr->serial_number;
2286
2287                 map2->region_offset = memdev->region_offset;
2288                 map2->serial_number = dcr->serial_number;
2289                 map2->vendor_id = dcr->vendor_id;
2290                 map2->manufacturing_date = dcr->manufacturing_date;
2291                 map2->manufacturing_location = dcr->manufacturing_location;
2292         }
2293
2294         /* v1.1 namespaces */
2295         sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2296                         cmp_map, NULL);
2297         nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2298
2299         /* v1.2 namespaces */
2300         sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2),
2301                         cmp_map2, NULL);
2302         nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0);
2303
2304         /* support v1.1 namespaces created with the wrong sort order */
2305         sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2306                         cmp_map_compat, NULL);
2307         nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2308
2309         /* record the result of the sort for the mapping position */
2310         for (i = 0; i < nr; i++) {
2311                 struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2312                 int j;
2313
2314                 for (j = 0; j < nr; j++) {
2315                         struct nd_mapping_desc *mapping = &ndr_desc->mapping[j];
2316                         struct nvdimm *nvdimm = mapping->nvdimm;
2317                         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2318                         struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2319
2320                         if (map2->serial_number == dcr->serial_number &&
2321                             map2->vendor_id == dcr->vendor_id &&
2322                             map2->manufacturing_date == dcr->manufacturing_date &&
2323                             map2->manufacturing_location
2324                                     == dcr->manufacturing_location) {
2325                                 mapping->position = i;
2326                                 break;
2327                         }
2328                 }
2329         }
2330
2331         ndr_desc->nd_set = nd_set;
2332         devm_kfree(dev, info);
2333         devm_kfree(dev, info2);
2334
2335         return 0;
2336 }
2337
2338 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
2339 {
2340         struct acpi_nfit_interleave *idt = mmio->idt;
2341         u32 sub_line_offset, line_index, line_offset;
2342         u64 line_no, table_skip_count, table_offset;
2343
2344         line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
2345         table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
2346         line_offset = idt->line_offset[line_index]
2347                 * mmio->line_size;
2348         table_offset = table_skip_count * mmio->table_size;
2349
2350         return mmio->base_offset + line_offset + table_offset + sub_line_offset;
2351 }
2352
2353 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
2354 {
2355         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2356         u64 offset = nfit_blk->stat_offset + mmio->size * bw;
2357         const u32 STATUS_MASK = 0x80000037;
2358
2359         if (mmio->num_lines)
2360                 offset = to_interleave_offset(offset, mmio);
2361
2362         return readl(mmio->addr.base + offset) & STATUS_MASK;
2363 }
2364
2365 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
2366                 resource_size_t dpa, unsigned int len, unsigned int write)
2367 {
2368         u64 cmd, offset;
2369         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2370
2371         enum {
2372                 BCW_OFFSET_MASK = (1ULL << 48)-1,
2373                 BCW_LEN_SHIFT = 48,
2374                 BCW_LEN_MASK = (1ULL << 8) - 1,
2375                 BCW_CMD_SHIFT = 56,
2376         };
2377
2378         cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
2379         len = len >> L1_CACHE_SHIFT;
2380         cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
2381         cmd |= ((u64) write) << BCW_CMD_SHIFT;
2382
2383         offset = nfit_blk->cmd_offset + mmio->size * bw;
2384         if (mmio->num_lines)
2385                 offset = to_interleave_offset(offset, mmio);
2386
2387         writeq(cmd, mmio->addr.base + offset);
2388         nvdimm_flush(nfit_blk->nd_region);
2389
2390         if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
2391                 readq(mmio->addr.base + offset);
2392 }
2393
2394 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
2395                 resource_size_t dpa, void *iobuf, size_t len, int rw,
2396                 unsigned int lane)
2397 {
2398         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2399         unsigned int copied = 0;
2400         u64 base_offset;
2401         int rc;
2402
2403         base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
2404                 + lane * mmio->size;
2405         write_blk_ctl(nfit_blk, lane, dpa, len, rw);
2406         while (len) {
2407                 unsigned int c;
2408                 u64 offset;
2409
2410                 if (mmio->num_lines) {
2411                         u32 line_offset;
2412
2413                         offset = to_interleave_offset(base_offset + copied,
2414                                         mmio);
2415                         div_u64_rem(offset, mmio->line_size, &line_offset);
2416                         c = min_t(size_t, len, mmio->line_size - line_offset);
2417                 } else {
2418                         offset = base_offset + nfit_blk->bdw_offset;
2419                         c = len;
2420                 }
2421
2422                 if (rw)
2423                         memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c);
2424                 else {
2425                         if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
2426                                 arch_invalidate_pmem((void __force *)
2427                                         mmio->addr.aperture + offset, c);
2428
2429                         memcpy(iobuf + copied, mmio->addr.aperture + offset, c);
2430                 }
2431
2432                 copied += c;
2433                 len -= c;
2434         }
2435
2436         if (rw)
2437                 nvdimm_flush(nfit_blk->nd_region);
2438
2439         rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
2440         return rc;
2441 }
2442
2443 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
2444                 resource_size_t dpa, void *iobuf, u64 len, int rw)
2445 {
2446         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
2447         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2448         struct nd_region *nd_region = nfit_blk->nd_region;
2449         unsigned int lane, copied = 0;
2450         int rc = 0;
2451
2452         lane = nd_region_acquire_lane(nd_region);
2453         while (len) {
2454                 u64 c = min(len, mmio->size);
2455
2456                 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
2457                                 iobuf + copied, c, rw, lane);
2458                 if (rc)
2459                         break;
2460
2461                 copied += c;
2462                 len -= c;
2463         }
2464         nd_region_release_lane(nd_region, lane);
2465
2466         return rc;
2467 }
2468
2469 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
2470                 struct acpi_nfit_interleave *idt, u16 interleave_ways)
2471 {
2472         if (idt) {
2473                 mmio->num_lines = idt->line_count;
2474                 mmio->line_size = idt->line_size;
2475                 if (interleave_ways == 0)
2476                         return -ENXIO;
2477                 mmio->table_size = mmio->num_lines * interleave_ways
2478                         * mmio->line_size;
2479         }
2480
2481         return 0;
2482 }
2483
2484 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
2485                 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
2486 {
2487         struct nd_cmd_dimm_flags flags;
2488         int rc;
2489
2490         memset(&flags, 0, sizeof(flags));
2491         rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
2492                         sizeof(flags), NULL);
2493
2494         if (rc >= 0 && flags.status == 0)
2495                 nfit_blk->dimm_flags = flags.flags;
2496         else if (rc == -ENOTTY) {
2497                 /* fall back to a conservative default */
2498                 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
2499                 rc = 0;
2500         } else
2501                 rc = -ENXIO;
2502
2503         return rc;
2504 }
2505
2506 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
2507                 struct device *dev)
2508 {
2509         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
2510         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
2511         struct nfit_blk_mmio *mmio;
2512         struct nfit_blk *nfit_blk;
2513         struct nfit_mem *nfit_mem;
2514         struct nvdimm *nvdimm;
2515         int rc;
2516
2517         nvdimm = nd_blk_region_to_dimm(ndbr);
2518         nfit_mem = nvdimm_provider_data(nvdimm);
2519         if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
2520                 dev_dbg(dev, "missing%s%s%s\n",
2521                                 nfit_mem ? "" : " nfit_mem",
2522                                 (nfit_mem && nfit_mem->dcr) ? "" : " dcr",
2523                                 (nfit_mem && nfit_mem->bdw) ? "" : " bdw");
2524                 return -ENXIO;
2525         }
2526
2527         nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
2528         if (!nfit_blk)
2529                 return -ENOMEM;
2530         nd_blk_region_set_provider_data(ndbr, nfit_blk);
2531         nfit_blk->nd_region = to_nd_region(dev);
2532
2533         /* map block aperture memory */
2534         nfit_blk->bdw_offset = nfit_mem->bdw->offset;
2535         mmio = &nfit_blk->mmio[BDW];
2536         mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address,
2537                         nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr));
2538         if (!mmio->addr.base) {
2539                 dev_dbg(dev, "%s failed to map bdw\n",
2540                                 nvdimm_name(nvdimm));
2541                 return -ENOMEM;
2542         }
2543         mmio->size = nfit_mem->bdw->size;
2544         mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
2545         mmio->idt = nfit_mem->idt_bdw;
2546         mmio->spa = nfit_mem->spa_bdw;
2547         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
2548                         nfit_mem->memdev_bdw->interleave_ways);
2549         if (rc) {
2550                 dev_dbg(dev, "%s failed to init bdw interleave\n",
2551                                 nvdimm_name(nvdimm));
2552                 return rc;
2553         }
2554
2555         /* map block control memory */
2556         nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
2557         nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
2558         mmio = &nfit_blk->mmio[DCR];
2559         mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address,
2560                         nfit_mem->spa_dcr->length);
2561         if (!mmio->addr.base) {
2562                 dev_dbg(dev, "%s failed to map dcr\n",
2563                                 nvdimm_name(nvdimm));
2564                 return -ENOMEM;
2565         }
2566         mmio->size = nfit_mem->dcr->window_size;
2567         mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
2568         mmio->idt = nfit_mem->idt_dcr;
2569         mmio->spa = nfit_mem->spa_dcr;
2570         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
2571                         nfit_mem->memdev_dcr->interleave_ways);
2572         if (rc) {
2573                 dev_dbg(dev, "%s failed to init dcr interleave\n",
2574                                 nvdimm_name(nvdimm));
2575                 return rc;
2576         }
2577
2578         rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
2579         if (rc < 0) {
2580                 dev_dbg(dev, "%s failed get DIMM flags\n",
2581                                 nvdimm_name(nvdimm));
2582                 return rc;
2583         }
2584
2585         if (nvdimm_has_flush(nfit_blk->nd_region) < 0)
2586                 dev_warn(dev, "unable to guarantee persistence of writes\n");
2587
2588         if (mmio->line_size == 0)
2589                 return 0;
2590
2591         if ((u32) nfit_blk->cmd_offset % mmio->line_size
2592                         + 8 > mmio->line_size) {
2593                 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
2594                 return -ENXIO;
2595         } else if ((u32) nfit_blk->stat_offset % mmio->line_size
2596                         + 8 > mmio->line_size) {
2597                 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
2598                 return -ENXIO;
2599         }
2600
2601         return 0;
2602 }
2603
2604 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
2605                 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
2606 {
2607         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2608         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2609         int cmd_rc, rc;
2610
2611         cmd->address = spa->address;
2612         cmd->length = spa->length;
2613         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
2614                         sizeof(*cmd), &cmd_rc);
2615         if (rc < 0)
2616                 return rc;
2617         return cmd_rc;
2618 }
2619
2620 static int ars_start(struct acpi_nfit_desc *acpi_desc,
2621                 struct nfit_spa *nfit_spa, enum nfit_ars_state req_type)
2622 {
2623         int rc;
2624         int cmd_rc;
2625         struct nd_cmd_ars_start ars_start;
2626         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2627         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2628
2629         memset(&ars_start, 0, sizeof(ars_start));
2630         ars_start.address = spa->address;
2631         ars_start.length = spa->length;
2632         if (req_type == ARS_REQ_SHORT)
2633                 ars_start.flags = ND_ARS_RETURN_PREV_DATA;
2634         if (nfit_spa_type(spa) == NFIT_SPA_PM)
2635                 ars_start.type = ND_ARS_PERSISTENT;
2636         else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
2637                 ars_start.type = ND_ARS_VOLATILE;
2638         else
2639                 return -ENOTTY;
2640
2641         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2642                         sizeof(ars_start), &cmd_rc);
2643
2644         if (rc < 0)
2645                 return rc;
2646         return cmd_rc;
2647 }
2648
2649 static int ars_continue(struct acpi_nfit_desc *acpi_desc)
2650 {
2651         int rc, cmd_rc;
2652         struct nd_cmd_ars_start ars_start;
2653         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2654         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2655
2656         memset(&ars_start, 0, sizeof(ars_start));
2657         ars_start.address = ars_status->restart_address;
2658         ars_start.length = ars_status->restart_length;
2659         ars_start.type = ars_status->type;
2660         ars_start.flags = acpi_desc->ars_start_flags;
2661         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2662                         sizeof(ars_start), &cmd_rc);
2663         if (rc < 0)
2664                 return rc;
2665         return cmd_rc;
2666 }
2667
2668 static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
2669 {
2670         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2671         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2672         int rc, cmd_rc;
2673
2674         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
2675                         acpi_desc->max_ars, &cmd_rc);
2676         if (rc < 0)
2677                 return rc;
2678         return cmd_rc;
2679 }
2680
2681 static void ars_complete(struct acpi_nfit_desc *acpi_desc,
2682                 struct nfit_spa *nfit_spa)
2683 {
2684         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2685         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2686         struct nd_region *nd_region = nfit_spa->nd_region;
2687         struct device *dev;
2688
2689         lockdep_assert_held(&acpi_desc->init_mutex);
2690         /*
2691          * Only advance the ARS state for ARS runs initiated by the
2692          * kernel, ignore ARS results from BIOS initiated runs for scrub
2693          * completion tracking.
2694          */
2695         if (acpi_desc->scrub_spa != nfit_spa)
2696                 return;
2697
2698         if ((ars_status->address >= spa->address && ars_status->address
2699                                 < spa->address + spa->length)
2700                         || (ars_status->address < spa->address)) {
2701                 /*
2702                  * Assume that if a scrub starts at an offset from the
2703                  * start of nfit_spa that we are in the continuation
2704                  * case.
2705                  *
2706                  * Otherwise, if the scrub covers the spa range, mark
2707                  * any pending request complete.
2708                  */
2709                 if (ars_status->address + ars_status->length
2710                                 >= spa->address + spa->length)
2711                                 /* complete */;
2712                 else
2713                         return;
2714         } else
2715                 return;
2716
2717         acpi_desc->scrub_spa = NULL;
2718         if (nd_region) {
2719                 dev = nd_region_dev(nd_region);
2720                 nvdimm_region_notify(nd_region, NVDIMM_REVALIDATE_POISON);
2721         } else
2722                 dev = acpi_desc->dev;
2723         dev_dbg(dev, "ARS: range %d complete\n", spa->range_index);
2724 }
2725
2726 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc)
2727 {
2728         struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus;
2729         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2730         int rc;
2731         u32 i;
2732
2733         /*
2734          * First record starts at 44 byte offset from the start of the
2735          * payload.
2736          */
2737         if (ars_status->out_length < 44)
2738                 return 0;
2739         for (i = 0; i < ars_status->num_records; i++) {
2740                 /* only process full records */
2741                 if (ars_status->out_length
2742                                 < 44 + sizeof(struct nd_ars_record) * (i + 1))
2743                         break;
2744                 rc = nvdimm_bus_add_badrange(nvdimm_bus,
2745                                 ars_status->records[i].err_address,
2746                                 ars_status->records[i].length);
2747                 if (rc)
2748                         return rc;
2749         }
2750         if (i < ars_status->num_records)
2751                 dev_warn(acpi_desc->dev, "detected truncated ars results\n");
2752
2753         return 0;
2754 }
2755
2756 static void acpi_nfit_remove_resource(void *data)
2757 {
2758         struct resource *res = data;
2759
2760         remove_resource(res);
2761 }
2762
2763 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
2764                 struct nd_region_desc *ndr_desc)
2765 {
2766         struct resource *res, *nd_res = ndr_desc->res;
2767         int is_pmem, ret;
2768
2769         /* No operation if the region is already registered as PMEM */
2770         is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
2771                                 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
2772         if (is_pmem == REGION_INTERSECTS)
2773                 return 0;
2774
2775         res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
2776         if (!res)
2777                 return -ENOMEM;
2778
2779         res->name = "Persistent Memory";
2780         res->start = nd_res->start;
2781         res->end = nd_res->end;
2782         res->flags = IORESOURCE_MEM;
2783         res->desc = IORES_DESC_PERSISTENT_MEMORY;
2784
2785         ret = insert_resource(&iomem_resource, res);
2786         if (ret)
2787                 return ret;
2788
2789         ret = devm_add_action_or_reset(acpi_desc->dev,
2790                                         acpi_nfit_remove_resource,
2791                                         res);
2792         if (ret)
2793                 return ret;
2794
2795         return 0;
2796 }
2797
2798 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
2799                 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc,
2800                 struct acpi_nfit_memory_map *memdev,
2801                 struct nfit_spa *nfit_spa)
2802 {
2803         struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
2804                         memdev->device_handle);
2805         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2806         struct nd_blk_region_desc *ndbr_desc;
2807         struct nfit_mem *nfit_mem;
2808         int rc;
2809
2810         if (!nvdimm) {
2811                 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
2812                                 spa->range_index, memdev->device_handle);
2813                 return -ENODEV;
2814         }
2815
2816         mapping->nvdimm = nvdimm;
2817         switch (nfit_spa_type(spa)) {
2818         case NFIT_SPA_PM:
2819         case NFIT_SPA_VOLATILE:
2820                 mapping->start = memdev->address;
2821                 mapping->size = memdev->region_size;
2822                 break;
2823         case NFIT_SPA_DCR:
2824                 nfit_mem = nvdimm_provider_data(nvdimm);
2825                 if (!nfit_mem || !nfit_mem->bdw) {
2826                         dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
2827                                         spa->range_index, nvdimm_name(nvdimm));
2828                         break;
2829                 }
2830
2831                 mapping->size = nfit_mem->bdw->capacity;
2832                 mapping->start = nfit_mem->bdw->start_address;
2833                 ndr_desc->num_lanes = nfit_mem->bdw->windows;
2834                 ndr_desc->mapping = mapping;
2835                 ndr_desc->num_mappings = 1;
2836                 ndbr_desc = to_blk_region_desc(ndr_desc);
2837                 ndbr_desc->enable = acpi_nfit_blk_region_enable;
2838                 ndbr_desc->do_io = acpi_desc->blk_do_io;
2839                 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2840                 if (rc)
2841                         return rc;
2842                 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
2843                                 ndr_desc);
2844                 if (!nfit_spa->nd_region)
2845                         return -ENOMEM;
2846                 break;
2847         }
2848
2849         return 0;
2850 }
2851
2852 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa)
2853 {
2854         return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2855                 nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2856                 nfit_spa_type(spa) == NFIT_SPA_PDISK ||
2857                 nfit_spa_type(spa) == NFIT_SPA_PCD);
2858 }
2859
2860 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa)
2861 {
2862         return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2863                 nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2864                 nfit_spa_type(spa) == NFIT_SPA_VOLATILE);
2865 }
2866
2867 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
2868                 struct nfit_spa *nfit_spa)
2869 {
2870         static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS];
2871         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2872         struct nd_blk_region_desc ndbr_desc;
2873         struct nd_region_desc *ndr_desc;
2874         struct nfit_memdev *nfit_memdev;
2875         struct nvdimm_bus *nvdimm_bus;
2876         struct resource res;
2877         int count = 0, rc;
2878
2879         if (nfit_spa->nd_region)
2880                 return 0;
2881
2882         if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) {
2883                 dev_dbg(acpi_desc->dev, "detected invalid spa index\n");
2884                 return 0;
2885         }
2886
2887         memset(&res, 0, sizeof(res));
2888         memset(&mappings, 0, sizeof(mappings));
2889         memset(&ndbr_desc, 0, sizeof(ndbr_desc));
2890         res.start = spa->address;
2891         res.end = res.start + spa->length - 1;
2892         ndr_desc = &ndbr_desc.ndr_desc;
2893         ndr_desc->res = &res;
2894         ndr_desc->provider_data = nfit_spa;
2895         ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
2896         if (spa->flags & ACPI_NFIT_PROXIMITY_VALID)
2897                 ndr_desc->numa_node = acpi_map_pxm_to_online_node(
2898                                                 spa->proximity_domain);
2899         else
2900                 ndr_desc->numa_node = NUMA_NO_NODE;
2901
2902         /*
2903          * Persistence domain bits are hierarchical, if
2904          * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then
2905          * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied.
2906          */
2907         if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH)
2908                 set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags);
2909         else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH)
2910                 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags);
2911
2912         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2913                 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
2914                 struct nd_mapping_desc *mapping;
2915
2916                 if (memdev->range_index != spa->range_index)
2917                         continue;
2918                 if (count >= ND_MAX_MAPPINGS) {
2919                         dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
2920                                         spa->range_index, ND_MAX_MAPPINGS);
2921                         return -ENXIO;
2922                 }
2923                 mapping = &mappings[count++];
2924                 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc,
2925                                 memdev, nfit_spa);
2926                 if (rc)
2927                         goto out;
2928         }
2929
2930         ndr_desc->mapping = mappings;
2931         ndr_desc->num_mappings = count;
2932         rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2933         if (rc)
2934                 goto out;
2935
2936         nvdimm_bus = acpi_desc->nvdimm_bus;
2937         if (nfit_spa_type(spa) == NFIT_SPA_PM) {
2938                 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
2939                 if (rc) {
2940                         dev_warn(acpi_desc->dev,
2941                                 "failed to insert pmem resource to iomem: %d\n",
2942                                 rc);
2943                         goto out;
2944                 }
2945
2946                 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
2947                                 ndr_desc);
2948                 if (!nfit_spa->nd_region)
2949                         rc = -ENOMEM;
2950         } else if (nfit_spa_is_volatile(spa)) {
2951                 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
2952                                 ndr_desc);
2953                 if (!nfit_spa->nd_region)
2954                         rc = -ENOMEM;
2955         } else if (nfit_spa_is_virtual(spa)) {
2956                 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
2957                                 ndr_desc);
2958                 if (!nfit_spa->nd_region)
2959                         rc = -ENOMEM;
2960         }
2961
2962  out:
2963         if (rc)
2964                 dev_err(acpi_desc->dev, "failed to register spa range %d\n",
2965                                 nfit_spa->spa->range_index);
2966         return rc;
2967 }
2968
2969 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc)
2970 {
2971         struct device *dev = acpi_desc->dev;
2972         struct nd_cmd_ars_status *ars_status;
2973
2974         if (acpi_desc->ars_status) {
2975                 memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
2976                 return 0;
2977         }
2978
2979         ars_status = devm_kzalloc(dev, acpi_desc->max_ars, GFP_KERNEL);
2980         if (!ars_status)
2981                 return -ENOMEM;
2982         acpi_desc->ars_status = ars_status;
2983         return 0;
2984 }
2985
2986 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc)
2987 {
2988         int rc;
2989
2990         if (ars_status_alloc(acpi_desc))
2991                 return -ENOMEM;
2992
2993         rc = ars_get_status(acpi_desc);
2994
2995         if (rc < 0 && rc != -ENOSPC)
2996                 return rc;
2997
2998         if (ars_status_process_records(acpi_desc))
2999                 dev_err(acpi_desc->dev, "Failed to process ARS records\n");
3000
3001         return rc;
3002 }
3003
3004 static int ars_register(struct acpi_nfit_desc *acpi_desc,
3005                 struct nfit_spa *nfit_spa)
3006 {
3007         int rc;
3008
3009         if (no_init_ars || test_bit(ARS_FAILED, &nfit_spa->ars_state))
3010                 return acpi_nfit_register_region(acpi_desc, nfit_spa);
3011
3012         set_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3013         set_bit(ARS_REQ_LONG, &nfit_spa->ars_state);
3014
3015         switch (acpi_nfit_query_poison(acpi_desc)) {
3016         case 0:
3017         case -EAGAIN:
3018                 rc = ars_start(acpi_desc, nfit_spa, ARS_REQ_SHORT);
3019                 /* shouldn't happen, try again later */
3020                 if (rc == -EBUSY)
3021                         break;
3022                 if (rc) {
3023                         set_bit(ARS_FAILED, &nfit_spa->ars_state);
3024                         break;
3025                 }
3026                 clear_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3027                 rc = acpi_nfit_query_poison(acpi_desc);
3028                 if (rc)
3029                         break;
3030                 acpi_desc->scrub_spa = nfit_spa;
3031                 ars_complete(acpi_desc, nfit_spa);
3032                 /*
3033                  * If ars_complete() says we didn't complete the
3034                  * short scrub, we'll try again with a long
3035                  * request.
3036                  */
3037                 acpi_desc->scrub_spa = NULL;
3038                 break;
3039         case -EBUSY:
3040         case -ENOMEM:
3041         case -ENOSPC:
3042                 /*
3043                  * BIOS was using ARS, wait for it to complete (or
3044                  * resources to become available) and then perform our
3045                  * own scrubs.
3046                  */
3047                 break;
3048         default:
3049                 set_bit(ARS_FAILED, &nfit_spa->ars_state);
3050                 break;
3051         }
3052
3053         return acpi_nfit_register_region(acpi_desc, nfit_spa);
3054 }
3055
3056 static void ars_complete_all(struct acpi_nfit_desc *acpi_desc)
3057 {
3058         struct nfit_spa *nfit_spa;
3059
3060         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3061                 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3062                         continue;
3063                 ars_complete(acpi_desc, nfit_spa);
3064         }
3065 }
3066
3067 static unsigned int __acpi_nfit_scrub(struct acpi_nfit_desc *acpi_desc,
3068                 int query_rc)
3069 {
3070         unsigned int tmo = acpi_desc->scrub_tmo;
3071         struct device *dev = acpi_desc->dev;
3072         struct nfit_spa *nfit_spa;
3073
3074         lockdep_assert_held(&acpi_desc->init_mutex);
3075
3076         if (acpi_desc->cancel)
3077                 return 0;
3078
3079         if (query_rc == -EBUSY) {
3080                 dev_dbg(dev, "ARS: ARS busy\n");
3081                 return min(30U * 60U, tmo * 2);
3082         }
3083         if (query_rc == -ENOSPC) {
3084                 dev_dbg(dev, "ARS: ARS continue\n");
3085                 ars_continue(acpi_desc);
3086                 return 1;
3087         }
3088         if (query_rc && query_rc != -EAGAIN) {
3089                 unsigned long long addr, end;
3090
3091                 addr = acpi_desc->ars_status->address;
3092                 end = addr + acpi_desc->ars_status->length;
3093                 dev_dbg(dev, "ARS: %llx-%llx failed (%d)\n", addr, end,
3094                                 query_rc);
3095         }
3096
3097         ars_complete_all(acpi_desc);
3098         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3099                 enum nfit_ars_state req_type;
3100                 int rc;
3101
3102                 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3103                         continue;
3104
3105                 /* prefer short ARS requests first */
3106                 if (test_bit(ARS_REQ_SHORT, &nfit_spa->ars_state))
3107                         req_type = ARS_REQ_SHORT;
3108                 else if (test_bit(ARS_REQ_LONG, &nfit_spa->ars_state))
3109                         req_type = ARS_REQ_LONG;
3110                 else
3111                         continue;
3112                 rc = ars_start(acpi_desc, nfit_spa, req_type);
3113
3114                 dev = nd_region_dev(nfit_spa->nd_region);
3115                 dev_dbg(dev, "ARS: range %d ARS start %s (%d)\n",
3116                                 nfit_spa->spa->range_index,
3117                                 req_type == ARS_REQ_SHORT ? "short" : "long",
3118                                 rc);
3119                 /*
3120                  * Hmm, we raced someone else starting ARS? Try again in
3121                  * a bit.
3122                  */
3123                 if (rc == -EBUSY)
3124                         return 1;
3125                 if (rc == 0) {
3126                         dev_WARN_ONCE(dev, acpi_desc->scrub_spa,
3127                                         "scrub start while range %d active\n",
3128                                         acpi_desc->scrub_spa->spa->range_index);
3129                         clear_bit(req_type, &nfit_spa->ars_state);
3130                         acpi_desc->scrub_spa = nfit_spa;
3131                         /*
3132                          * Consider this spa last for future scrub
3133                          * requests
3134                          */
3135                         list_move_tail(&nfit_spa->list, &acpi_desc->spas);
3136                         return 1;
3137                 }
3138
3139                 dev_err(dev, "ARS: range %d ARS failed (%d)\n",
3140                                 nfit_spa->spa->range_index, rc);
3141                 set_bit(ARS_FAILED, &nfit_spa->ars_state);
3142         }
3143         return 0;
3144 }
3145
3146 static void __sched_ars(struct acpi_nfit_desc *acpi_desc, unsigned int tmo)
3147 {
3148         lockdep_assert_held(&acpi_desc->init_mutex);
3149
3150         acpi_desc->scrub_busy = 1;
3151         /* note this should only be set from within the workqueue */
3152         if (tmo)
3153                 acpi_desc->scrub_tmo = tmo;
3154         queue_delayed_work(nfit_wq, &acpi_desc->dwork, tmo * HZ);
3155 }
3156
3157 static void sched_ars(struct acpi_nfit_desc *acpi_desc)
3158 {
3159         __sched_ars(acpi_desc, 0);
3160 }
3161
3162 static void notify_ars_done(struct acpi_nfit_desc *acpi_desc)
3163 {
3164         lockdep_assert_held(&acpi_desc->init_mutex);
3165
3166         acpi_desc->scrub_busy = 0;
3167         acpi_desc->scrub_count++;
3168         if (acpi_desc->scrub_count_state)
3169                 sysfs_notify_dirent(acpi_desc->scrub_count_state);
3170 }
3171
3172 static void acpi_nfit_scrub(struct work_struct *work)
3173 {
3174         struct acpi_nfit_desc *acpi_desc;
3175         unsigned int tmo;
3176         int query_rc;
3177
3178         acpi_desc = container_of(work, typeof(*acpi_desc), dwork.work);
3179         mutex_lock(&acpi_desc->init_mutex);
3180         query_rc = acpi_nfit_query_poison(acpi_desc);
3181         tmo = __acpi_nfit_scrub(acpi_desc, query_rc);
3182         if (tmo)
3183                 __sched_ars(acpi_desc, tmo);
3184         else
3185                 notify_ars_done(acpi_desc);
3186         memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3187         mutex_unlock(&acpi_desc->init_mutex);
3188 }
3189
3190 static void acpi_nfit_init_ars(struct acpi_nfit_desc *acpi_desc,
3191                 struct nfit_spa *nfit_spa)
3192 {
3193         int type = nfit_spa_type(nfit_spa->spa);
3194         struct nd_cmd_ars_cap ars_cap;
3195         int rc;
3196
3197         set_bit(ARS_FAILED, &nfit_spa->ars_state);
3198         memset(&ars_cap, 0, sizeof(ars_cap));
3199         rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
3200         if (rc < 0)
3201                 return;
3202         /* check that the supported scrub types match the spa type */
3203         if (type == NFIT_SPA_VOLATILE && ((ars_cap.status >> 16)
3204                                 & ND_ARS_VOLATILE) == 0)
3205                 return;
3206         if (type == NFIT_SPA_PM && ((ars_cap.status >> 16)
3207                                 & ND_ARS_PERSISTENT) == 0)
3208                 return;
3209
3210         nfit_spa->max_ars = ars_cap.max_ars_out;
3211         nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
3212         acpi_desc->max_ars = max(nfit_spa->max_ars, acpi_desc->max_ars);
3213         clear_bit(ARS_FAILED, &nfit_spa->ars_state);
3214 }
3215
3216 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
3217 {
3218         struct nfit_spa *nfit_spa;
3219         int rc;
3220
3221         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3222                 switch (nfit_spa_type(nfit_spa->spa)) {
3223                 case NFIT_SPA_VOLATILE:
3224                 case NFIT_SPA_PM:
3225                         acpi_nfit_init_ars(acpi_desc, nfit_spa);
3226                         break;
3227                 }
3228         }
3229
3230         list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
3231                 switch (nfit_spa_type(nfit_spa->spa)) {
3232                 case NFIT_SPA_VOLATILE:
3233                 case NFIT_SPA_PM:
3234                         /* register regions and kick off initial ARS run */
3235                         rc = ars_register(acpi_desc, nfit_spa);
3236                         if (rc)
3237                                 return rc;
3238                         break;
3239                 case NFIT_SPA_BDW:
3240                         /* nothing to register */
3241                         break;
3242                 case NFIT_SPA_DCR:
3243                 case NFIT_SPA_VDISK:
3244                 case NFIT_SPA_VCD:
3245                 case NFIT_SPA_PDISK:
3246                 case NFIT_SPA_PCD:
3247                         /* register known regions that don't support ARS */
3248                         rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
3249                         if (rc)
3250                                 return rc;
3251                         break;
3252                 default:
3253                         /* don't register unknown regions */
3254                         break;
3255                 }
3256
3257         sched_ars(acpi_desc);
3258         return 0;
3259 }
3260
3261 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
3262                 struct nfit_table_prev *prev)
3263 {
3264         struct device *dev = acpi_desc->dev;
3265
3266         if (!list_empty(&prev->spas) ||
3267                         !list_empty(&prev->memdevs) ||
3268                         !list_empty(&prev->dcrs) ||
3269                         !list_empty(&prev->bdws) ||
3270                         !list_empty(&prev->idts) ||
3271                         !list_empty(&prev->flushes)) {
3272                 dev_err(dev, "new nfit deletes entries (unsupported)\n");
3273                 return -ENXIO;
3274         }
3275         return 0;
3276 }
3277
3278 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc)
3279 {
3280         struct device *dev = acpi_desc->dev;
3281         struct kernfs_node *nfit;
3282         struct device *bus_dev;
3283
3284         if (!ars_supported(acpi_desc->nvdimm_bus))
3285                 return 0;
3286
3287         bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3288         nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit");
3289         if (!nfit) {
3290                 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n");
3291                 return -ENODEV;
3292         }
3293         acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub");
3294         sysfs_put(nfit);
3295         if (!acpi_desc->scrub_count_state) {
3296                 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n");
3297                 return -ENODEV;
3298         }
3299
3300         return 0;
3301 }
3302
3303 static void acpi_nfit_unregister(void *data)
3304 {
3305         struct acpi_nfit_desc *acpi_desc = data;
3306
3307         nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
3308 }
3309
3310 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
3311 {
3312         struct device *dev = acpi_desc->dev;
3313         struct nfit_table_prev prev;
3314         const void *end;
3315         int rc;
3316
3317         if (!acpi_desc->nvdimm_bus) {
3318                 acpi_nfit_init_dsms(acpi_desc);
3319
3320                 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev,
3321                                 &acpi_desc->nd_desc);
3322                 if (!acpi_desc->nvdimm_bus)
3323                         return -ENOMEM;
3324
3325                 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister,
3326                                 acpi_desc);
3327                 if (rc)
3328                         return rc;
3329
3330                 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc);
3331                 if (rc)
3332                         return rc;
3333
3334                 /* register this acpi_desc for mce notifications */
3335                 mutex_lock(&acpi_desc_lock);
3336                 list_add_tail(&acpi_desc->list, &acpi_descs);
3337                 mutex_unlock(&acpi_desc_lock);
3338         }
3339
3340         mutex_lock(&acpi_desc->init_mutex);
3341
3342         INIT_LIST_HEAD(&prev.spas);
3343         INIT_LIST_HEAD(&prev.memdevs);
3344         INIT_LIST_HEAD(&prev.dcrs);
3345         INIT_LIST_HEAD(&prev.bdws);
3346         INIT_LIST_HEAD(&prev.idts);
3347         INIT_LIST_HEAD(&prev.flushes);
3348
3349         list_cut_position(&prev.spas, &acpi_desc->spas,
3350                                 acpi_desc->spas.prev);
3351         list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
3352                                 acpi_desc->memdevs.prev);
3353         list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
3354                                 acpi_desc->dcrs.prev);
3355         list_cut_position(&prev.bdws, &acpi_desc->bdws,
3356                                 acpi_desc->bdws.prev);
3357         list_cut_position(&prev.idts, &acpi_desc->idts,
3358                                 acpi_desc->idts.prev);
3359         list_cut_position(&prev.flushes, &acpi_desc->flushes,
3360                                 acpi_desc->flushes.prev);
3361
3362         end = data + sz;
3363         while (!IS_ERR_OR_NULL(data))
3364                 data = add_table(acpi_desc, &prev, data, end);
3365
3366         if (IS_ERR(data)) {
3367                 dev_dbg(dev, "nfit table parsing error: %ld\n", PTR_ERR(data));
3368                 rc = PTR_ERR(data);
3369                 goto out_unlock;
3370         }
3371
3372         rc = acpi_nfit_check_deletions(acpi_desc, &prev);
3373         if (rc)
3374                 goto out_unlock;
3375
3376         rc = nfit_mem_init(acpi_desc);
3377         if (rc)
3378                 goto out_unlock;
3379
3380         rc = acpi_nfit_register_dimms(acpi_desc);
3381         if (rc)
3382                 goto out_unlock;
3383
3384         rc = acpi_nfit_register_regions(acpi_desc);
3385
3386  out_unlock:
3387         mutex_unlock(&acpi_desc->init_mutex);
3388         return rc;
3389 }
3390 EXPORT_SYMBOL_GPL(acpi_nfit_init);
3391
3392 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
3393 {
3394         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3395         struct device *dev = acpi_desc->dev;
3396
3397         /* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
3398         device_lock(dev);
3399         device_unlock(dev);
3400
3401         /* Bounce the init_mutex to complete initial registration */
3402         mutex_lock(&acpi_desc->init_mutex);
3403         mutex_unlock(&acpi_desc->init_mutex);
3404
3405         return 0;
3406 }
3407
3408 static int __acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3409                 struct nvdimm *nvdimm, unsigned int cmd)
3410 {
3411         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3412
3413         if (nvdimm)
3414                 return 0;
3415         if (cmd != ND_CMD_ARS_START)
3416                 return 0;
3417
3418         /*
3419          * The kernel and userspace may race to initiate a scrub, but
3420          * the scrub thread is prepared to lose that initial race.  It
3421          * just needs guarantees that any ARS it initiates are not
3422          * interrupted by any intervening start requests from userspace.
3423          */
3424         if (work_busy(&acpi_desc->dwork.work))
3425                 return -EBUSY;
3426
3427         return 0;
3428 }
3429
3430 /* prevent security commands from being issued via ioctl */
3431 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3432                 struct nvdimm *nvdimm, unsigned int cmd, void *buf)
3433 {
3434         struct nd_cmd_pkg *call_pkg = buf;
3435         unsigned int func;
3436
3437         if (nvdimm && cmd == ND_CMD_CALL &&
3438                         call_pkg->nd_family == NVDIMM_FAMILY_INTEL) {
3439                 func = call_pkg->nd_command;
3440                 if ((1 << func) & NVDIMM_INTEL_SECURITY_CMDMASK)
3441                         return -EOPNOTSUPP;
3442         }
3443
3444         return __acpi_nfit_clear_to_send(nd_desc, nvdimm, cmd);
3445 }
3446
3447 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc,
3448                 enum nfit_ars_state req_type)
3449 {
3450         struct device *dev = acpi_desc->dev;
3451         int scheduled = 0, busy = 0;
3452         struct nfit_spa *nfit_spa;
3453
3454         mutex_lock(&acpi_desc->init_mutex);
3455         if (acpi_desc->cancel) {
3456                 mutex_unlock(&acpi_desc->init_mutex);
3457                 return 0;
3458         }
3459
3460         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3461                 int type = nfit_spa_type(nfit_spa->spa);
3462
3463                 if (type != NFIT_SPA_PM && type != NFIT_SPA_VOLATILE)
3464                         continue;
3465                 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3466                         continue;
3467
3468                 if (test_and_set_bit(req_type, &nfit_spa->ars_state))
3469                         busy++;
3470                 else
3471                         scheduled++;
3472         }
3473         if (scheduled) {
3474                 sched_ars(acpi_desc);
3475                 dev_dbg(dev, "ars_scan triggered\n");
3476         }
3477         mutex_unlock(&acpi_desc->init_mutex);
3478
3479         if (scheduled)
3480                 return 0;
3481         if (busy)
3482                 return -EBUSY;
3483         return -ENOTTY;
3484 }
3485
3486 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
3487 {
3488         struct nvdimm_bus_descriptor *nd_desc;
3489
3490         dev_set_drvdata(dev, acpi_desc);
3491         acpi_desc->dev = dev;
3492         acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
3493         nd_desc = &acpi_desc->nd_desc;
3494         nd_desc->provider_name = "ACPI.NFIT";
3495         nd_desc->module = THIS_MODULE;
3496         nd_desc->ndctl = acpi_nfit_ctl;
3497         nd_desc->flush_probe = acpi_nfit_flush_probe;
3498         nd_desc->clear_to_send = acpi_nfit_clear_to_send;
3499         nd_desc->attr_groups = acpi_nfit_attribute_groups;
3500
3501         INIT_LIST_HEAD(&acpi_desc->spas);
3502         INIT_LIST_HEAD(&acpi_desc->dcrs);
3503         INIT_LIST_HEAD(&acpi_desc->bdws);
3504         INIT_LIST_HEAD(&acpi_desc->idts);
3505         INIT_LIST_HEAD(&acpi_desc->flushes);
3506         INIT_LIST_HEAD(&acpi_desc->memdevs);
3507         INIT_LIST_HEAD(&acpi_desc->dimms);
3508         INIT_LIST_HEAD(&acpi_desc->list);
3509         mutex_init(&acpi_desc->init_mutex);
3510         acpi_desc->scrub_tmo = 1;
3511         INIT_DELAYED_WORK(&acpi_desc->dwork, acpi_nfit_scrub);
3512 }
3513 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
3514
3515 static void acpi_nfit_put_table(void *table)
3516 {
3517         acpi_put_table(table);
3518 }
3519
3520 void acpi_nfit_shutdown(void *data)
3521 {
3522         struct acpi_nfit_desc *acpi_desc = data;
3523         struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3524
3525         /*
3526          * Destruct under acpi_desc_lock so that nfit_handle_mce does not
3527          * race teardown
3528          */
3529         mutex_lock(&acpi_desc_lock);
3530         list_del(&acpi_desc->list);
3531         mutex_unlock(&acpi_desc_lock);
3532
3533         mutex_lock(&acpi_desc->init_mutex);
3534         acpi_desc->cancel = 1;
3535         cancel_delayed_work_sync(&acpi_desc->dwork);
3536         mutex_unlock(&acpi_desc->init_mutex);
3537
3538         /*
3539          * Bounce the nvdimm bus lock to make sure any in-flight
3540          * acpi_nfit_ars_rescan() submissions have had a chance to
3541          * either submit or see ->cancel set.
3542          */
3543         device_lock(bus_dev);
3544         device_unlock(bus_dev);
3545
3546         flush_workqueue(nfit_wq);
3547 }
3548 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown);
3549
3550 static int acpi_nfit_add(struct acpi_device *adev)
3551 {
3552         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3553         struct acpi_nfit_desc *acpi_desc;
3554         struct device *dev = &adev->dev;
3555         struct acpi_table_header *tbl;
3556         acpi_status status = AE_OK;
3557         acpi_size sz;
3558         int rc = 0;
3559
3560         status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl);
3561         if (ACPI_FAILURE(status)) {
3562                 /* The NVDIMM root device allows OS to trigger enumeration of
3563                  * NVDIMMs through NFIT at boot time and re-enumeration at
3564                  * root level via the _FIT method during runtime.
3565                  * This is ok to return 0 here, we could have an nvdimm
3566                  * hotplugged later and evaluate _FIT method which returns
3567                  * data in the format of a series of NFIT Structures.
3568                  */
3569                 dev_dbg(dev, "failed to find NFIT at startup\n");
3570                 return 0;
3571         }
3572
3573         rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl);
3574         if (rc)
3575                 return rc;
3576         sz = tbl->length;
3577
3578         acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3579         if (!acpi_desc)
3580                 return -ENOMEM;
3581         acpi_nfit_desc_init(acpi_desc, &adev->dev);
3582
3583         /* Save the acpi header for exporting the revision via sysfs */
3584         acpi_desc->acpi_header = *tbl;
3585
3586         /* Evaluate _FIT and override with that if present */
3587         status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
3588         if (ACPI_SUCCESS(status) && buf.length > 0) {
3589                 union acpi_object *obj = buf.pointer;
3590
3591                 if (obj->type == ACPI_TYPE_BUFFER)
3592                         rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3593                                         obj->buffer.length);
3594                 else
3595                         dev_dbg(dev, "invalid type %d, ignoring _FIT\n",
3596                                 (int) obj->type);
3597                 kfree(buf.pointer);
3598         } else
3599                 /* skip over the lead-in header table */
3600                 rc = acpi_nfit_init(acpi_desc, (void *) tbl
3601                                 + sizeof(struct acpi_table_nfit),
3602                                 sz - sizeof(struct acpi_table_nfit));
3603
3604         if (rc)
3605                 return rc;
3606         return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc);
3607 }
3608
3609 static int acpi_nfit_remove(struct acpi_device *adev)
3610 {
3611         /* see acpi_nfit_unregister */
3612         return 0;
3613 }
3614
3615 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle)
3616 {
3617         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3618         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3619         union acpi_object *obj;
3620         acpi_status status;
3621         int ret;
3622
3623         if (!dev->driver) {
3624                 /* dev->driver may be null if we're being removed */
3625                 dev_dbg(dev, "no driver found for dev\n");
3626                 return;
3627         }
3628
3629         if (!acpi_desc) {
3630                 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3631                 if (!acpi_desc)
3632                         return;
3633                 acpi_nfit_desc_init(acpi_desc, dev);
3634         } else {
3635                 /*
3636                  * Finish previous registration before considering new
3637                  * regions.
3638                  */
3639                 flush_workqueue(nfit_wq);
3640         }
3641
3642         /* Evaluate _FIT */
3643         status = acpi_evaluate_object(handle, "_FIT", NULL, &buf);
3644         if (ACPI_FAILURE(status)) {
3645                 dev_err(dev, "failed to evaluate _FIT\n");
3646                 return;
3647         }
3648
3649         obj = buf.pointer;
3650         if (obj->type == ACPI_TYPE_BUFFER) {
3651                 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3652                                 obj->buffer.length);
3653                 if (ret)
3654                         dev_err(dev, "failed to merge updated NFIT\n");
3655         } else
3656                 dev_err(dev, "Invalid _FIT\n");
3657         kfree(buf.pointer);
3658 }
3659
3660 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle)
3661 {
3662         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3663
3664         if (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON)
3665                 acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
3666         else
3667                 acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_SHORT);
3668 }
3669
3670 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event)
3671 {
3672         dev_dbg(dev, "event: 0x%x\n", event);
3673
3674         switch (event) {
3675         case NFIT_NOTIFY_UPDATE:
3676                 return acpi_nfit_update_notify(dev, handle);
3677         case NFIT_NOTIFY_UC_MEMORY_ERROR:
3678                 return acpi_nfit_uc_error_notify(dev, handle);
3679         default:
3680                 return;
3681         }
3682 }
3683 EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
3684
3685 static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
3686 {
3687         device_lock(&adev->dev);
3688         __acpi_nfit_notify(&adev->dev, adev->handle, event);
3689         device_unlock(&adev->dev);
3690 }
3691
3692 static const struct acpi_device_id acpi_nfit_ids[] = {
3693         { "ACPI0012", 0 },
3694         { "", 0 },
3695 };
3696 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
3697
3698 static struct acpi_driver acpi_nfit_driver = {
3699         .name = KBUILD_MODNAME,
3700         .ids = acpi_nfit_ids,
3701         .ops = {
3702                 .add = acpi_nfit_add,
3703                 .remove = acpi_nfit_remove,
3704                 .notify = acpi_nfit_notify,
3705         },
3706 };
3707
3708 static __init int nfit_init(void)
3709 {
3710         int ret;
3711
3712         BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
3713         BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
3714         BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
3715         BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
3716         BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
3717         BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
3718         BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
3719         BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16);
3720
3721         guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]);
3722         guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]);
3723         guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]);
3724         guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]);
3725         guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]);
3726         guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]);
3727         guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]);
3728         guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]);
3729         guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]);
3730         guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]);
3731         guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
3732         guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
3733         guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]);
3734
3735         nfit_wq = create_singlethread_workqueue("nfit");
3736         if (!nfit_wq)
3737                 return -ENOMEM;
3738
3739         nfit_mce_register();
3740         ret = acpi_bus_register_driver(&acpi_nfit_driver);
3741         if (ret) {
3742                 nfit_mce_unregister();
3743                 destroy_workqueue(nfit_wq);
3744         }
3745
3746         return ret;
3747
3748 }
3749
3750 static __exit void nfit_exit(void)
3751 {
3752         nfit_mce_unregister();
3753         acpi_bus_unregister_driver(&acpi_nfit_driver);
3754         destroy_workqueue(nfit_wq);
3755         WARN_ON(!list_empty(&acpi_descs));
3756 }
3757
3758 module_init(nfit_init);
3759 module_exit(nfit_exit);
3760 MODULE_LICENSE("GPL v2");
3761 MODULE_AUTHOR("Intel Corporation");