drivers/acpi/nfit/core.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/list_sort.h>
  14 #include <linux/libnvdimm.h>
  15 #include <linux/module.h>
  16 #include <linux/mutex.h>
  17 #include <linux/ndctl.h>
  18 #include <linux/sysfs.h>
  19 #include <linux/delay.h>
  20 #include <linux/list.h>
  21 #include <linux/acpi.h>
  22 #include <linux/sort.h>
  23 #include <linux/io.h>
  24 #include <linux/nd.h>
  25 #include <asm/cacheflush.h>
  26 #include <acpi/nfit.h>
  27 #include "nfit.h"
  28
  29 /*
  30  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
  31  * irrelevant.
  32  */
  33 #include <linux/io-64-nonatomic-hi-lo.h>
  34
  35 static bool force_enable_dimms;
  36 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
  37 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
  38
  39 static bool disable_vendor_specific;
  40 module_param(disable_vendor_specific, bool, S_IRUGO);
  41 MODULE_PARM_DESC(disable_vendor_specific,
  42                 "Limit commands to the publicly specified set");
  43
  44 static unsigned long override_dsm_mask;
  45 module_param(override_dsm_mask, ulong, S_IRUGO);
  46 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions");
  47
  48 static int default_dsm_family = -1;
  49 module_param(default_dsm_family, int, S_IRUGO);
  50 MODULE_PARM_DESC(default_dsm_family,
  51                 "Try this DSM type first when identifying NVDIMM family");
  52
  53 static bool no_init_ars;
  54 module_param(no_init_ars, bool, 0644);
  55 MODULE_PARM_DESC(no_init_ars, "Skip ARS run at nfit init time");
  56
  57 LIST_HEAD(acpi_descs);
  58 DEFINE_MUTEX(acpi_desc_lock);
  59
  60 static struct workqueue_struct *nfit_wq;
  61
  62 struct nfit_table_prev {
  63         struct list_head spas;
  64         struct list_head memdevs;
  65         struct list_head dcrs;
  66         struct list_head bdws;
  67         struct list_head idts;
  68         struct list_head flushes;
  69 };
  70
  71 static guid_t nfit_uuid[NFIT_UUID_MAX];
  72
  73 const guid_t *to_nfit_uuid(enum nfit_uuids id)
  74 {
  75         return &nfit_uuid[id];
  76 }
  77 EXPORT_SYMBOL(to_nfit_uuid);
  78
  79 static struct acpi_nfit_desc *to_acpi_nfit_desc(
  80                 struct nvdimm_bus_descriptor *nd_desc)
  81 {
  82         return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
  83 }
  84
  85 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
  86 {
  87         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
  88
  89         /*
  90          * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
  91          * acpi_device.
  92          */
  93         if (!nd_desc->provider_name
  94                         || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
  95                 return NULL;
  96
  97         return to_acpi_device(acpi_desc->dev);
  98 }
  99
 100 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status)
 101 {
 102         struct nd_cmd_clear_error *clear_err;
 103         struct nd_cmd_ars_status *ars_status;
 104         u16 flags;
 105
 106         switch (cmd) {
 107         case ND_CMD_ARS_CAP:
 108                 if ((status & 0xffff) == NFIT_ARS_CAP_NONE)
 109                         return -ENOTTY;
 110
 111                 /* Command failed */
 112                 if (status & 0xffff)
 113                         return -EIO;
 114
 115                 /* No supported scan types for this range */
 116                 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
 117                 if ((status >> 16 & flags) == 0)
 118                         return -ENOTTY;
 119                 return 0;
 120         case ND_CMD_ARS_START:
 121                 /* ARS is in progress */
 122                 if ((status & 0xffff) == NFIT_ARS_START_BUSY)
 123                         return -EBUSY;
 124
 125                 /* Command failed */
 126                 if (status & 0xffff)
 127                         return -EIO;
 128                 return 0;
 129         case ND_CMD_ARS_STATUS:
 130                 ars_status = buf;
 131                 /* Command failed */
 132                 if (status & 0xffff)
 133                         return -EIO;
 134                 /* Check extended status (Upper two bytes) */
 135                 if (status == NFIT_ARS_STATUS_DONE)
 136                         return 0;
 137
 138                 /* ARS is in progress */
 139                 if (status == NFIT_ARS_STATUS_BUSY)
 140                         return -EBUSY;
 141
 142                 /* No ARS performed for the current boot */
 143                 if (status == NFIT_ARS_STATUS_NONE)
 144                         return -EAGAIN;
 145
 146                 /*
 147                  * ARS interrupted, either we overflowed or some other
 148                  * agent wants the scan to stop.  If we didn't overflow
 149                  * then just continue with the returned results.
 150                  */
 151                 if (status == NFIT_ARS_STATUS_INTR) {
 152                         if (ars_status->out_length >= 40 && (ars_status->flags
 153                                                 & NFIT_ARS_F_OVERFLOW))
 154                                 return -ENOSPC;
 155                         return 0;
 156                 }
 157
 158                 /* Unknown status */
 159                 if (status >> 16)
 160                         return -EIO;
 161                 return 0;
 162         case ND_CMD_CLEAR_ERROR:
 163                 clear_err = buf;
 164                 if (status & 0xffff)
 165                         return -EIO;
 166                 if (!clear_err->cleared)
 167                         return -EIO;
 168                 if (clear_err->length > clear_err->cleared)
 169                         return clear_err->cleared;
 170                 return 0;
 171         default:
 172                 break;
 173         }
 174
 175         /* all other non-zero status results in an error */
 176         if (status)
 177                 return -EIO;
 178         return 0;
 179 }
 180
 181 #define ACPI_LABELS_LOCKED 3
 182
 183 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
 184                 u32 status)
 185 {
 186         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 187
 188         switch (cmd) {
 189         case ND_CMD_GET_CONFIG_SIZE:
 190                 /*
 191                  * In the _LSI, _LSR, _LSW case the locked status is
 192                  * communicated via the read/write commands
 193                  */
 194                 if (nfit_mem->has_lsr)
 195                         break;
 196
 197                 if (status >> 16 & ND_CONFIG_LOCKED)
 198                         return -EACCES;
 199                 break;
 200         case ND_CMD_GET_CONFIG_DATA:
 201                 if (nfit_mem->has_lsr && status == ACPI_LABELS_LOCKED)
 202                         return -EACCES;
 203                 break;
 204         case ND_CMD_SET_CONFIG_DATA:
 205                 if (nfit_mem->has_lsw && status == ACPI_LABELS_LOCKED)
 206                         return -EACCES;
 207                 break;
 208         default:
 209                 break;
 210         }
 211
 212         /* all other non-zero status results in an error */
 213         if (status)
 214                 return -EIO;
 215         return 0;
 216 }
 217
 218 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
 219                 u32 status)
 220 {
 221         if (!nvdimm)
 222                 return xlat_bus_status(buf, cmd, status);
 223         return xlat_nvdimm_status(nvdimm, buf, cmd, status);
 224 }
 225
 226 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */
 227 static union acpi_object *pkg_to_buf(union acpi_object *pkg)
 228 {
 229         int i;
 230         void *dst;
 231         size_t size = 0;
 232         union acpi_object *buf = NULL;
 233
 234         if (pkg->type != ACPI_TYPE_PACKAGE) {
 235                 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 236                                 pkg->type);
 237                 goto err;
 238         }
 239
 240         for (i = 0; i < pkg->package.count; i++) {
 241                 union acpi_object *obj = &pkg->package.elements[i];
 242
 243                 if (obj->type == ACPI_TYPE_INTEGER)
 244                         size += 4;
 245                 else if (obj->type == ACPI_TYPE_BUFFER)
 246                         size += obj->buffer.length;
 247                 else {
 248                         WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 249                                         obj->type);
 250                         goto err;
 251                 }
 252         }
 253
 254         buf = ACPI_ALLOCATE(sizeof(*buf) + size);
 255         if (!buf)
 256                 goto err;
 257
 258         dst = buf + 1;
 259         buf->type = ACPI_TYPE_BUFFER;
 260         buf->buffer.length = size;
 261         buf->buffer.pointer = dst;
 262         for (i = 0; i < pkg->package.count; i++) {
 263                 union acpi_object *obj = &pkg->package.elements[i];
 264
 265                 if (obj->type == ACPI_TYPE_INTEGER) {
 266                         memcpy(dst, &obj->integer.value, 4);
 267                         dst += 4;
 268                 } else if (obj->type == ACPI_TYPE_BUFFER) {
 269                         memcpy(dst, obj->buffer.pointer, obj->buffer.length);
 270                         dst += obj->buffer.length;
 271                 }
 272         }
 273 err:
 274         ACPI_FREE(pkg);
 275         return buf;
 276 }
 277
 278 static union acpi_object *int_to_buf(union acpi_object *integer)
 279 {
 280         union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4);
 281         void *dst = NULL;
 282
 283         if (!buf)
 284                 goto err;
 285
 286         if (integer->type != ACPI_TYPE_INTEGER) {
 287                 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
 288                                 integer->type);
 289                 goto err;
 290         }
 291
 292         dst = buf + 1;
 293         buf->type = ACPI_TYPE_BUFFER;
 294         buf->buffer.length = 4;
 295         buf->buffer.pointer = dst;
 296         memcpy(dst, &integer->integer.value, 4);
 297 err:
 298         ACPI_FREE(integer);
 299         return buf;
 300 }
 301
 302 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset,
 303                 u32 len, void *data)
 304 {
 305         acpi_status rc;
 306         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 307         struct acpi_object_list input = {
 308                 .count = 3,
 309                 .pointer = (union acpi_object []) {
 310                         [0] = {
 311                                 .integer.type = ACPI_TYPE_INTEGER,
 312                                 .integer.value = offset,
 313                         },
 314                         [1] = {
 315                                 .integer.type = ACPI_TYPE_INTEGER,
 316                                 .integer.value = len,
 317                         },
 318                         [2] = {
 319                                 .buffer.type = ACPI_TYPE_BUFFER,
 320                                 .buffer.pointer = data,
 321                                 .buffer.length = len,
 322                         },
 323                 },
 324         };
 325
 326         rc = acpi_evaluate_object(handle, "_LSW", &input, &buf);
 327         if (ACPI_FAILURE(rc))
 328                 return NULL;
 329         return int_to_buf(buf.pointer);
 330 }
 331
 332 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset,
 333                 u32 len)
 334 {
 335         acpi_status rc;
 336         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 337         struct acpi_object_list input = {
 338                 .count = 2,
 339                 .pointer = (union acpi_object []) {
 340                         [0] = {
 341                                 .integer.type = ACPI_TYPE_INTEGER,
 342                                 .integer.value = offset,
 343                         },
 344                         [1] = {
 345                                 .integer.type = ACPI_TYPE_INTEGER,
 346                                 .integer.value = len,
 347                         },
 348                 },
 349         };
 350
 351         rc = acpi_evaluate_object(handle, "_LSR", &input, &buf);
 352         if (ACPI_FAILURE(rc))
 353                 return NULL;
 354         return pkg_to_buf(buf.pointer);
 355 }
 356
 357 static union acpi_object *acpi_label_info(acpi_handle handle)
 358 {
 359         acpi_status rc;
 360         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 361
 362         rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf);
 363         if (ACPI_FAILURE(rc))
 364                 return NULL;
 365         return pkg_to_buf(buf.pointer);
 366 }
 367
 368 static u8 nfit_dsm_revid(unsigned family, unsigned func)
 369 {
 370         static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = {
 371                 [NVDIMM_FAMILY_INTEL] = {
 372                         [NVDIMM_INTEL_GET_MODES] = 2,
 373                         [NVDIMM_INTEL_GET_FWINFO] = 2,
 374                         [NVDIMM_INTEL_START_FWUPDATE] = 2,
 375                         [NVDIMM_INTEL_SEND_FWUPDATE] = 2,
 376                         [NVDIMM_INTEL_FINISH_FWUPDATE] = 2,
 377                         [NVDIMM_INTEL_QUERY_FWUPDATE] = 2,
 378                         [NVDIMM_INTEL_SET_THRESHOLD] = 2,
 379                         [NVDIMM_INTEL_INJECT_ERROR] = 2,
 380                 },
 381         };
 382         u8 id;
 383
 384         if (family > NVDIMM_FAMILY_MAX)
 385                 return 0;
 386         if (func > 31)
 387                 return 0;
 388         id = revid_table[family][func];
 389         if (id == 0)
 390                 return 1; /* default */
 391         return id;
 392 }
 393
 394 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
 395                 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc)
 396 {
 397         struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
 398         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 399         union acpi_object in_obj, in_buf, *out_obj;
 400         const struct nd_cmd_desc *desc = NULL;
 401         struct device *dev = acpi_desc->dev;
 402         struct nd_cmd_pkg *call_pkg = NULL;
 403         const char *cmd_name, *dimm_name;
 404         unsigned long cmd_mask, dsm_mask;
 405         u32 offset, fw_status = 0;
 406         acpi_handle handle;
 407         unsigned int func;
 408         const guid_t *guid;
 409         int rc, i;
 410
 411         if (cmd_rc)
 412                 *cmd_rc = -EINVAL;
 413         func = cmd;
 414         if (cmd == ND_CMD_CALL) {
 415                 call_pkg = buf;
 416                 func = call_pkg->nd_command;
 417
 418                 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++)
 419                         if (call_pkg->nd_reserved2[i])
 420                                 return -EINVAL;
 421         }
 422
 423         if (nvdimm) {
 424                 struct acpi_device *adev = nfit_mem->adev;
 425
 426                 if (!adev)
 427                         return -ENOTTY;
 428                 if (call_pkg && nfit_mem->family != call_pkg->nd_family)
 429                         return -ENOTTY;
 430
 431                 dimm_name = nvdimm_name(nvdimm);
 432                 cmd_name = nvdimm_cmd_name(cmd);
 433                 cmd_mask = nvdimm_cmd_mask(nvdimm);
 434                 dsm_mask = nfit_mem->dsm_mask;
 435                 desc = nd_cmd_dimm_desc(cmd);
 436                 guid = to_nfit_uuid(nfit_mem->family);
 437                 handle = adev->handle;
 438         } else {
 439                 struct acpi_device *adev = to_acpi_dev(acpi_desc);
 440
 441                 cmd_name = nvdimm_bus_cmd_name(cmd);
 442                 cmd_mask = nd_desc->cmd_mask;
 443                 dsm_mask = cmd_mask;
 444                 if (cmd == ND_CMD_CALL)
 445                         dsm_mask = nd_desc->bus_dsm_mask;
 446                 desc = nd_cmd_bus_desc(cmd);
 447                 guid = to_nfit_uuid(NFIT_DEV_BUS);
 448                 handle = adev->handle;
 449                 dimm_name = "bus";
 450         }
 451
 452         if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
 453                 return -ENOTTY;
 454
 455         if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask))
 456                 return -ENOTTY;
 457
 458         in_obj.type = ACPI_TYPE_PACKAGE;
 459         in_obj.package.count = 1;
 460         in_obj.package.elements = &in_buf;
 461         in_buf.type = ACPI_TYPE_BUFFER;
 462         in_buf.buffer.pointer = buf;
 463         in_buf.buffer.length = 0;
 464
 465         /* libnvdimm has already validated the input envelope */
 466         for (i = 0; i < desc->in_num; i++)
 467                 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
 468                                 i, buf);
 469
 470         if (call_pkg) {
 471                 /* skip over package wrapper */
 472                 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
 473                 in_buf.buffer.length = call_pkg->nd_size_in;
 474         }
 475
 476         dev_dbg(dev, "%s cmd: %d: func: %d input length: %d\n",
 477                 dimm_name, cmd, func, in_buf.buffer.length);
 478         print_hex_dump_debug("nvdimm in  ", DUMP_PREFIX_OFFSET, 4, 4,
 479                         in_buf.buffer.pointer,
 480                         min_t(u32, 256, in_buf.buffer.length), true);
 481
 482         /* call the BIOS, prefer the named methods over _DSM if available */
 483         if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE && nfit_mem->has_lsr)
 484                 out_obj = acpi_label_info(handle);
 485         else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && nfit_mem->has_lsr) {
 486                 struct nd_cmd_get_config_data_hdr *p = buf;
 487
 488                 out_obj = acpi_label_read(handle, p->in_offset, p->in_length);
 489         } else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA
 490                         && nfit_mem->has_lsw) {
 491                 struct nd_cmd_set_config_hdr *p = buf;
 492
 493                 out_obj = acpi_label_write(handle, p->in_offset, p->in_length,
 494                                 p->in_buf);
 495         } else {
 496                 u8 revid;
 497
 498                 if (nvdimm)
 499                         revid = nfit_dsm_revid(nfit_mem->family, func);
 500                 else
 501                         revid = 1;
 502                 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
 503         }
 504
 505         if (!out_obj) {
 506                 dev_dbg(dev, "%s _DSM failed cmd: %s\n", dimm_name, cmd_name);
 507                 return -EINVAL;
 508         }
 509
 510         if (call_pkg) {
 511                 call_pkg->nd_fw_size = out_obj->buffer.length;
 512                 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
 513                         out_obj->buffer.pointer,
 514                         min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
 515
 516                 ACPI_FREE(out_obj);
 517                 /*
 518                  * Need to support FW function w/o known size in advance.
 519                  * Caller can determine required size based upon nd_fw_size.
 520                  * If we return an error (like elsewhere) then caller wouldn't
 521                  * be able to rely upon data returned to make calculation.
 522                  */
 523                 if (cmd_rc)
 524                         *cmd_rc = 0;
 525                 return 0;
 526         }
 527
 528         if (out_obj->package.type != ACPI_TYPE_BUFFER) {
 529                 dev_dbg(dev, "%s unexpected output object type cmd: %s type: %d\n",
 530                                 dimm_name, cmd_name, out_obj->type);
 531                 rc = -EINVAL;
 532                 goto out;
 533         }
 534
 535         dev_dbg(dev, "%s cmd: %s output length: %d\n", dimm_name,
 536                         cmd_name, out_obj->buffer.length);
 537         print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4,
 538                         out_obj->buffer.pointer,
 539                         min_t(u32, 128, out_obj->buffer.length), true);
 540
 541         for (i = 0, offset = 0; i < desc->out_num; i++) {
 542                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
 543                                 (u32 *) out_obj->buffer.pointer,
 544                                 out_obj->buffer.length - offset);
 545
 546                 if (offset + out_size > out_obj->buffer.length) {
 547                         dev_dbg(dev, "%s output object underflow cmd: %s field: %d\n",
 548                                         dimm_name, cmd_name, i);
 549                         break;
 550                 }
 551
 552                 if (in_buf.buffer.length + offset + out_size > buf_len) {
 553                         dev_dbg(dev, "%s output overrun cmd: %s field: %d\n",
 554                                         dimm_name, cmd_name, i);
 555                         rc = -ENXIO;
 556                         goto out;
 557                 }
 558                 memcpy(buf + in_buf.buffer.length + offset,
 559                                 out_obj->buffer.pointer + offset, out_size);
 560                 offset += out_size;
 561         }
 562
 563         /*
 564          * Set fw_status for all the commands with a known format to be
 565          * later interpreted by xlat_status().
 566          */
 567         if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP
 568                                         && cmd <= ND_CMD_CLEAR_ERROR)
 569                                 || (nvdimm && cmd >= ND_CMD_SMART
 570                                         && cmd <= ND_CMD_VENDOR)))
 571                 fw_status = *(u32 *) out_obj->buffer.pointer;
 572
 573         if (offset + in_buf.buffer.length < buf_len) {
 574                 if (i >= 1) {
 575                         /*
 576                          * status valid, return the number of bytes left
 577                          * unfilled in the output buffer
 578                          */
 579                         rc = buf_len - offset - in_buf.buffer.length;
 580                         if (cmd_rc)
 581                                 *cmd_rc = xlat_status(nvdimm, buf, cmd,
 582                                                 fw_status);
 583                 } else {
 584                         dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
 585                                         __func__, dimm_name, cmd_name, buf_len,
 586                                         offset);
 587                         rc = -ENXIO;
 588                 }
 589         } else {
 590                 rc = 0;
 591                 if (cmd_rc)
 592                         *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status);
 593         }
 594
 595  out:
 596         ACPI_FREE(out_obj);
 597
 598         return rc;
 599 }
 600 EXPORT_SYMBOL_GPL(acpi_nfit_ctl);
 601
 602 static const char *spa_type_name(u16 type)
 603 {
 604         static const char *to_name[] = {
 605                 [NFIT_SPA_VOLATILE] = "volatile",
 606                 [NFIT_SPA_PM] = "pmem",
 607                 [NFIT_SPA_DCR] = "dimm-control-region",
 608                 [NFIT_SPA_BDW] = "block-data-window",
 609                 [NFIT_SPA_VDISK] = "volatile-disk",
 610                 [NFIT_SPA_VCD] = "volatile-cd",
 611                 [NFIT_SPA_PDISK] = "persistent-disk",
 612                 [NFIT_SPA_PCD] = "persistent-cd",
 613
 614         };
 615
 616         if (type > NFIT_SPA_PCD)
 617                 return "unknown";
 618
 619         return to_name[type];
 620 }
 621
 622 int nfit_spa_type(struct acpi_nfit_system_address *spa)
 623 {
 624         int i;
 625
 626         for (i = 0; i < NFIT_UUID_MAX; i++)
 627                 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid))
 628                         return i;
 629         return -1;
 630 }
 631
 632 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
 633                 struct nfit_table_prev *prev,
 634                 struct acpi_nfit_system_address *spa)
 635 {
 636         struct device *dev = acpi_desc->dev;
 637         struct nfit_spa *nfit_spa;
 638
 639         if (spa->header.length != sizeof(*spa))
 640                 return false;
 641
 642         list_for_each_entry(nfit_spa, &prev->spas, list) {
 643                 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) {
 644                         list_move_tail(&nfit_spa->list, &acpi_desc->spas);
 645                         return true;
 646                 }
 647         }
 648
 649         nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa),
 650                         GFP_KERNEL);
 651         if (!nfit_spa)
 652                 return false;
 653         INIT_LIST_HEAD(&nfit_spa->list);
 654         memcpy(nfit_spa->spa, spa, sizeof(*spa));
 655         list_add_tail(&nfit_spa->list, &acpi_desc->spas);
 656         dev_dbg(dev, "spa index: %d type: %s\n",
 657                         spa->range_index,
 658                         spa_type_name(nfit_spa_type(spa)));
 659         return true;
 660 }
 661
 662 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
 663                 struct nfit_table_prev *prev,
 664                 struct acpi_nfit_memory_map *memdev)
 665 {
 666         struct device *dev = acpi_desc->dev;
 667         struct nfit_memdev *nfit_memdev;
 668
 669         if (memdev->header.length != sizeof(*memdev))
 670                 return false;
 671
 672         list_for_each_entry(nfit_memdev, &prev->memdevs, list)
 673                 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) {
 674                         list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 675                         return true;
 676                 }
 677
 678         nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev),
 679                         GFP_KERNEL);
 680         if (!nfit_memdev)
 681                 return false;
 682         INIT_LIST_HEAD(&nfit_memdev->list);
 683         memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev));
 684         list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 685         dev_dbg(dev, "memdev handle: %#x spa: %d dcr: %d flags: %#x\n",
 686                         memdev->device_handle, memdev->range_index,
 687                         memdev->region_index, memdev->flags);
 688         return true;
 689 }
 690
 691 int nfit_get_smbios_id(u32 device_handle, u16 *flags)
 692 {
 693         struct acpi_nfit_memory_map *memdev;
 694         struct acpi_nfit_desc *acpi_desc;
 695         struct nfit_mem *nfit_mem;
 696
 697         mutex_lock(&acpi_desc_lock);
 698         list_for_each_entry(acpi_desc, &acpi_descs, list) {
 699                 mutex_lock(&acpi_desc->init_mutex);
 700                 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
 701                         memdev = __to_nfit_memdev(nfit_mem);
 702                         if (memdev->device_handle == device_handle) {
 703                                 mutex_unlock(&acpi_desc->init_mutex);
 704                                 mutex_unlock(&acpi_desc_lock);
 705                                 *flags = memdev->flags;
 706                                 return memdev->physical_id;
 707                         }
 708                 }
 709                 mutex_unlock(&acpi_desc->init_mutex);
 710         }
 711         mutex_unlock(&acpi_desc_lock);
 712
 713         return -ENODEV;
 714 }
 715 EXPORT_SYMBOL_GPL(nfit_get_smbios_id);
 716
 717 /*
 718  * An implementation may provide a truncated control region if no block windows
 719  * are defined.
 720  */
 721 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr)
 722 {
 723         if (dcr->header.length < offsetof(struct acpi_nfit_control_region,
 724                                 window_size))
 725                 return 0;
 726         if (dcr->windows)
 727                 return sizeof(*dcr);
 728         return offsetof(struct acpi_nfit_control_region, window_size);
 729 }
 730
 731 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
 732                 struct nfit_table_prev *prev,
 733                 struct acpi_nfit_control_region *dcr)
 734 {
 735         struct device *dev = acpi_desc->dev;
 736         struct nfit_dcr *nfit_dcr;
 737
 738         if (!sizeof_dcr(dcr))
 739                 return false;
 740
 741         list_for_each_entry(nfit_dcr, &prev->dcrs, list)
 742                 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) {
 743                         list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 744                         return true;
 745                 }
 746
 747         nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr),
 748                         GFP_KERNEL);
 749         if (!nfit_dcr)
 750                 return false;
 751         INIT_LIST_HEAD(&nfit_dcr->list);
 752         memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr));
 753         list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 754         dev_dbg(dev, "dcr index: %d windows: %d\n",
 755                         dcr->region_index, dcr->windows);
 756         return true;
 757 }
 758
 759 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
 760                 struct nfit_table_prev *prev,
 761                 struct acpi_nfit_data_region *bdw)
 762 {
 763         struct device *dev = acpi_desc->dev;
 764         struct nfit_bdw *nfit_bdw;
 765
 766         if (bdw->header.length != sizeof(*bdw))
 767                 return false;
 768         list_for_each_entry(nfit_bdw, &prev->bdws, list)
 769                 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) {
 770                         list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
 771                         return true;
 772                 }
 773
 774         nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw),
 775                         GFP_KERNEL);
 776         if (!nfit_bdw)
 777                 return false;
 778         INIT_LIST_HEAD(&nfit_bdw->list);
 779         memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw));
 780         list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
 781         dev_dbg(dev, "bdw dcr: %d windows: %d\n",
 782                         bdw->region_index, bdw->windows);
 783         return true;
 784 }
 785
 786 static size_t sizeof_idt(struct acpi_nfit_interleave *idt)
 787 {
 788         if (idt->header.length < sizeof(*idt))
 789                 return 0;
 790         return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1);
 791 }
 792
 793 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
 794                 struct nfit_table_prev *prev,
 795                 struct acpi_nfit_interleave *idt)
 796 {
 797         struct device *dev = acpi_desc->dev;
 798         struct nfit_idt *nfit_idt;
 799
 800         if (!sizeof_idt(idt))
 801                 return false;
 802
 803         list_for_each_entry(nfit_idt, &prev->idts, list) {
 804                 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt))
 805                         continue;
 806
 807                 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) {
 808                         list_move_tail(&nfit_idt->list, &acpi_desc->idts);
 809                         return true;
 810                 }
 811         }
 812
 813         nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt),
 814                         GFP_KERNEL);
 815         if (!nfit_idt)
 816                 return false;
 817         INIT_LIST_HEAD(&nfit_idt->list);
 818         memcpy(nfit_idt->idt, idt, sizeof_idt(idt));
 819         list_add_tail(&nfit_idt->list, &acpi_desc->idts);
 820         dev_dbg(dev, "idt index: %d num_lines: %d\n",
 821                         idt->interleave_index, idt->line_count);
 822         return true;
 823 }
 824
 825 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush)
 826 {
 827         if (flush->header.length < sizeof(*flush))
 828                 return 0;
 829         return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1);
 830 }
 831
 832 static bool add_flush(struct acpi_nfit_desc *acpi_desc,
 833                 struct nfit_table_prev *prev,
 834                 struct acpi_nfit_flush_address *flush)
 835 {
 836         struct device *dev = acpi_desc->dev;
 837         struct nfit_flush *nfit_flush;
 838
 839         if (!sizeof_flush(flush))
 840                 return false;
 841
 842         list_for_each_entry(nfit_flush, &prev->flushes, list) {
 843                 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush))
 844                         continue;
 845
 846                 if (memcmp(nfit_flush->flush, flush,
 847                                         sizeof_flush(flush)) == 0) {
 848                         list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
 849                         return true;
 850                 }
 851         }
 852
 853         nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush)
 854                         + sizeof_flush(flush), GFP_KERNEL);
 855         if (!nfit_flush)
 856                 return false;
 857         INIT_LIST_HEAD(&nfit_flush->list);
 858         memcpy(nfit_flush->flush, flush, sizeof_flush(flush));
 859         list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
 860         dev_dbg(dev, "nfit_flush handle: %d hint_count: %d\n",
 861                         flush->device_handle, flush->hint_count);
 862         return true;
 863 }
 864
 865 static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc,
 866                 struct acpi_nfit_capabilities *pcap)
 867 {
 868         struct device *dev = acpi_desc->dev;
 869         u32 mask;
 870
 871         mask = (1 << (pcap->highest_capability + 1)) - 1;
 872         acpi_desc->platform_cap = pcap->capabilities & mask;
 873         dev_dbg(dev, "cap: %#x\n", acpi_desc->platform_cap);
 874         return true;
 875 }
 876
 877 static void *add_table(struct acpi_nfit_desc *acpi_desc,
 878                 struct nfit_table_prev *prev, void *table, const void *end)
 879 {
 880         struct device *dev = acpi_desc->dev;
 881         struct acpi_nfit_header *hdr;
 882         void *err = ERR_PTR(-ENOMEM);
 883
 884         if (table >= end)
 885                 return NULL;
 886
 887         hdr = table;
 888         if (!hdr->length) {
 889                 dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
 890                         hdr->type);
 891                 return NULL;
 892         }
 893
 894         switch (hdr->type) {
 895         case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
 896                 if (!add_spa(acpi_desc, prev, table))
 897                         return err;
 898                 break;
 899         case ACPI_NFIT_TYPE_MEMORY_MAP:
 900                 if (!add_memdev(acpi_desc, prev, table))
 901                         return err;
 902                 break;
 903         case ACPI_NFIT_TYPE_CONTROL_REGION:
 904                 if (!add_dcr(acpi_desc, prev, table))
 905                         return err;
 906                 break;
 907         case ACPI_NFIT_TYPE_DATA_REGION:
 908                 if (!add_bdw(acpi_desc, prev, table))
 909                         return err;
 910                 break;
 911         case ACPI_NFIT_TYPE_INTERLEAVE:
 912                 if (!add_idt(acpi_desc, prev, table))
 913                         return err;
 914                 break;
 915         case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
 916                 if (!add_flush(acpi_desc, prev, table))
 917                         return err;
 918                 break;
 919         case ACPI_NFIT_TYPE_SMBIOS:
 920                 dev_dbg(dev, "smbios\n");
 921                 break;
 922         case ACPI_NFIT_TYPE_CAPABILITIES:
 923                 if (!add_platform_cap(acpi_desc, table))
 924                         return err;
 925                 break;
 926         default:
 927                 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
 928                 break;
 929         }
 930
 931         return table + hdr->length;
 932 }
 933
 934 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
 935                 struct nfit_mem *nfit_mem)
 936 {
 937         u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 938         u16 dcr = nfit_mem->dcr->region_index;
 939         struct nfit_spa *nfit_spa;
 940
 941         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 942                 u16 range_index = nfit_spa->spa->range_index;
 943                 int type = nfit_spa_type(nfit_spa->spa);
 944                 struct nfit_memdev *nfit_memdev;
 945
 946                 if (type != NFIT_SPA_BDW)
 947                         continue;
 948
 949                 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 950                         if (nfit_memdev->memdev->range_index != range_index)
 951                                 continue;
 952                         if (nfit_memdev->memdev->device_handle != device_handle)
 953                                 continue;
 954                         if (nfit_memdev->memdev->region_index != dcr)
 955                                 continue;
 956
 957                         nfit_mem->spa_bdw = nfit_spa->spa;
 958                         return;
 959                 }
 960         }
 961
 962         dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
 963                         nfit_mem->spa_dcr->range_index);
 964         nfit_mem->bdw = NULL;
 965 }
 966
 967 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
 968                 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
 969 {
 970         u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
 971         struct nfit_memdev *nfit_memdev;
 972         struct nfit_bdw *nfit_bdw;
 973         struct nfit_idt *nfit_idt;
 974         u16 idt_idx, range_index;
 975
 976         list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
 977                 if (nfit_bdw->bdw->region_index != dcr)
 978                         continue;
 979                 nfit_mem->bdw = nfit_bdw->bdw;
 980                 break;
 981         }
 982
 983         if (!nfit_mem->bdw)
 984                 return;
 985
 986         nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
 987
 988         if (!nfit_mem->spa_bdw)
 989                 return;
 990
 991         range_index = nfit_mem->spa_bdw->range_index;
 992         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 993                 if (nfit_memdev->memdev->range_index != range_index ||
 994                                 nfit_memdev->memdev->region_index != dcr)
 995                         continue;
 996                 nfit_mem->memdev_bdw = nfit_memdev->memdev;
 997                 idt_idx = nfit_memdev->memdev->interleave_index;
 998                 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
 999                         if (nfit_idt->idt->interleave_index != idt_idx)
1000                                 continue;
1001                         nfit_mem->idt_bdw = nfit_idt->idt;
1002                         break;
1003                 }
1004                 break;
1005         }
1006 }
1007
1008 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc,
1009                 struct acpi_nfit_system_address *spa)
1010 {
1011         struct nfit_mem *nfit_mem, *found;
1012         struct nfit_memdev *nfit_memdev;
1013         int type = spa ? nfit_spa_type(spa) : 0;
1014
1015         switch (type) {
1016         case NFIT_SPA_DCR:
1017         case NFIT_SPA_PM:
1018                 break;
1019         default:
1020                 if (spa)
1021                         return 0;
1022         }
1023
1024         /*
1025          * This loop runs in two modes, when a dimm is mapped the loop
1026          * adds memdev associations to an existing dimm, or creates a
1027          * dimm. In the unmapped dimm case this loop sweeps for memdev
1028          * instances with an invalid / zero range_index and adds those
1029          * dimms without spa associations.
1030          */
1031         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1032                 struct nfit_flush *nfit_flush;
1033                 struct nfit_dcr *nfit_dcr;
1034                 u32 device_handle;
1035                 u16 dcr;
1036
1037                 if (spa && nfit_memdev->memdev->range_index != spa->range_index)
1038                         continue;
1039                 if (!spa && nfit_memdev->memdev->range_index)
1040                         continue;
1041                 found = NULL;
1042                 dcr = nfit_memdev->memdev->region_index;
1043                 device_handle = nfit_memdev->memdev->device_handle;
1044                 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1045                         if (__to_nfit_memdev(nfit_mem)->device_handle
1046                                         == device_handle) {
1047                                 found = nfit_mem;
1048                                 break;
1049                         }
1050
1051                 if (found)
1052                         nfit_mem = found;
1053                 else {
1054                         nfit_mem = devm_kzalloc(acpi_desc->dev,
1055                                         sizeof(*nfit_mem), GFP_KERNEL);
1056                         if (!nfit_mem)
1057                                 return -ENOMEM;
1058                         INIT_LIST_HEAD(&nfit_mem->list);
1059                         nfit_mem->acpi_desc = acpi_desc;
1060                         list_add(&nfit_mem->list, &acpi_desc->dimms);
1061                 }
1062
1063                 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1064                         if (nfit_dcr->dcr->region_index != dcr)
1065                                 continue;
1066                         /*
1067                          * Record the control region for the dimm.  For
1068                          * the ACPI 6.1 case, where there are separate
1069                          * control regions for the pmem vs blk
1070                          * interfaces, be sure to record the extended
1071                          * blk details.
1072                          */
1073                         if (!nfit_mem->dcr)
1074                                 nfit_mem->dcr = nfit_dcr->dcr;
1075                         else if (nfit_mem->dcr->windows == 0
1076                                         && nfit_dcr->dcr->windows)
1077                                 nfit_mem->dcr = nfit_dcr->dcr;
1078                         break;
1079                 }
1080
1081                 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
1082                         struct acpi_nfit_flush_address *flush;
1083                         u16 i;
1084
1085                         if (nfit_flush->flush->device_handle != device_handle)
1086                                 continue;
1087                         nfit_mem->nfit_flush = nfit_flush;
1088                         flush = nfit_flush->flush;
1089                         nfit_mem->flush_wpq = devm_kcalloc(acpi_desc->dev,
1090                                         flush->hint_count,
1091                                         sizeof(struct resource),
1092                                         GFP_KERNEL);
1093                         if (!nfit_mem->flush_wpq)
1094                                 return -ENOMEM;
1095                         for (i = 0; i < flush->hint_count; i++) {
1096                                 struct resource *res = &nfit_mem->flush_wpq[i];
1097
1098                                 res->start = flush->hint_address[i];
1099                                 res->end = res->start + 8 - 1;
1100                         }
1101                         break;
1102                 }
1103
1104                 if (dcr && !nfit_mem->dcr) {
1105                         dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
1106                                         spa->range_index, dcr);
1107                         return -ENODEV;
1108                 }
1109
1110                 if (type == NFIT_SPA_DCR) {
1111                         struct nfit_idt *nfit_idt;
1112                         u16 idt_idx;
1113
1114                         /* multiple dimms may share a SPA when interleaved */
1115                         nfit_mem->spa_dcr = spa;
1116                         nfit_mem->memdev_dcr = nfit_memdev->memdev;
1117                         idt_idx = nfit_memdev->memdev->interleave_index;
1118                         list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1119                                 if (nfit_idt->idt->interleave_index != idt_idx)
1120                                         continue;
1121                                 nfit_mem->idt_dcr = nfit_idt->idt;
1122                                 break;
1123                         }
1124                         nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
1125                 } else if (type == NFIT_SPA_PM) {
1126                         /*
1127                          * A single dimm may belong to multiple SPA-PM
1128                          * ranges, record at least one in addition to
1129                          * any SPA-DCR range.
1130                          */
1131                         nfit_mem->memdev_pmem = nfit_memdev->memdev;
1132                 } else
1133                         nfit_mem->memdev_dcr = nfit_memdev->memdev;
1134         }
1135
1136         return 0;
1137 }
1138
1139 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
1140 {
1141         struct nfit_mem *a = container_of(_a, typeof(*a), list);
1142         struct nfit_mem *b = container_of(_b, typeof(*b), list);
1143         u32 handleA, handleB;
1144
1145         handleA = __to_nfit_memdev(a)->device_handle;
1146         handleB = __to_nfit_memdev(b)->device_handle;
1147         if (handleA < handleB)
1148                 return -1;
1149         else if (handleA > handleB)
1150                 return 1;
1151         return 0;
1152 }
1153
1154 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
1155 {
1156         struct nfit_spa *nfit_spa;
1157         int rc;
1158
1159
1160         /*
1161          * For each SPA-DCR or SPA-PMEM address range find its
1162          * corresponding MEMDEV(s).  From each MEMDEV find the
1163          * corresponding DCR.  Then, if we're operating on a SPA-DCR,
1164          * try to find a SPA-BDW and a corresponding BDW that references
1165          * the DCR.  Throw it all into an nfit_mem object.  Note, that
1166          * BDWs are optional.
1167          */
1168         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1169                 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa);
1170                 if (rc)
1171                         return rc;
1172         }
1173
1174         /*
1175          * If a DIMM has failed to be mapped into SPA there will be no
1176          * SPA entries above. Find and register all the unmapped DIMMs
1177          * for reporting and recovery purposes.
1178          */
1179         rc = __nfit_mem_init(acpi_desc, NULL);
1180         if (rc)
1181                 return rc;
1182
1183         list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
1184
1185         return 0;
1186 }
1187
1188 static ssize_t bus_dsm_mask_show(struct device *dev,
1189                 struct device_attribute *attr, char *buf)
1190 {
1191         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1192         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1193
1194         return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask);
1195 }
1196 static struct device_attribute dev_attr_bus_dsm_mask =
1197                 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL);
1198
1199 static ssize_t revision_show(struct device *dev,
1200                 struct device_attribute *attr, char *buf)
1201 {
1202         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1203         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1204         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1205
1206         return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
1207 }
1208 static DEVICE_ATTR_RO(revision);
1209
1210 static ssize_t hw_error_scrub_show(struct device *dev,
1211                 struct device_attribute *attr, char *buf)
1212 {
1213         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1214         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1215         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1216
1217         return sprintf(buf, "%d\n", acpi_desc->scrub_mode);
1218 }
1219
1220 /*
1221  * The 'hw_error_scrub' attribute can have the following values written to it:
1222  * '0': Switch to the default mode where an exception will only insert
1223  *      the address of the memory error into the poison and badblocks lists.
1224  * '1': Enable a full scrub to happen if an exception for a memory error is
1225  *      received.
1226  */
1227 static ssize_t hw_error_scrub_store(struct device *dev,
1228                 struct device_attribute *attr, const char *buf, size_t size)
1229 {
1230         struct nvdimm_bus_descriptor *nd_desc;
1231         ssize_t rc;
1232         long val;
1233
1234         rc = kstrtol(buf, 0, &val);
1235         if (rc)
1236                 return rc;
1237
1238         device_lock(dev);
1239         nd_desc = dev_get_drvdata(dev);
1240         if (nd_desc) {
1241                 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1242
1243                 switch (val) {
1244                 case HW_ERROR_SCRUB_ON:
1245                         acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON;
1246                         break;
1247                 case HW_ERROR_SCRUB_OFF:
1248                         acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF;
1249                         break;
1250                 default:
1251                         rc = -EINVAL;
1252                         break;
1253                 }
1254         }
1255         device_unlock(dev);
1256         if (rc)
1257                 return rc;
1258         return size;
1259 }
1260 static DEVICE_ATTR_RW(hw_error_scrub);
1261
1262 /*
1263  * This shows the number of full Address Range Scrubs that have been
1264  * completed since driver load time. Userspace can wait on this using
1265  * select/poll etc. A '+' at the end indicates an ARS is in progress
1266  */
1267 static ssize_t scrub_show(struct device *dev,
1268                 struct device_attribute *attr, char *buf)
1269 {
1270         struct nvdimm_bus_descriptor *nd_desc;
1271         ssize_t rc = -ENXIO;
1272
1273         device_lock(dev);
1274         nd_desc = dev_get_drvdata(dev);
1275         if (nd_desc) {
1276                 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1277
1278                 mutex_lock(&acpi_desc->init_mutex);
1279                 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count,
1280                                 acpi_desc->scrub_busy
1281                                 && !acpi_desc->cancel ? "+\n" : "\n");
1282                 mutex_unlock(&acpi_desc->init_mutex);
1283         }
1284         device_unlock(dev);
1285         return rc;
1286 }
1287
1288 static ssize_t scrub_store(struct device *dev,
1289                 struct device_attribute *attr, const char *buf, size_t size)
1290 {
1291         struct nvdimm_bus_descriptor *nd_desc;
1292         ssize_t rc;
1293         long val;
1294
1295         rc = kstrtol(buf, 0, &val);
1296         if (rc)
1297                 return rc;
1298         if (val != 1)
1299                 return -EINVAL;
1300
1301         device_lock(dev);
1302         nd_desc = dev_get_drvdata(dev);
1303         if (nd_desc) {
1304                 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1305
1306                 rc = acpi_nfit_ars_rescan(acpi_desc, 0);
1307         }
1308         device_unlock(dev);
1309         if (rc)
1310                 return rc;
1311         return size;
1312 }
1313 static DEVICE_ATTR_RW(scrub);
1314
1315 static bool ars_supported(struct nvdimm_bus *nvdimm_bus)
1316 {
1317         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1318         const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START
1319                 | 1 << ND_CMD_ARS_STATUS;
1320
1321         return (nd_desc->cmd_mask & mask) == mask;
1322 }
1323
1324 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n)
1325 {
1326         struct device *dev = container_of(kobj, struct device, kobj);
1327         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1328
1329         if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus))
1330                 return 0;
1331         return a->mode;
1332 }
1333
1334 static struct attribute *acpi_nfit_attributes[] = {
1335         &dev_attr_revision.attr,
1336         &dev_attr_scrub.attr,
1337         &dev_attr_hw_error_scrub.attr,
1338         &dev_attr_bus_dsm_mask.attr,
1339         NULL,
1340 };
1341
1342 static const struct attribute_group acpi_nfit_attribute_group = {
1343         .name = "nfit",
1344         .attrs = acpi_nfit_attributes,
1345         .is_visible = nfit_visible,
1346 };
1347
1348 static const struct attribute_group *acpi_nfit_attribute_groups[] = {
1349         &nvdimm_bus_attribute_group,
1350         &acpi_nfit_attribute_group,
1351         NULL,
1352 };
1353
1354 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
1355 {
1356         struct nvdimm *nvdimm = to_nvdimm(dev);
1357         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1358
1359         return __to_nfit_memdev(nfit_mem);
1360 }
1361
1362 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
1363 {
1364         struct nvdimm *nvdimm = to_nvdimm(dev);
1365         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1366
1367         return nfit_mem->dcr;
1368 }
1369
1370 static ssize_t handle_show(struct device *dev,
1371                 struct device_attribute *attr, char *buf)
1372 {
1373         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1374
1375         return sprintf(buf, "%#x\n", memdev->device_handle);
1376 }
1377 static DEVICE_ATTR_RO(handle);
1378
1379 static ssize_t phys_id_show(struct device *dev,
1380                 struct device_attribute *attr, char *buf)
1381 {
1382         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1383
1384         return sprintf(buf, "%#x\n", memdev->physical_id);
1385 }
1386 static DEVICE_ATTR_RO(phys_id);
1387
1388 static ssize_t vendor_show(struct device *dev,
1389                 struct device_attribute *attr, char *buf)
1390 {
1391         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1392
1393         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id));
1394 }
1395 static DEVICE_ATTR_RO(vendor);
1396
1397 static ssize_t rev_id_show(struct device *dev,
1398                 struct device_attribute *attr, char *buf)
1399 {
1400         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1401
1402         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id));
1403 }
1404 static DEVICE_ATTR_RO(rev_id);
1405
1406 static ssize_t device_show(struct device *dev,
1407                 struct device_attribute *attr, char *buf)
1408 {
1409         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1410
1411         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id));
1412 }
1413 static DEVICE_ATTR_RO(device);
1414
1415 static ssize_t subsystem_vendor_show(struct device *dev,
1416                 struct device_attribute *attr, char *buf)
1417 {
1418         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1419
1420         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id));
1421 }
1422 static DEVICE_ATTR_RO(subsystem_vendor);
1423
1424 static ssize_t subsystem_rev_id_show(struct device *dev,
1425                 struct device_attribute *attr, char *buf)
1426 {
1427         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1428
1429         return sprintf(buf, "0x%04x\n",
1430                         be16_to_cpu(dcr->subsystem_revision_id));
1431 }
1432 static DEVICE_ATTR_RO(subsystem_rev_id);
1433
1434 static ssize_t subsystem_device_show(struct device *dev,
1435                 struct device_attribute *attr, char *buf)
1436 {
1437         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1438
1439         return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id));
1440 }
1441 static DEVICE_ATTR_RO(subsystem_device);
1442
1443 static int num_nvdimm_formats(struct nvdimm *nvdimm)
1444 {
1445         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1446         int formats = 0;
1447
1448         if (nfit_mem->memdev_pmem)
1449                 formats++;
1450         if (nfit_mem->memdev_bdw)
1451                 formats++;
1452         return formats;
1453 }
1454
1455 static ssize_t format_show(struct device *dev,
1456                 struct device_attribute *attr, char *buf)
1457 {
1458         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1459
1460         return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code));
1461 }
1462 static DEVICE_ATTR_RO(format);
1463
1464 static ssize_t format1_show(struct device *dev,
1465                 struct device_attribute *attr, char *buf)
1466 {
1467         u32 handle;
1468         ssize_t rc = -ENXIO;
1469         struct nfit_mem *nfit_mem;
1470         struct nfit_memdev *nfit_memdev;
1471         struct acpi_nfit_desc *acpi_desc;
1472         struct nvdimm *nvdimm = to_nvdimm(dev);
1473         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1474
1475         nfit_mem = nvdimm_provider_data(nvdimm);
1476         acpi_desc = nfit_mem->acpi_desc;
1477         handle = to_nfit_memdev(dev)->device_handle;
1478
1479         /* assumes DIMMs have at most 2 published interface codes */
1480         mutex_lock(&acpi_desc->init_mutex);
1481         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1482                 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1483                 struct nfit_dcr *nfit_dcr;
1484
1485                 if (memdev->device_handle != handle)
1486                         continue;
1487
1488                 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1489                         if (nfit_dcr->dcr->region_index != memdev->region_index)
1490                                 continue;
1491                         if (nfit_dcr->dcr->code == dcr->code)
1492                                 continue;
1493                         rc = sprintf(buf, "0x%04x\n",
1494                                         le16_to_cpu(nfit_dcr->dcr->code));
1495                         break;
1496                 }
1497                 if (rc != ENXIO)
1498                         break;
1499         }
1500         mutex_unlock(&acpi_desc->init_mutex);
1501         return rc;
1502 }
1503 static DEVICE_ATTR_RO(format1);
1504
1505 static ssize_t formats_show(struct device *dev,
1506                 struct device_attribute *attr, char *buf)
1507 {
1508         struct nvdimm *nvdimm = to_nvdimm(dev);
1509
1510         return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm));
1511 }
1512 static DEVICE_ATTR_RO(formats);
1513
1514 static ssize_t serial_show(struct device *dev,
1515                 struct device_attribute *attr, char *buf)
1516 {
1517         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1518
1519         return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number));
1520 }
1521 static DEVICE_ATTR_RO(serial);
1522
1523 static ssize_t family_show(struct device *dev,
1524                 struct device_attribute *attr, char *buf)
1525 {
1526         struct nvdimm *nvdimm = to_nvdimm(dev);
1527         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1528
1529         if (nfit_mem->family < 0)
1530                 return -ENXIO;
1531         return sprintf(buf, "%d\n", nfit_mem->family);
1532 }
1533 static DEVICE_ATTR_RO(family);
1534
1535 static ssize_t dsm_mask_show(struct device *dev,
1536                 struct device_attribute *attr, char *buf)
1537 {
1538         struct nvdimm *nvdimm = to_nvdimm(dev);
1539         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1540
1541         if (nfit_mem->family < 0)
1542                 return -ENXIO;
1543         return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask);
1544 }
1545 static DEVICE_ATTR_RO(dsm_mask);
1546
1547 static ssize_t flags_show(struct device *dev,
1548                 struct device_attribute *attr, char *buf)
1549 {
1550         u16 flags = to_nfit_memdev(dev)->flags;
1551
1552         return sprintf(buf, "%s%s%s%s%s%s%s\n",
1553                 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
1554                 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
1555                 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
1556                 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
1557                 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "",
1558                 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "",
1559                 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : "");
1560 }
1561 static DEVICE_ATTR_RO(flags);
1562
1563 static ssize_t id_show(struct device *dev,
1564                 struct device_attribute *attr, char *buf)
1565 {
1566         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1567
1568         if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID)
1569                 return sprintf(buf, "%04x-%02x-%04x-%08x\n",
1570                                 be16_to_cpu(dcr->vendor_id),
1571                                 dcr->manufacturing_location,
1572                                 be16_to_cpu(dcr->manufacturing_date),
1573                                 be32_to_cpu(dcr->serial_number));
1574         else
1575                 return sprintf(buf, "%04x-%08x\n",
1576                                 be16_to_cpu(dcr->vendor_id),
1577                                 be32_to_cpu(dcr->serial_number));
1578 }
1579 static DEVICE_ATTR_RO(id);
1580
1581 static struct attribute *acpi_nfit_dimm_attributes[] = {
1582         &dev_attr_handle.attr,
1583         &dev_attr_phys_id.attr,
1584         &dev_attr_vendor.attr,
1585         &dev_attr_device.attr,
1586         &dev_attr_rev_id.attr,
1587         &dev_attr_subsystem_vendor.attr,
1588         &dev_attr_subsystem_device.attr,
1589         &dev_attr_subsystem_rev_id.attr,
1590         &dev_attr_format.attr,
1591         &dev_attr_formats.attr,
1592         &dev_attr_format1.attr,
1593         &dev_attr_serial.attr,
1594         &dev_attr_flags.attr,
1595         &dev_attr_id.attr,
1596         &dev_attr_family.attr,
1597         &dev_attr_dsm_mask.attr,
1598         NULL,
1599 };
1600
1601 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
1602                 struct attribute *a, int n)
1603 {
1604         struct device *dev = container_of(kobj, struct device, kobj);
1605         struct nvdimm *nvdimm = to_nvdimm(dev);
1606
1607         if (!to_nfit_dcr(dev)) {
1608                 /* Without a dcr only the memdev attributes can be surfaced */
1609                 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr
1610                                 || a == &dev_attr_flags.attr
1611                                 || a == &dev_attr_family.attr
1612                                 || a == &dev_attr_dsm_mask.attr)
1613                         return a->mode;
1614                 return 0;
1615         }
1616
1617         if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1)
1618                 return 0;
1619         return a->mode;
1620 }
1621
1622 static const struct attribute_group acpi_nfit_dimm_attribute_group = {
1623         .name = "nfit",
1624         .attrs = acpi_nfit_dimm_attributes,
1625         .is_visible = acpi_nfit_dimm_attr_visible,
1626 };
1627
1628 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
1629         &nvdimm_attribute_group,
1630         &nd_device_attribute_group,
1631         &acpi_nfit_dimm_attribute_group,
1632         NULL,
1633 };
1634
1635 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
1636                 u32 device_handle)
1637 {
1638         struct nfit_mem *nfit_mem;
1639
1640         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1641                 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
1642                         return nfit_mem->nvdimm;
1643
1644         return NULL;
1645 }
1646
1647 void __acpi_nvdimm_notify(struct device *dev, u32 event)
1648 {
1649         struct nfit_mem *nfit_mem;
1650         struct acpi_nfit_desc *acpi_desc;
1651
1652         dev_dbg(dev->parent, "%s: event: %d\n", dev_name(dev),
1653                         event);
1654
1655         if (event != NFIT_NOTIFY_DIMM_HEALTH) {
1656                 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev),
1657                                 event);
1658                 return;
1659         }
1660
1661         acpi_desc = dev_get_drvdata(dev->parent);
1662         if (!acpi_desc)
1663                 return;
1664
1665         /*
1666          * If we successfully retrieved acpi_desc, then we know nfit_mem data
1667          * is still valid.
1668          */
1669         nfit_mem = dev_get_drvdata(dev);
1670         if (nfit_mem && nfit_mem->flags_attr)
1671                 sysfs_notify_dirent(nfit_mem->flags_attr);
1672 }
1673 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify);
1674
1675 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
1676 {
1677         struct acpi_device *adev = data;
1678         struct device *dev = &adev->dev;
1679
1680         device_lock(dev->parent);
1681         __acpi_nvdimm_notify(dev, event);
1682         device_unlock(dev->parent);
1683 }
1684
1685 static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method)
1686 {
1687         acpi_handle handle;
1688         acpi_status status;
1689
1690         status = acpi_get_handle(adev->handle, method, &handle);
1691
1692         if (ACPI_SUCCESS(status))
1693                 return true;
1694         return false;
1695 }
1696
1697 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
1698                 struct nfit_mem *nfit_mem, u32 device_handle)
1699 {
1700         struct acpi_device *adev, *adev_dimm;
1701         struct device *dev = acpi_desc->dev;
1702         unsigned long dsm_mask, label_mask;
1703         const guid_t *guid;
1704         int i;
1705         int family = -1;
1706
1707         /* nfit test assumes 1:1 relationship between commands and dsms */
1708         nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
1709         nfit_mem->family = NVDIMM_FAMILY_INTEL;
1710         adev = to_acpi_dev(acpi_desc);
1711         if (!adev)
1712                 return 0;
1713
1714         adev_dimm = acpi_find_child_device(adev, device_handle, false);
1715         nfit_mem->adev = adev_dimm;
1716         if (!adev_dimm) {
1717                 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
1718                                 device_handle);
1719                 return force_enable_dimms ? 0 : -ENODEV;
1720         }
1721
1722         if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle,
1723                 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) {
1724                 dev_err(dev, "%s: notification registration failed\n",
1725                                 dev_name(&adev_dimm->dev));
1726                 return -ENXIO;
1727         }
1728         /*
1729          * Record nfit_mem for the notification path to track back to
1730          * the nfit sysfs attributes for this dimm device object.
1731          */
1732         dev_set_drvdata(&adev_dimm->dev, nfit_mem);
1733
1734         /*
1735          * Until standardization materializes we need to consider 4
1736          * different command sets.  Note, that checking for function0 (bit0)
1737          * tells us if any commands are reachable through this GUID.
1738          */
1739         for (i = 0; i <= NVDIMM_FAMILY_MAX; i++)
1740                 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1))
1741                         if (family < 0 || i == default_dsm_family)
1742                                 family = i;
1743
1744         /* limit the supported commands to those that are publicly documented */
1745         nfit_mem->family = family;
1746         if (override_dsm_mask && !disable_vendor_specific)
1747                 dsm_mask = override_dsm_mask;
1748         else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
1749                 dsm_mask = NVDIMM_INTEL_CMDMASK;
1750                 if (disable_vendor_specific)
1751                         dsm_mask &= ~(1 << ND_CMD_VENDOR);
1752         } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) {
1753                 dsm_mask = 0x1c3c76;
1754         } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) {
1755                 dsm_mask = 0x1fe;
1756                 if (disable_vendor_specific)
1757                         dsm_mask &= ~(1 << 8);
1758         } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) {
1759                 dsm_mask = 0xffffffff;
1760         } else {
1761                 dev_dbg(dev, "unknown dimm command family\n");
1762                 nfit_mem->family = -1;
1763                 /* DSMs are optional, continue loading the driver... */
1764                 return 0;
1765         }
1766
1767         guid = to_nfit_uuid(nfit_mem->family);
1768         for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
1769                 if (acpi_check_dsm(adev_dimm->handle, guid,
1770                                         nfit_dsm_revid(nfit_mem->family, i),
1771                                         1ULL << i))
1772                         set_bit(i, &nfit_mem->dsm_mask);
1773
1774         /*
1775          * Prefer the NVDIMM_FAMILY_INTEL label read commands if present
1776          * due to their better semantics handling locked capacity.
1777          */
1778         label_mask = 1 << ND_CMD_GET_CONFIG_SIZE | 1 << ND_CMD_GET_CONFIG_DATA
1779                 | 1 << ND_CMD_SET_CONFIG_DATA;
1780         if (family == NVDIMM_FAMILY_INTEL
1781                         && (dsm_mask & label_mask) == label_mask)
1782                 return 0;
1783
1784         if (acpi_nvdimm_has_method(adev_dimm, "_LSI")
1785                         && acpi_nvdimm_has_method(adev_dimm, "_LSR")) {
1786                 dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev));
1787                 nfit_mem->has_lsr = true;
1788         }
1789
1790         if (nfit_mem->has_lsr && acpi_nvdimm_has_method(adev_dimm, "_LSW")) {
1791                 dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev));
1792                 nfit_mem->has_lsw = true;
1793         }
1794
1795         return 0;
1796 }
1797
1798 static void shutdown_dimm_notify(void *data)
1799 {
1800         struct acpi_nfit_desc *acpi_desc = data;
1801         struct nfit_mem *nfit_mem;
1802
1803         mutex_lock(&acpi_desc->init_mutex);
1804         /*
1805          * Clear out the nfit_mem->flags_attr and shut down dimm event
1806          * notifications.
1807          */
1808         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1809                 struct acpi_device *adev_dimm = nfit_mem->adev;
1810
1811                 if (nfit_mem->flags_attr) {
1812                         sysfs_put(nfit_mem->flags_attr);
1813                         nfit_mem->flags_attr = NULL;
1814                 }
1815                 if (adev_dimm) {
1816                         acpi_remove_notify_handler(adev_dimm->handle,
1817                                         ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify);
1818                         dev_set_drvdata(&adev_dimm->dev, NULL);
1819                 }
1820         }
1821         mutex_unlock(&acpi_desc->init_mutex);
1822 }
1823
1824 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
1825 {
1826         struct nfit_mem *nfit_mem;
1827         int dimm_count = 0, rc;
1828         struct nvdimm *nvdimm;
1829
1830         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1831                 struct acpi_nfit_flush_address *flush;
1832                 unsigned long flags = 0, cmd_mask;
1833                 struct nfit_memdev *nfit_memdev;
1834                 u32 device_handle;
1835                 u16 mem_flags;
1836
1837                 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
1838                 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
1839                 if (nvdimm) {
1840                         dimm_count++;
1841                         continue;
1842                 }
1843
1844                 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
1845                         set_bit(NDD_ALIASING, &flags);
1846
1847                 /* collate flags across all memdevs for this dimm */
1848                 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1849                         struct acpi_nfit_memory_map *dimm_memdev;
1850
1851                         dimm_memdev = __to_nfit_memdev(nfit_mem);
1852                         if (dimm_memdev->device_handle
1853                                         != nfit_memdev->memdev->device_handle)
1854                                 continue;
1855                         dimm_memdev->flags |= nfit_memdev->memdev->flags;
1856                 }
1857
1858                 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
1859                 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
1860                         set_bit(NDD_UNARMED, &flags);
1861
1862                 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
1863                 if (rc)
1864                         continue;
1865
1866                 /*
1867                  * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
1868                  * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
1869                  * userspace interface.
1870                  */
1871                 cmd_mask = 1UL << ND_CMD_CALL;
1872                 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
1873                         /*
1874                          * These commands have a 1:1 correspondence
1875                          * between DSM payload and libnvdimm ioctl
1876                          * payload format.
1877                          */
1878                         cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK;
1879                 }
1880
1881                 if (nfit_mem->has_lsr) {
1882                         set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
1883                         set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
1884                 }
1885                 if (nfit_mem->has_lsw)
1886                         set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
1887
1888                 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush
1889                         : NULL;
1890                 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
1891                                 acpi_nfit_dimm_attribute_groups,
1892                                 flags, cmd_mask, flush ? flush->hint_count : 0,
1893                                 nfit_mem->flush_wpq);
1894                 if (!nvdimm)
1895                         return -ENOMEM;
1896
1897                 nfit_mem->nvdimm = nvdimm;
1898                 dimm_count++;
1899
1900                 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
1901                         continue;
1902
1903                 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s%s\n",
1904                                 nvdimm_name(nvdimm),
1905                   mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
1906                   mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
1907                   mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
1908                   mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "",
1909                   mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : "");
1910
1911         }
1912
1913         rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
1914         if (rc)
1915                 return rc;
1916
1917         /*
1918          * Now that dimms are successfully registered, and async registration
1919          * is flushed, attempt to enable event notification.
1920          */
1921         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1922                 struct kernfs_node *nfit_kernfs;
1923
1924                 nvdimm = nfit_mem->nvdimm;
1925                 if (!nvdimm)
1926                         continue;
1927
1928                 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
1929                 if (nfit_kernfs)
1930                         nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
1931                                         "flags");
1932                 sysfs_put(nfit_kernfs);
1933                 if (!nfit_mem->flags_attr)
1934                         dev_warn(acpi_desc->dev, "%s: notifications disabled\n",
1935                                         nvdimm_name(nvdimm));
1936         }
1937
1938         return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify,
1939                         acpi_desc);
1940 }
1941
1942 /*
1943  * These constants are private because there are no kernel consumers of
1944  * these commands.
1945  */
1946 enum nfit_aux_cmds {
1947         NFIT_CMD_TRANSLATE_SPA = 5,
1948         NFIT_CMD_ARS_INJECT_SET = 7,
1949         NFIT_CMD_ARS_INJECT_CLEAR = 8,
1950         NFIT_CMD_ARS_INJECT_GET = 9,
1951 };
1952
1953 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
1954 {
1955         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1956         const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS);
1957         struct acpi_device *adev;
1958         unsigned long dsm_mask;
1959         int i;
1960
1961         nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
1962         nd_desc->bus_dsm_mask = acpi_desc->bus_nfit_cmd_force_en;
1963         adev = to_acpi_dev(acpi_desc);
1964         if (!adev)
1965                 return;
1966
1967         for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
1968                 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
1969                         set_bit(i, &nd_desc->cmd_mask);
1970         set_bit(ND_CMD_CALL, &nd_desc->cmd_mask);
1971
1972         dsm_mask =
1973                 (1 << ND_CMD_ARS_CAP) |
1974                 (1 << ND_CMD_ARS_START) |
1975                 (1 << ND_CMD_ARS_STATUS) |
1976                 (1 << ND_CMD_CLEAR_ERROR) |
1977                 (1 << NFIT_CMD_TRANSLATE_SPA) |
1978                 (1 << NFIT_CMD_ARS_INJECT_SET) |
1979                 (1 << NFIT_CMD_ARS_INJECT_CLEAR) |
1980                 (1 << NFIT_CMD_ARS_INJECT_GET);
1981         for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
1982                 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
1983                         set_bit(i, &nd_desc->bus_dsm_mask);
1984 }
1985
1986 static ssize_t range_index_show(struct device *dev,
1987                 struct device_attribute *attr, char *buf)
1988 {
1989         struct nd_region *nd_region = to_nd_region(dev);
1990         struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
1991
1992         return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
1993 }
1994 static DEVICE_ATTR_RO(range_index);
1995
1996 static struct attribute *acpi_nfit_region_attributes[] = {
1997         &dev_attr_range_index.attr,
1998         NULL,
1999 };
2000
2001 static const struct attribute_group acpi_nfit_region_attribute_group = {
2002         .name = "nfit",
2003         .attrs = acpi_nfit_region_attributes,
2004 };
2005
2006 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
2007         &nd_region_attribute_group,
2008         &nd_mapping_attribute_group,
2009         &nd_device_attribute_group,
2010         &nd_numa_attribute_group,
2011         &acpi_nfit_region_attribute_group,
2012         NULL,
2013 };
2014
2015 /* enough info to uniquely specify an interleave set */
2016 struct nfit_set_info {
2017         struct nfit_set_info_map {
2018                 u64 region_offset;
2019                 u32 serial_number;
2020                 u32 pad;
2021         } mapping[0];
2022 };
2023
2024 struct nfit_set_info2 {
2025         struct nfit_set_info_map2 {
2026                 u64 region_offset;
2027                 u32 serial_number;
2028                 u16 vendor_id;
2029                 u16 manufacturing_date;
2030                 u8  manufacturing_location;
2031                 u8  reserved[31];
2032         } mapping[0];
2033 };
2034
2035 static size_t sizeof_nfit_set_info(int num_mappings)
2036 {
2037         return sizeof(struct nfit_set_info)
2038                 + num_mappings * sizeof(struct nfit_set_info_map);
2039 }
2040
2041 static size_t sizeof_nfit_set_info2(int num_mappings)
2042 {
2043         return sizeof(struct nfit_set_info2)
2044                 + num_mappings * sizeof(struct nfit_set_info_map2);
2045 }
2046
2047 static int cmp_map_compat(const void *m0, const void *m1)
2048 {
2049         const struct nfit_set_info_map *map0 = m0;
2050         const struct nfit_set_info_map *map1 = m1;
2051
2052         return memcmp(&map0->region_offset, &map1->region_offset,
2053                         sizeof(u64));
2054 }
2055
2056 static int cmp_map(const void *m0, const void *m1)
2057 {
2058         const struct nfit_set_info_map *map0 = m0;
2059         const struct nfit_set_info_map *map1 = m1;
2060
2061         if (map0->region_offset < map1->region_offset)
2062                 return -1;
2063         else if (map0->region_offset > map1->region_offset)
2064                 return 1;
2065         return 0;
2066 }
2067
2068 static int cmp_map2(const void *m0, const void *m1)
2069 {
2070         const struct nfit_set_info_map2 *map0 = m0;
2071         const struct nfit_set_info_map2 *map1 = m1;
2072
2073         if (map0->region_offset < map1->region_offset)
2074                 return -1;
2075         else if (map0->region_offset > map1->region_offset)
2076                 return 1;
2077         return 0;
2078 }
2079
2080 /* Retrieve the nth entry referencing this spa */
2081 static struct acpi_nfit_memory_map *memdev_from_spa(
2082                 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
2083 {
2084         struct nfit_memdev *nfit_memdev;
2085
2086         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
2087                 if (nfit_memdev->memdev->range_index == range_index)
2088                         if (n-- == 0)
2089                                 return nfit_memdev->memdev;
2090         return NULL;
2091 }
2092
2093 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
2094                 struct nd_region_desc *ndr_desc,
2095                 struct acpi_nfit_system_address *spa)
2096 {
2097         struct device *dev = acpi_desc->dev;
2098         struct nd_interleave_set *nd_set;
2099         u16 nr = ndr_desc->num_mappings;
2100         struct nfit_set_info2 *info2;
2101         struct nfit_set_info *info;
2102         int i;
2103
2104         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
2105         if (!nd_set)
2106                 return -ENOMEM;
2107         ndr_desc->nd_set = nd_set;
2108         guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid);
2109
2110         info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
2111         if (!info)
2112                 return -ENOMEM;
2113
2114         info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL);
2115         if (!info2)
2116                 return -ENOMEM;
2117
2118         for (i = 0; i < nr; i++) {
2119                 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
2120                 struct nfit_set_info_map *map = &info->mapping[i];
2121                 struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2122                 struct nvdimm *nvdimm = mapping->nvdimm;
2123                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2124                 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
2125                                 spa->range_index, i);
2126                 struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2127
2128                 if (!memdev || !nfit_mem->dcr) {
2129                         dev_err(dev, "%s: failed to find DCR\n", __func__);
2130                         return -ENODEV;
2131                 }
2132
2133                 map->region_offset = memdev->region_offset;
2134                 map->serial_number = dcr->serial_number;
2135
2136                 map2->region_offset = memdev->region_offset;
2137                 map2->serial_number = dcr->serial_number;
2138                 map2->vendor_id = dcr->vendor_id;
2139                 map2->manufacturing_date = dcr->manufacturing_date;
2140                 map2->manufacturing_location = dcr->manufacturing_location;
2141         }
2142
2143         /* v1.1 namespaces */
2144         sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2145                         cmp_map, NULL);
2146         nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2147
2148         /* v1.2 namespaces */
2149         sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2),
2150                         cmp_map2, NULL);
2151         nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0);
2152
2153         /* support v1.1 namespaces created with the wrong sort order */
2154         sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2155                         cmp_map_compat, NULL);
2156         nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2157
2158         /* record the result of the sort for the mapping position */
2159         for (i = 0; i < nr; i++) {
2160                 struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2161                 int j;
2162
2163                 for (j = 0; j < nr; j++) {
2164                         struct nd_mapping_desc *mapping = &ndr_desc->mapping[j];
2165                         struct nvdimm *nvdimm = mapping->nvdimm;
2166                         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2167                         struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2168
2169                         if (map2->serial_number == dcr->serial_number &&
2170                             map2->vendor_id == dcr->vendor_id &&
2171                             map2->manufacturing_date == dcr->manufacturing_date &&
2172                             map2->manufacturing_location
2173                                     == dcr->manufacturing_location) {
2174                                 mapping->position = i;
2175                                 break;
2176                         }
2177                 }
2178         }
2179
2180         ndr_desc->nd_set = nd_set;
2181         devm_kfree(dev, info);
2182         devm_kfree(dev, info2);
2183
2184         return 0;
2185 }
2186
2187 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
2188 {
2189         struct acpi_nfit_interleave *idt = mmio->idt;
2190         u32 sub_line_offset, line_index, line_offset;
2191         u64 line_no, table_skip_count, table_offset;
2192
2193         line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
2194         table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
2195         line_offset = idt->line_offset[line_index]
2196                 * mmio->line_size;
2197         table_offset = table_skip_count * mmio->table_size;
2198
2199         return mmio->base_offset + line_offset + table_offset + sub_line_offset;
2200 }
2201
2202 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
2203 {
2204         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2205         u64 offset = nfit_blk->stat_offset + mmio->size * bw;
2206         const u32 STATUS_MASK = 0x80000037;
2207
2208         if (mmio->num_lines)
2209                 offset = to_interleave_offset(offset, mmio);
2210
2211         return readl(mmio->addr.base + offset) & STATUS_MASK;
2212 }
2213
2214 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
2215                 resource_size_t dpa, unsigned int len, unsigned int write)
2216 {
2217         u64 cmd, offset;
2218         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2219
2220         enum {
2221                 BCW_OFFSET_MASK = (1ULL << 48)-1,
2222                 BCW_LEN_SHIFT = 48,
2223                 BCW_LEN_MASK = (1ULL << 8) - 1,
2224                 BCW_CMD_SHIFT = 56,
2225         };
2226
2227         cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
2228         len = len >> L1_CACHE_SHIFT;
2229         cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
2230         cmd |= ((u64) write) << BCW_CMD_SHIFT;
2231
2232         offset = nfit_blk->cmd_offset + mmio->size * bw;
2233         if (mmio->num_lines)
2234                 offset = to_interleave_offset(offset, mmio);
2235
2236         writeq(cmd, mmio->addr.base + offset);
2237         nvdimm_flush(nfit_blk->nd_region);
2238
2239         if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
2240                 readq(mmio->addr.base + offset);
2241 }
2242
2243 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
2244                 resource_size_t dpa, void *iobuf, size_t len, int rw,
2245                 unsigned int lane)
2246 {
2247         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2248         unsigned int copied = 0;
2249         u64 base_offset;
2250         int rc;
2251
2252         base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
2253                 + lane * mmio->size;
2254         write_blk_ctl(nfit_blk, lane, dpa, len, rw);
2255         while (len) {
2256                 unsigned int c;
2257                 u64 offset;
2258
2259                 if (mmio->num_lines) {
2260                         u32 line_offset;
2261
2262                         offset = to_interleave_offset(base_offset + copied,
2263                                         mmio);
2264                         div_u64_rem(offset, mmio->line_size, &line_offset);
2265                         c = min_t(size_t, len, mmio->line_size - line_offset);
2266                 } else {
2267                         offset = base_offset + nfit_blk->bdw_offset;
2268                         c = len;
2269                 }
2270
2271                 if (rw)
2272                         memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c);
2273                 else {
2274                         if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
2275                                 arch_invalidate_pmem((void __force *)
2276                                         mmio->addr.aperture + offset, c);
2277
2278                         memcpy(iobuf + copied, mmio->addr.aperture + offset, c);
2279                 }
2280
2281                 copied += c;
2282                 len -= c;
2283         }
2284
2285         if (rw)
2286                 nvdimm_flush(nfit_blk->nd_region);
2287
2288         rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
2289         return rc;
2290 }
2291
2292 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
2293                 resource_size_t dpa, void *iobuf, u64 len, int rw)
2294 {
2295         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
2296         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2297         struct nd_region *nd_region = nfit_blk->nd_region;
2298         unsigned int lane, copied = 0;
2299         int rc = 0;
2300
2301         lane = nd_region_acquire_lane(nd_region);
2302         while (len) {
2303                 u64 c = min(len, mmio->size);
2304
2305                 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
2306                                 iobuf + copied, c, rw, lane);
2307                 if (rc)
2308                         break;
2309
2310                 copied += c;
2311                 len -= c;
2312         }
2313         nd_region_release_lane(nd_region, lane);
2314
2315         return rc;
2316 }
2317
2318 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
2319                 struct acpi_nfit_interleave *idt, u16 interleave_ways)
2320 {
2321         if (idt) {
2322                 mmio->num_lines = idt->line_count;
2323                 mmio->line_size = idt->line_size;
2324                 if (interleave_ways == 0)
2325                         return -ENXIO;
2326                 mmio->table_size = mmio->num_lines * interleave_ways
2327                         * mmio->line_size;
2328         }
2329
2330         return 0;
2331 }
2332
2333 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
2334                 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
2335 {
2336         struct nd_cmd_dimm_flags flags;
2337         int rc;
2338
2339         memset(&flags, 0, sizeof(flags));
2340         rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
2341                         sizeof(flags), NULL);
2342
2343         if (rc >= 0 && flags.status == 0)
2344                 nfit_blk->dimm_flags = flags.flags;
2345         else if (rc == -ENOTTY) {
2346                 /* fall back to a conservative default */
2347                 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
2348                 rc = 0;
2349         } else
2350                 rc = -ENXIO;
2351
2352         return rc;
2353 }
2354
2355 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
2356                 struct device *dev)
2357 {
2358         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
2359         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
2360         struct nfit_blk_mmio *mmio;
2361         struct nfit_blk *nfit_blk;
2362         struct nfit_mem *nfit_mem;
2363         struct nvdimm *nvdimm;
2364         int rc;
2365
2366         nvdimm = nd_blk_region_to_dimm(ndbr);
2367         nfit_mem = nvdimm_provider_data(nvdimm);
2368         if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
2369                 dev_dbg(dev, "missing%s%s%s\n",
2370                                 nfit_mem ? "" : " nfit_mem",
2371                                 (nfit_mem && nfit_mem->dcr) ? "" : " dcr",
2372                                 (nfit_mem && nfit_mem->bdw) ? "" : " bdw");
2373                 return -ENXIO;
2374         }
2375
2376         nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
2377         if (!nfit_blk)
2378                 return -ENOMEM;
2379         nd_blk_region_set_provider_data(ndbr, nfit_blk);
2380         nfit_blk->nd_region = to_nd_region(dev);
2381
2382         /* map block aperture memory */
2383         nfit_blk->bdw_offset = nfit_mem->bdw->offset;
2384         mmio = &nfit_blk->mmio[BDW];
2385         mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address,
2386                         nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr));
2387         if (!mmio->addr.base) {
2388                 dev_dbg(dev, "%s failed to map bdw\n",
2389                                 nvdimm_name(nvdimm));
2390                 return -ENOMEM;
2391         }
2392         mmio->size = nfit_mem->bdw->size;
2393         mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
2394         mmio->idt = nfit_mem->idt_bdw;
2395         mmio->spa = nfit_mem->spa_bdw;
2396         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
2397                         nfit_mem->memdev_bdw->interleave_ways);
2398         if (rc) {
2399                 dev_dbg(dev, "%s failed to init bdw interleave\n",
2400                                 nvdimm_name(nvdimm));
2401                 return rc;
2402         }
2403
2404         /* map block control memory */
2405         nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
2406         nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
2407         mmio = &nfit_blk->mmio[DCR];
2408         mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address,
2409                         nfit_mem->spa_dcr->length);
2410         if (!mmio->addr.base) {
2411                 dev_dbg(dev, "%s failed to map dcr\n",
2412                                 nvdimm_name(nvdimm));
2413                 return -ENOMEM;
2414         }
2415         mmio->size = nfit_mem->dcr->window_size;
2416         mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
2417         mmio->idt = nfit_mem->idt_dcr;
2418         mmio->spa = nfit_mem->spa_dcr;
2419         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
2420                         nfit_mem->memdev_dcr->interleave_ways);
2421         if (rc) {
2422                 dev_dbg(dev, "%s failed to init dcr interleave\n",
2423                                 nvdimm_name(nvdimm));
2424                 return rc;
2425         }
2426
2427         rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
2428         if (rc < 0) {
2429                 dev_dbg(dev, "%s failed get DIMM flags\n",
2430                                 nvdimm_name(nvdimm));
2431                 return rc;
2432         }
2433
2434         if (nvdimm_has_flush(nfit_blk->nd_region) < 0)
2435                 dev_warn(dev, "unable to guarantee persistence of writes\n");
2436
2437         if (mmio->line_size == 0)
2438                 return 0;
2439
2440         if ((u32) nfit_blk->cmd_offset % mmio->line_size
2441                         + 8 > mmio->line_size) {
2442                 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
2443                 return -ENXIO;
2444         } else if ((u32) nfit_blk->stat_offset % mmio->line_size
2445                         + 8 > mmio->line_size) {
2446                 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
2447                 return -ENXIO;
2448         }
2449
2450         return 0;
2451 }
2452
2453 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
2454                 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
2455 {
2456         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2457         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2458         int cmd_rc, rc;
2459
2460         cmd->address = spa->address;
2461         cmd->length = spa->length;
2462         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
2463                         sizeof(*cmd), &cmd_rc);
2464         if (rc < 0)
2465                 return rc;
2466         return cmd_rc;
2467 }
2468
2469 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa)
2470 {
2471         int rc;
2472         int cmd_rc;
2473         struct nd_cmd_ars_start ars_start;
2474         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2475         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2476
2477         memset(&ars_start, 0, sizeof(ars_start));
2478         ars_start.address = spa->address;
2479         ars_start.length = spa->length;
2480         if (test_bit(ARS_SHORT, &nfit_spa->ars_state))
2481                 ars_start.flags = ND_ARS_RETURN_PREV_DATA;
2482         if (nfit_spa_type(spa) == NFIT_SPA_PM)
2483                 ars_start.type = ND_ARS_PERSISTENT;
2484         else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
2485                 ars_start.type = ND_ARS_VOLATILE;
2486         else
2487                 return -ENOTTY;
2488
2489         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2490                         sizeof(ars_start), &cmd_rc);
2491
2492         if (rc < 0)
2493                 return rc;
2494         return cmd_rc;
2495 }
2496
2497 static int ars_continue(struct acpi_nfit_desc *acpi_desc)
2498 {
2499         int rc, cmd_rc;
2500         struct nd_cmd_ars_start ars_start;
2501         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2502         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2503
2504         memset(&ars_start, 0, sizeof(ars_start));
2505         ars_start.address = ars_status->restart_address;
2506         ars_start.length = ars_status->restart_length;
2507         ars_start.type = ars_status->type;
2508         ars_start.flags = acpi_desc->ars_start_flags;
2509         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2510                         sizeof(ars_start), &cmd_rc);
2511         if (rc < 0)
2512                 return rc;
2513         return cmd_rc;
2514 }
2515
2516 static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
2517 {
2518         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2519         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2520         int rc, cmd_rc;
2521
2522         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
2523                         acpi_desc->max_ars, &cmd_rc);
2524         if (rc < 0)
2525                 return rc;
2526         return cmd_rc;
2527 }
2528
2529 static void ars_complete(struct acpi_nfit_desc *acpi_desc,
2530                 struct nfit_spa *nfit_spa)
2531 {
2532         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2533         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2534         struct nd_region *nd_region = nfit_spa->nd_region;
2535         struct device *dev;
2536
2537         if ((ars_status->address >= spa->address && ars_status->address
2538                                 < spa->address + spa->length)
2539                         || (ars_status->address < spa->address)) {
2540                 /*
2541                  * Assume that if a scrub starts at an offset from the
2542                  * start of nfit_spa that we are in the continuation
2543                  * case.
2544                  *
2545                  * Otherwise, if the scrub covers the spa range, mark
2546                  * any pending request complete.
2547                  */
2548                 if (ars_status->address + ars_status->length
2549                                 >= spa->address + spa->length)
2550                                 /* complete */;
2551                 else
2552                         return;
2553         } else
2554                 return;
2555
2556         if (test_bit(ARS_DONE, &nfit_spa->ars_state))
2557                 return;
2558
2559         if (!test_and_clear_bit(ARS_REQ, &nfit_spa->ars_state))
2560                 return;
2561
2562         if (nd_region) {
2563                 dev = nd_region_dev(nd_region);
2564                 nvdimm_region_notify(nd_region, NVDIMM_REVALIDATE_POISON);
2565         } else
2566                 dev = acpi_desc->dev;
2567
2568         dev_dbg(dev, "ARS: range %d %s complete\n", spa->range_index,
2569                         test_bit(ARS_SHORT, &nfit_spa->ars_state)
2570                         ? "short" : "long");
2571         clear_bit(ARS_SHORT, &nfit_spa->ars_state);
2572         if (test_and_clear_bit(ARS_REQ_REDO, &nfit_spa->ars_state)) {
2573                 set_bit(ARS_SHORT, &nfit_spa->ars_state);
2574                 set_bit(ARS_REQ, &nfit_spa->ars_state);
2575                 dev_dbg(dev, "ARS: processing scrub request received while in progress\n");
2576         } else
2577                 set_bit(ARS_DONE, &nfit_spa->ars_state);
2578 }
2579
2580 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc)
2581 {
2582         struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus;
2583         struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2584         int rc;
2585         u32 i;
2586
2587         /*
2588          * First record starts at 44 byte offset from the start of the
2589          * payload.
2590          */
2591         if (ars_status->out_length < 44)
2592                 return 0;
2593         for (i = 0; i < ars_status->num_records; i++) {
2594                 /* only process full records */
2595                 if (ars_status->out_length
2596                                 < 44 + sizeof(struct nd_ars_record) * (i + 1))
2597                         break;
2598                 rc = nvdimm_bus_add_badrange(nvdimm_bus,
2599                                 ars_status->records[i].err_address,
2600                                 ars_status->records[i].length);
2601                 if (rc)
2602                         return rc;
2603         }
2604         if (i < ars_status->num_records)
2605                 dev_warn(acpi_desc->dev, "detected truncated ars results\n");
2606
2607         return 0;
2608 }
2609
2610 static void acpi_nfit_remove_resource(void *data)
2611 {
2612         struct resource *res = data;
2613
2614         remove_resource(res);
2615 }
2616
2617 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
2618                 struct nd_region_desc *ndr_desc)
2619 {
2620         struct resource *res, *nd_res = ndr_desc->res;
2621         int is_pmem, ret;
2622
2623         /* No operation if the region is already registered as PMEM */
2624         is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
2625                                 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
2626         if (is_pmem == REGION_INTERSECTS)
2627                 return 0;
2628
2629         res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
2630         if (!res)
2631                 return -ENOMEM;
2632
2633         res->name = "Persistent Memory";
2634         res->start = nd_res->start;
2635         res->end = nd_res->end;
2636         res->flags = IORESOURCE_MEM;
2637         res->desc = IORES_DESC_PERSISTENT_MEMORY;
2638
2639         ret = insert_resource(&iomem_resource, res);
2640         if (ret)
2641                 return ret;
2642
2643         ret = devm_add_action_or_reset(acpi_desc->dev,
2644                                         acpi_nfit_remove_resource,
2645                                         res);
2646         if (ret)
2647                 return ret;
2648
2649         return 0;
2650 }
2651
2652 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
2653                 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc,
2654                 struct acpi_nfit_memory_map *memdev,
2655                 struct nfit_spa *nfit_spa)
2656 {
2657         struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
2658                         memdev->device_handle);
2659         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2660         struct nd_blk_region_desc *ndbr_desc;
2661         struct nfit_mem *nfit_mem;
2662         int rc;
2663
2664         if (!nvdimm) {
2665                 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
2666                                 spa->range_index, memdev->device_handle);
2667                 return -ENODEV;
2668         }
2669
2670         mapping->nvdimm = nvdimm;
2671         switch (nfit_spa_type(spa)) {
2672         case NFIT_SPA_PM:
2673         case NFIT_SPA_VOLATILE:
2674                 mapping->start = memdev->address;
2675                 mapping->size = memdev->region_size;
2676                 break;
2677         case NFIT_SPA_DCR:
2678                 nfit_mem = nvdimm_provider_data(nvdimm);
2679                 if (!nfit_mem || !nfit_mem->bdw) {
2680                         dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
2681                                         spa->range_index, nvdimm_name(nvdimm));
2682                         break;
2683                 }
2684
2685                 mapping->size = nfit_mem->bdw->capacity;
2686                 mapping->start = nfit_mem->bdw->start_address;
2687                 ndr_desc->num_lanes = nfit_mem->bdw->windows;
2688                 ndr_desc->mapping = mapping;
2689                 ndr_desc->num_mappings = 1;
2690                 ndbr_desc = to_blk_region_desc(ndr_desc);
2691                 ndbr_desc->enable = acpi_nfit_blk_region_enable;
2692                 ndbr_desc->do_io = acpi_desc->blk_do_io;
2693                 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2694                 if (rc)
2695                         return rc;
2696                 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
2697                                 ndr_desc);
2698                 if (!nfit_spa->nd_region)
2699                         return -ENOMEM;
2700                 break;
2701         }
2702
2703         return 0;
2704 }
2705
2706 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa)
2707 {
2708         return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2709                 nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2710                 nfit_spa_type(spa) == NFIT_SPA_PDISK ||
2711                 nfit_spa_type(spa) == NFIT_SPA_PCD);
2712 }
2713
2714 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa)
2715 {
2716         return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2717                 nfit_spa_type(spa) == NFIT_SPA_VCD   ||
2718                 nfit_spa_type(spa) == NFIT_SPA_VOLATILE);
2719 }
2720
2721 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
2722                 struct nfit_spa *nfit_spa)
2723 {
2724         static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS];
2725         struct acpi_nfit_system_address *spa = nfit_spa->spa;
2726         struct nd_blk_region_desc ndbr_desc;
2727         struct nd_region_desc *ndr_desc;
2728         struct nfit_memdev *nfit_memdev;
2729         struct nvdimm_bus *nvdimm_bus;
2730         struct resource res;
2731         int count = 0, rc;
2732
2733         if (nfit_spa->nd_region)
2734                 return 0;
2735
2736         if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) {
2737                 dev_dbg(acpi_desc->dev, "detected invalid spa index\n");
2738                 return 0;
2739         }
2740
2741         memset(&res, 0, sizeof(res));
2742         memset(&mappings, 0, sizeof(mappings));
2743         memset(&ndbr_desc, 0, sizeof(ndbr_desc));
2744         res.start = spa->address;
2745         res.end = res.start + spa->length - 1;
2746         ndr_desc = &ndbr_desc.ndr_desc;
2747         ndr_desc->res = &res;
2748         ndr_desc->provider_data = nfit_spa;
2749         ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
2750         if (spa->flags & ACPI_NFIT_PROXIMITY_VALID)
2751                 ndr_desc->numa_node = acpi_map_pxm_to_online_node(
2752                                                 spa->proximity_domain);
2753         else
2754                 ndr_desc->numa_node = NUMA_NO_NODE;
2755
2756         /*
2757          * Persistence domain bits are hierarchical, if
2758          * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then
2759          * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied.
2760          */
2761         if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH)
2762                 set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags);
2763         else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH)
2764                 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags);
2765
2766         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2767                 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
2768                 struct nd_mapping_desc *mapping;
2769
2770                 if (memdev->range_index != spa->range_index)
2771                         continue;
2772                 if (count >= ND_MAX_MAPPINGS) {
2773                         dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
2774                                         spa->range_index, ND_MAX_MAPPINGS);
2775                         return -ENXIO;
2776                 }
2777                 mapping = &mappings[count++];
2778                 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc,
2779                                 memdev, nfit_spa);
2780                 if (rc)
2781                         goto out;
2782         }
2783
2784         ndr_desc->mapping = mappings;
2785         ndr_desc->num_mappings = count;
2786         rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2787         if (rc)
2788                 goto out;
2789
2790         nvdimm_bus = acpi_desc->nvdimm_bus;
2791         if (nfit_spa_type(spa) == NFIT_SPA_PM) {
2792                 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
2793                 if (rc) {
2794                         dev_warn(acpi_desc->dev,
2795                                 "failed to insert pmem resource to iomem: %d\n",
2796                                 rc);
2797                         goto out;
2798                 }
2799
2800                 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
2801                                 ndr_desc);
2802                 if (!nfit_spa->nd_region)
2803                         rc = -ENOMEM;
2804         } else if (nfit_spa_is_volatile(spa)) {
2805                 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
2806                                 ndr_desc);
2807                 if (!nfit_spa->nd_region)
2808                         rc = -ENOMEM;
2809         } else if (nfit_spa_is_virtual(spa)) {
2810                 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
2811                                 ndr_desc);
2812                 if (!nfit_spa->nd_region)
2813                         rc = -ENOMEM;
2814         }
2815
2816  out:
2817         if (rc)
2818                 dev_err(acpi_desc->dev, "failed to register spa range %d\n",
2819                                 nfit_spa->spa->range_index);
2820         return rc;
2821 }
2822
2823 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc)
2824 {
2825         struct device *dev = acpi_desc->dev;
2826         struct nd_cmd_ars_status *ars_status;
2827
2828         if (acpi_desc->ars_status) {
2829                 memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
2830                 return 0;
2831         }
2832
2833         ars_status = devm_kzalloc(dev, acpi_desc->max_ars, GFP_KERNEL);
2834         if (!ars_status)
2835                 return -ENOMEM;
2836         acpi_desc->ars_status = ars_status;
2837         return 0;
2838 }
2839
2840 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc)
2841 {
2842         int rc;
2843
2844         if (ars_status_alloc(acpi_desc))
2845                 return -ENOMEM;
2846
2847         rc = ars_get_status(acpi_desc);
2848
2849         if (rc < 0 && rc != -ENOSPC)
2850                 return rc;
2851
2852         if (ars_status_process_records(acpi_desc))
2853                 return -ENOMEM;
2854
2855         return 0;
2856 }
2857
2858 static int ars_register(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa,
2859                 int *query_rc)
2860 {
2861         int rc = *query_rc;
2862
2863         if (no_init_ars)
2864                 return acpi_nfit_register_region(acpi_desc, nfit_spa);
2865
2866         set_bit(ARS_REQ, &nfit_spa->ars_state);
2867         set_bit(ARS_SHORT, &nfit_spa->ars_state);
2868
2869         switch (rc) {
2870         case 0:
2871         case -EAGAIN:
2872                 rc = ars_start(acpi_desc, nfit_spa);
2873                 if (rc == -EBUSY) {
2874                         *query_rc = rc;
2875                         break;
2876                 } else if (rc == 0) {
2877                         rc = acpi_nfit_query_poison(acpi_desc);
2878                 } else {
2879                         set_bit(ARS_FAILED, &nfit_spa->ars_state);
2880                         break;
2881                 }
2882                 if (rc == -EAGAIN)
2883                         clear_bit(ARS_SHORT, &nfit_spa->ars_state);
2884                 else if (rc == 0)
2885                         ars_complete(acpi_desc, nfit_spa);
2886                 break;
2887         case -EBUSY:
2888         case -ENOSPC:
2889                 break;
2890         default:
2891                 set_bit(ARS_FAILED, &nfit_spa->ars_state);
2892                 break;
2893         }
2894
2895         if (test_and_clear_bit(ARS_DONE, &nfit_spa->ars_state))
2896                 set_bit(ARS_REQ, &nfit_spa->ars_state);
2897
2898         return acpi_nfit_register_region(acpi_desc, nfit_spa);
2899 }
2900
2901 static void ars_complete_all(struct acpi_nfit_desc *acpi_desc)
2902 {
2903         struct nfit_spa *nfit_spa;
2904
2905         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
2906                 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
2907                         continue;
2908                 ars_complete(acpi_desc, nfit_spa);
2909         }
2910 }
2911
2912 static unsigned int __acpi_nfit_scrub(struct acpi_nfit_desc *acpi_desc,
2913                 int query_rc)
2914 {
2915         unsigned int tmo = acpi_desc->scrub_tmo;
2916         struct device *dev = acpi_desc->dev;
2917         struct nfit_spa *nfit_spa;
2918
2919         if (acpi_desc->cancel)
2920                 return 0;
2921
2922         if (query_rc == -EBUSY) {
2923                 dev_dbg(dev, "ARS: ARS busy\n");
2924                 return min(30U * 60U, tmo * 2);
2925         }
2926         if (query_rc == -ENOSPC) {
2927                 dev_dbg(dev, "ARS: ARS continue\n");
2928                 ars_continue(acpi_desc);
2929                 return 1;
2930         }
2931         if (query_rc && query_rc != -EAGAIN) {
2932                 unsigned long long addr, end;
2933
2934                 addr = acpi_desc->ars_status->address;
2935                 end = addr + acpi_desc->ars_status->length;
2936                 dev_dbg(dev, "ARS: %llx-%llx failed (%d)\n", addr, end,
2937                                 query_rc);
2938         }
2939
2940         ars_complete_all(acpi_desc);
2941         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
2942                 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
2943                         continue;
2944                 if (test_bit(ARS_REQ, &nfit_spa->ars_state)) {
2945                         int rc = ars_start(acpi_desc, nfit_spa);
2946
2947                         clear_bit(ARS_DONE, &nfit_spa->ars_state);
2948                         dev = nd_region_dev(nfit_spa->nd_region);
2949                         dev_dbg(dev, "ARS: range %d ARS start (%d)\n",
2950                                         nfit_spa->spa->range_index, rc);
2951                         if (rc == 0 || rc == -EBUSY)
2952                                 return 1;
2953                         dev_err(dev, "ARS: range %d ARS failed (%d)\n",
2954                                         nfit_spa->spa->range_index, rc);
2955                         set_bit(ARS_FAILED, &nfit_spa->ars_state);
2956                 }
2957         }
2958         return 0;
2959 }
2960
2961 static void __sched_ars(struct acpi_nfit_desc *acpi_desc, unsigned int tmo)
2962 {
2963         lockdep_assert_held(&acpi_desc->init_mutex);
2964
2965         acpi_desc->scrub_busy = 1;
2966         /* note this should only be set from within the workqueue */
2967         if (tmo)
2968                 acpi_desc->scrub_tmo = tmo;
2969         queue_delayed_work(nfit_wq, &acpi_desc->dwork, tmo * HZ);
2970 }
2971
2972 static void sched_ars(struct acpi_nfit_desc *acpi_desc)
2973 {
2974         __sched_ars(acpi_desc, 0);
2975 }
2976
2977 static void notify_ars_done(struct acpi_nfit_desc *acpi_desc)
2978 {
2979         lockdep_assert_held(&acpi_desc->init_mutex);
2980
2981         acpi_desc->scrub_busy = 0;
2982         acpi_desc->scrub_count++;
2983         if (acpi_desc->scrub_count_state)
2984                 sysfs_notify_dirent(acpi_desc->scrub_count_state);
2985 }
2986
2987 static void acpi_nfit_scrub(struct work_struct *work)
2988 {
2989         struct acpi_nfit_desc *acpi_desc;
2990         unsigned int tmo;
2991         int query_rc;
2992
2993         acpi_desc = container_of(work, typeof(*acpi_desc), dwork.work);
2994         mutex_lock(&acpi_desc->init_mutex);
2995         query_rc = acpi_nfit_query_poison(acpi_desc);
2996         tmo = __acpi_nfit_scrub(acpi_desc, query_rc);
2997         if (tmo)
2998                 __sched_ars(acpi_desc, tmo);
2999         else
3000                 notify_ars_done(acpi_desc);
3001         memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3002         mutex_unlock(&acpi_desc->init_mutex);
3003 }
3004
3005 static void acpi_nfit_init_ars(struct acpi_nfit_desc *acpi_desc,
3006                 struct nfit_spa *nfit_spa)
3007 {
3008         int type = nfit_spa_type(nfit_spa->spa);
3009         struct nd_cmd_ars_cap ars_cap;
3010         int rc;
3011
3012         memset(&ars_cap, 0, sizeof(ars_cap));
3013         rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
3014         if (rc < 0)
3015                 return;
3016         /* check that the supported scrub types match the spa type */
3017         if (type == NFIT_SPA_VOLATILE && ((ars_cap.status >> 16)
3018                                 & ND_ARS_VOLATILE) == 0)
3019                 return;
3020         if (type == NFIT_SPA_PM && ((ars_cap.status >> 16)
3021                                 & ND_ARS_PERSISTENT) == 0)
3022                 return;
3023
3024         nfit_spa->max_ars = ars_cap.max_ars_out;
3025         nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
3026         acpi_desc->max_ars = max(nfit_spa->max_ars, acpi_desc->max_ars);
3027         clear_bit(ARS_FAILED, &nfit_spa->ars_state);
3028         set_bit(ARS_REQ, &nfit_spa->ars_state);
3029 }
3030
3031 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
3032 {
3033         struct nfit_spa *nfit_spa;
3034         int rc, query_rc;
3035
3036         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3037                 set_bit(ARS_FAILED, &nfit_spa->ars_state);
3038                 switch (nfit_spa_type(nfit_spa->spa)) {
3039                 case NFIT_SPA_VOLATILE:
3040                 case NFIT_SPA_PM:
3041                         acpi_nfit_init_ars(acpi_desc, nfit_spa);
3042                         break;
3043                 }
3044         }
3045
3046         /*
3047          * Reap any results that might be pending before starting new
3048          * short requests.
3049          */
3050         query_rc = acpi_nfit_query_poison(acpi_desc);
3051         if (query_rc == 0)
3052                 ars_complete_all(acpi_desc);
3053
3054         list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
3055                 switch (nfit_spa_type(nfit_spa->spa)) {
3056                 case NFIT_SPA_VOLATILE:
3057                 case NFIT_SPA_PM:
3058                         /* register regions and kick off initial ARS run */
3059                         rc = ars_register(acpi_desc, nfit_spa, &query_rc);
3060                         if (rc)
3061                                 return rc;
3062                         break;
3063                 case NFIT_SPA_BDW:
3064                         /* nothing to register */
3065                         break;
3066                 case NFIT_SPA_DCR:
3067                 case NFIT_SPA_VDISK:
3068                 case NFIT_SPA_VCD:
3069                 case NFIT_SPA_PDISK:
3070                 case NFIT_SPA_PCD:
3071                         /* register known regions that don't support ARS */
3072                         rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
3073                         if (rc)
3074                                 return rc;
3075                         break;
3076                 default:
3077                         /* don't register unknown regions */
3078                         break;
3079                 }
3080
3081         sched_ars(acpi_desc);
3082         return 0;
3083 }
3084
3085 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
3086                 struct nfit_table_prev *prev)
3087 {
3088         struct device *dev = acpi_desc->dev;
3089
3090         if (!list_empty(&prev->spas) ||
3091                         !list_empty(&prev->memdevs) ||
3092                         !list_empty(&prev->dcrs) ||
3093                         !list_empty(&prev->bdws) ||
3094                         !list_empty(&prev->idts) ||
3095                         !list_empty(&prev->flushes)) {
3096                 dev_err(dev, "new nfit deletes entries (unsupported)\n");
3097                 return -ENXIO;
3098         }
3099         return 0;
3100 }
3101
3102 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc)
3103 {
3104         struct device *dev = acpi_desc->dev;
3105         struct kernfs_node *nfit;
3106         struct device *bus_dev;
3107
3108         if (!ars_supported(acpi_desc->nvdimm_bus))
3109                 return 0;
3110
3111         bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3112         nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit");
3113         if (!nfit) {
3114                 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n");
3115                 return -ENODEV;
3116         }
3117         acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub");
3118         sysfs_put(nfit);
3119         if (!acpi_desc->scrub_count_state) {
3120                 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n");
3121                 return -ENODEV;
3122         }
3123
3124         return 0;
3125 }
3126
3127 static void acpi_nfit_unregister(void *data)
3128 {
3129         struct acpi_nfit_desc *acpi_desc = data;
3130
3131         nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
3132 }
3133
3134 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
3135 {
3136         struct device *dev = acpi_desc->dev;
3137         struct nfit_table_prev prev;
3138         const void *end;
3139         int rc;
3140
3141         if (!acpi_desc->nvdimm_bus) {
3142                 acpi_nfit_init_dsms(acpi_desc);
3143
3144                 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev,
3145                                 &acpi_desc->nd_desc);
3146                 if (!acpi_desc->nvdimm_bus)
3147                         return -ENOMEM;
3148
3149                 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister,
3150                                 acpi_desc);
3151                 if (rc)
3152                         return rc;
3153
3154                 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc);
3155                 if (rc)
3156                         return rc;
3157
3158                 /* register this acpi_desc for mce notifications */
3159                 mutex_lock(&acpi_desc_lock);
3160                 list_add_tail(&acpi_desc->list, &acpi_descs);
3161                 mutex_unlock(&acpi_desc_lock);
3162         }
3163
3164         mutex_lock(&acpi_desc->init_mutex);
3165
3166         INIT_LIST_HEAD(&prev.spas);
3167         INIT_LIST_HEAD(&prev.memdevs);
3168         INIT_LIST_HEAD(&prev.dcrs);
3169         INIT_LIST_HEAD(&prev.bdws);
3170         INIT_LIST_HEAD(&prev.idts);
3171         INIT_LIST_HEAD(&prev.flushes);
3172
3173         list_cut_position(&prev.spas, &acpi_desc->spas,
3174                                 acpi_desc->spas.prev);
3175         list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
3176                                 acpi_desc->memdevs.prev);
3177         list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
3178                                 acpi_desc->dcrs.prev);
3179         list_cut_position(&prev.bdws, &acpi_desc->bdws,
3180                                 acpi_desc->bdws.prev);
3181         list_cut_position(&prev.idts, &acpi_desc->idts,
3182                                 acpi_desc->idts.prev);
3183         list_cut_position(&prev.flushes, &acpi_desc->flushes,
3184                                 acpi_desc->flushes.prev);
3185
3186         end = data + sz;
3187         while (!IS_ERR_OR_NULL(data))
3188                 data = add_table(acpi_desc, &prev, data, end);
3189
3190         if (IS_ERR(data)) {
3191                 dev_dbg(dev, "nfit table parsing error: %ld\n", PTR_ERR(data));
3192                 rc = PTR_ERR(data);
3193                 goto out_unlock;
3194         }
3195
3196         rc = acpi_nfit_check_deletions(acpi_desc, &prev);
3197         if (rc)
3198                 goto out_unlock;
3199
3200         rc = nfit_mem_init(acpi_desc);
3201         if (rc)
3202                 goto out_unlock;
3203
3204         rc = acpi_nfit_register_dimms(acpi_desc);
3205         if (rc)
3206                 goto out_unlock;
3207
3208         rc = acpi_nfit_register_regions(acpi_desc);
3209
3210  out_unlock:
3211         mutex_unlock(&acpi_desc->init_mutex);
3212         return rc;
3213 }
3214 EXPORT_SYMBOL_GPL(acpi_nfit_init);
3215
3216 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
3217 {
3218         struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
3219         struct device *dev = acpi_desc->dev;
3220
3221         /* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
3222         device_lock(dev);
3223         device_unlock(dev);
3224
3225         /* Bounce the init_mutex to complete initial registration */
3226         mutex_lock(&acpi_desc->init_mutex);
3227         mutex_unlock(&acpi_desc->init_mutex);
3228
3229         return 0;
3230 }
3231
3232 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3233                 struct nvdimm *nvdimm, unsigned int cmd)
3234 {
3235         struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
3236
3237         if (nvdimm)
3238                 return 0;
3239         if (cmd != ND_CMD_ARS_START)
3240                 return 0;
3241
3242         /*
3243          * The kernel and userspace may race to initiate a scrub, but
3244          * the scrub thread is prepared to lose that initial race.  It
3245          * just needs guarantees that any ars it initiates are not
3246          * interrupted by any intervening start reqeusts from userspace.
3247          */
3248         if (work_busy(&acpi_desc->dwork.work))
3249                 return -EBUSY;
3250
3251         return 0;
3252 }
3253
3254 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc, unsigned long flags)
3255 {
3256         struct device *dev = acpi_desc->dev;
3257         int scheduled = 0, busy = 0;
3258         struct nfit_spa *nfit_spa;
3259
3260         mutex_lock(&acpi_desc->init_mutex);
3261         if (acpi_desc->cancel) {
3262                 mutex_unlock(&acpi_desc->init_mutex);
3263                 return 0;
3264         }
3265
3266         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3267                 int type = nfit_spa_type(nfit_spa->spa);
3268
3269                 if (type != NFIT_SPA_PM && type != NFIT_SPA_VOLATILE)
3270                         continue;
3271                 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3272                         continue;
3273
3274                 if (test_and_set_bit(ARS_REQ, &nfit_spa->ars_state)) {
3275                         busy++;
3276                         set_bit(ARS_REQ_REDO, &nfit_spa->ars_state);
3277                 } else {
3278                         if (test_bit(ARS_SHORT, &flags))
3279                                 set_bit(ARS_SHORT, &nfit_spa->ars_state);
3280                         scheduled++;
3281                 }
3282         }
3283         if (scheduled) {
3284                 sched_ars(acpi_desc);
3285                 dev_dbg(dev, "ars_scan triggered\n");
3286         }
3287         mutex_unlock(&acpi_desc->init_mutex);
3288
3289         if (scheduled)
3290                 return 0;
3291         if (busy)
3292                 return -EBUSY;
3293         return -ENOTTY;
3294 }
3295
3296 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
3297 {
3298         struct nvdimm_bus_descriptor *nd_desc;
3299
3300         dev_set_drvdata(dev, acpi_desc);
3301         acpi_desc->dev = dev;
3302         acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
3303         nd_desc = &acpi_desc->nd_desc;
3304         nd_desc->provider_name = "ACPI.NFIT";
3305         nd_desc->module = THIS_MODULE;
3306         nd_desc->ndctl = acpi_nfit_ctl;
3307         nd_desc->flush_probe = acpi_nfit_flush_probe;
3308         nd_desc->clear_to_send = acpi_nfit_clear_to_send;
3309         nd_desc->attr_groups = acpi_nfit_attribute_groups;
3310
3311         INIT_LIST_HEAD(&acpi_desc->spas);
3312         INIT_LIST_HEAD(&acpi_desc->dcrs);
3313         INIT_LIST_HEAD(&acpi_desc->bdws);
3314         INIT_LIST_HEAD(&acpi_desc->idts);
3315         INIT_LIST_HEAD(&acpi_desc->flushes);
3316         INIT_LIST_HEAD(&acpi_desc->memdevs);
3317         INIT_LIST_HEAD(&acpi_desc->dimms);
3318         INIT_LIST_HEAD(&acpi_desc->list);
3319         mutex_init(&acpi_desc->init_mutex);
3320         acpi_desc->scrub_tmo = 1;
3321         INIT_DELAYED_WORK(&acpi_desc->dwork, acpi_nfit_scrub);
3322 }
3323 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
3324
3325 static void acpi_nfit_put_table(void *table)
3326 {
3327         acpi_put_table(table);
3328 }
3329
3330 void acpi_nfit_shutdown(void *data)
3331 {
3332         struct acpi_nfit_desc *acpi_desc = data;
3333         struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3334
3335         /*
3336          * Destruct under acpi_desc_lock so that nfit_handle_mce does not
3337          * race teardown
3338          */
3339         mutex_lock(&acpi_desc_lock);
3340         list_del(&acpi_desc->list);
3341         mutex_unlock(&acpi_desc_lock);
3342
3343         mutex_lock(&acpi_desc->init_mutex);
3344         acpi_desc->cancel = 1;
3345         cancel_delayed_work_sync(&acpi_desc->dwork);
3346         mutex_unlock(&acpi_desc->init_mutex);
3347
3348         /*
3349          * Bounce the nvdimm bus lock to make sure any in-flight
3350          * acpi_nfit_ars_rescan() submissions have had a chance to
3351          * either submit or see ->cancel set.
3352          */
3353         device_lock(bus_dev);
3354         device_unlock(bus_dev);
3355
3356         flush_workqueue(nfit_wq);
3357 }
3358 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown);
3359
3360 static int acpi_nfit_add(struct acpi_device *adev)
3361 {
3362         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3363         struct acpi_nfit_desc *acpi_desc;
3364         struct device *dev = &adev->dev;
3365         struct acpi_table_header *tbl;
3366         acpi_status status = AE_OK;
3367         acpi_size sz;
3368         int rc = 0;
3369
3370         status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl);
3371         if (ACPI_FAILURE(status)) {
3372                 /* This is ok, we could have an nvdimm hotplugged later */
3373                 dev_dbg(dev, "failed to find NFIT at startup\n");
3374                 return 0;
3375         }
3376
3377         rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl);
3378         if (rc)
3379                 return rc;
3380         sz = tbl->length;
3381
3382         acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3383         if (!acpi_desc)
3384                 return -ENOMEM;
3385         acpi_nfit_desc_init(acpi_desc, &adev->dev);
3386
3387         /* Save the acpi header for exporting the revision via sysfs */
3388         acpi_desc->acpi_header = *tbl;
3389
3390         /* Evaluate _FIT and override with that if present */
3391         status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
3392         if (ACPI_SUCCESS(status) && buf.length > 0) {
3393                 union acpi_object *obj = buf.pointer;
3394
3395                 if (obj->type == ACPI_TYPE_BUFFER)
3396                         rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3397                                         obj->buffer.length);
3398                 else
3399                         dev_dbg(dev, "invalid type %d, ignoring _FIT\n",
3400                                 (int) obj->type);
3401                 kfree(buf.pointer);
3402         } else
3403                 /* skip over the lead-in header table */
3404                 rc = acpi_nfit_init(acpi_desc, (void *) tbl
3405                                 + sizeof(struct acpi_table_nfit),
3406                                 sz - sizeof(struct acpi_table_nfit));
3407
3408         if (rc)
3409                 return rc;
3410         return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc);
3411 }
3412
3413 static int acpi_nfit_remove(struct acpi_device *adev)
3414 {
3415         /* see acpi_nfit_unregister */
3416         return 0;
3417 }
3418
3419 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle)
3420 {
3421         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3422         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3423         union acpi_object *obj;
3424         acpi_status status;
3425         int ret;
3426
3427         if (!dev->driver) {
3428                 /* dev->driver may be null if we're being removed */
3429                 dev_dbg(dev, "no driver found for dev\n");
3430                 return;
3431         }
3432
3433         if (!acpi_desc) {
3434                 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3435                 if (!acpi_desc)
3436                         return;
3437                 acpi_nfit_desc_init(acpi_desc, dev);
3438         } else {
3439                 /*
3440                  * Finish previous registration before considering new
3441                  * regions.
3442                  */
3443                 flush_workqueue(nfit_wq);
3444         }
3445
3446         /* Evaluate _FIT */
3447         status = acpi_evaluate_object(handle, "_FIT", NULL, &buf);
3448         if (ACPI_FAILURE(status)) {
3449                 dev_err(dev, "failed to evaluate _FIT\n");
3450                 return;
3451         }
3452
3453         obj = buf.pointer;
3454         if (obj->type == ACPI_TYPE_BUFFER) {
3455                 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3456                                 obj->buffer.length);
3457                 if (ret)
3458                         dev_err(dev, "failed to merge updated NFIT\n");
3459         } else
3460                 dev_err(dev, "Invalid _FIT\n");
3461         kfree(buf.pointer);
3462 }
3463
3464 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle)
3465 {
3466         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3467         unsigned long flags = (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) ?
3468                         0 : 1 << ARS_SHORT;
3469
3470         acpi_nfit_ars_rescan(acpi_desc, flags);
3471 }
3472
3473 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event)
3474 {
3475         dev_dbg(dev, "event: 0x%x\n", event);
3476
3477         switch (event) {
3478         case NFIT_NOTIFY_UPDATE:
3479                 return acpi_nfit_update_notify(dev, handle);
3480         case NFIT_NOTIFY_UC_MEMORY_ERROR:
3481                 return acpi_nfit_uc_error_notify(dev, handle);
3482         default:
3483                 return;
3484         }
3485 }
3486 EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
3487
3488 static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
3489 {
3490         device_lock(&adev->dev);
3491         __acpi_nfit_notify(&adev->dev, adev->handle, event);
3492         device_unlock(&adev->dev);
3493 }
3494
3495 static const struct acpi_device_id acpi_nfit_ids[] = {
3496         { "ACPI0012", 0 },
3497         { "", 0 },
3498 };
3499 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
3500
3501 static struct acpi_driver acpi_nfit_driver = {
3502         .name = KBUILD_MODNAME,
3503         .ids = acpi_nfit_ids,
3504         .ops = {
3505                 .add = acpi_nfit_add,
3506                 .remove = acpi_nfit_remove,
3507                 .notify = acpi_nfit_notify,
3508         },
3509 };
3510
3511 static __init int nfit_init(void)
3512 {
3513         int ret;
3514
3515         BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
3516         BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
3517         BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
3518         BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
3519         BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
3520         BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
3521         BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
3522         BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16);
3523
3524         guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]);
3525         guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]);
3526         guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]);
3527         guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]);
3528         guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]);
3529         guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]);
3530         guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]);
3531         guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]);
3532         guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]);
3533         guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]);
3534         guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
3535         guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
3536         guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]);
3537
3538         nfit_wq = create_singlethread_workqueue("nfit");
3539         if (!nfit_wq)
3540                 return -ENOMEM;
3541
3542         nfit_mce_register();
3543         ret = acpi_bus_register_driver(&acpi_nfit_driver);
3544         if (ret) {
3545                 nfit_mce_unregister();
3546                 destroy_workqueue(nfit_wq);
3547         }
3548
3549         return ret;
3550
3551 }
3552
3553 static __exit void nfit_exit(void)
3554 {
3555         nfit_mce_unregister();
3556         acpi_bus_unregister_driver(&acpi_nfit_driver);
3557         destroy_workqueue(nfit_wq);
3558         WARN_ON(!list_empty(&acpi_descs));
3559 }
3560
3561 module_init(nfit_init);
3562 module_exit(nfit_exit);
3563 MODULE_LICENSE("GPL v2");
3564 MODULE_AUTHOR("Intel Corporation");