drivers/iommu/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #define pr_fmt(fmt)     "DMAR: " fmt
  30
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/iova.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/timer.h>
  36 #include <linux/irq.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/slab.h>
  41 #include <linux/iommu.h>
  42 #include <linux/limits.h>
  43 #include <asm/irq_remapping.h>
  44 #include <asm/iommu_table.h>
  45
  46 #include "irq_remapping.h"
  47
  48 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  49 struct dmar_res_callback {
  50         dmar_res_handler_t      cb[ACPI_DMAR_TYPE_RESERVED];
  51         void                    *arg[ACPI_DMAR_TYPE_RESERVED];
  52         bool                    ignore_unhandled;
  53         bool                    print_entry;
  54 };
  55
  56 /*
  57  * Assumptions:
  58  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  59  *    before IO devices managed by that unit.
  60  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  61  *    after IO devices managed by that unit.
  62  * 3) Hotplug events are rare.
  63  *
  64  * Locking rules for DMA and interrupt remapping related global data structures:
  65  * 1) Use dmar_global_lock in process context
  66  * 2) Use RCU in interrupt context
  67  */
  68 DECLARE_RWSEM(dmar_global_lock);
  69 LIST_HEAD(dmar_drhd_units);
  70
  71 struct acpi_table_header * __initdata dmar_tbl;
  72 static int dmar_dev_scope_status = 1;
  73 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
  74
  75 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  76 static void free_iommu(struct intel_iommu *iommu);
  77
  78 extern const struct iommu_ops intel_iommu_ops;
  79
  80 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  81 {
  82         /*
  83          * add INCLUDE_ALL at the tail, so scan the list will find it at
  84          * the very end.
  85          */
  86         if (drhd->include_all)
  87                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  88         else
  89                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  90 }
  91
  92 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  93 {
  94         struct acpi_dmar_device_scope *scope;
  95
  96         *cnt = 0;
  97         while (start < end) {
  98                 scope = start;
  99                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
 100                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 101                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 102                         (*cnt)++;
 103                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
 104                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
 105                         pr_warn("Unsupported device scope\n");
 106                 }
 107                 start += scope->length;
 108         }
 109         if (*cnt == 0)
 110                 return NULL;
 111
 112         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 113 }
 114
 115 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 116 {
 117         int i;
 118         struct device *tmp_dev;
 119
 120         if (*devices && *cnt) {
 121                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 122                         put_device(tmp_dev);
 123                 kfree(*devices);
 124         }
 125
 126         *devices = NULL;
 127         *cnt = 0;
 128 }
 129
 130 /* Optimize out kzalloc()/kfree() for normal cases */
 131 static char dmar_pci_notify_info_buf[64];
 132
 133 static struct dmar_pci_notify_info *
 134 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 135 {
 136         int level = 0;
 137         size_t size;
 138         struct pci_dev *tmp;
 139         struct dmar_pci_notify_info *info;
 140
 141         BUG_ON(dev->is_virtfn);
 142
 143         /*
 144          * Ignore devices that have a domain number higher than what can
 145          * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
 146          */
 147         if (pci_domain_nr(dev->bus) > U16_MAX)
 148                 return NULL;
 149
 150         /* Only generate path[] for device addition event */
 151         if (event == BUS_NOTIFY_ADD_DEVICE)
 152                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 153                         level++;
 154
 155         size = sizeof(*info) + level * sizeof(info->path[0]);
 156         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 157                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 158         } else {
 159                 info = kzalloc(size, GFP_KERNEL);
 160                 if (!info) {
 161                         pr_warn("Out of memory when allocating notify_info "
 162                                 "for %s.\n", pci_name(dev));
 163                         if (dmar_dev_scope_status == 0)
 164                                 dmar_dev_scope_status = -ENOMEM;
 165                         return NULL;
 166                 }
 167         }
 168
 169         info->event = event;
 170         info->dev = dev;
 171         info->seg = pci_domain_nr(dev->bus);
 172         info->level = level;
 173         if (event == BUS_NOTIFY_ADD_DEVICE) {
 174                 for (tmp = dev; tmp; tmp = tmp->bus->self) {
 175                         level--;
 176                         info->path[level].bus = tmp->bus->number;
 177                         info->path[level].device = PCI_SLOT(tmp->devfn);
 178                         info->path[level].function = PCI_FUNC(tmp->devfn);
 179                         if (pci_is_root_bus(tmp->bus))
 180                                 info->bus = tmp->bus->number;
 181                 }
 182         }
 183
 184         return info;
 185 }
 186
 187 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 188 {
 189         if ((void *)info != dmar_pci_notify_info_buf)
 190                 kfree(info);
 191 }
 192
 193 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 194                                 struct acpi_dmar_pci_path *path, int count)
 195 {
 196         int i;
 197
 198         if (info->bus != bus)
 199                 goto fallback;
 200         if (info->level != count)
 201                 goto fallback;
 202
 203         for (i = 0; i < count; i++) {
 204                 if (path[i].device != info->path[i].device ||
 205                     path[i].function != info->path[i].function)
 206                         goto fallback;
 207         }
 208
 209         return true;
 210
 211 fallback:
 212
 213         if (count != 1)
 214                 return false;
 215
 216         i = info->level - 1;
 217         if (bus              == info->path[i].bus &&
 218             path[0].device   == info->path[i].device &&
 219             path[0].function == info->path[i].function) {
 220                 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 221                         bus, path[0].device, path[0].function);
 222                 return true;
 223         }
 224
 225         return false;
 226 }
 227
 228 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 229 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 230                           void *start, void*end, u16 segment,
 231                           struct dmar_dev_scope *devices,
 232                           int devices_cnt)
 233 {
 234         int i, level;
 235         struct device *tmp, *dev = &info->dev->dev;
 236         struct acpi_dmar_device_scope *scope;
 237         struct acpi_dmar_pci_path *path;
 238
 239         if (segment != info->seg)
 240                 return 0;
 241
 242         for (; start < end; start += scope->length) {
 243                 scope = start;
 244                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 245                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 246                         continue;
 247
 248                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 249                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 250                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 251                         continue;
 252
 253                 /*
 254                  * We expect devices with endpoint scope to have normal PCI
 255                  * headers, and devices with bridge scope to have bridge PCI
 256                  * headers.  However PCI NTB devices may be listed in the
 257                  * DMAR table with bridge scope, even though they have a
 258                  * normal PCI header.  NTB devices are identified by class
 259                  * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 260                  * for this special case.
 261                  */
 262                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 263                      info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 264                     (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 265                      (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 266                       info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
 267                         pr_warn("Device scope type does not match for %s\n",
 268                                 pci_name(info->dev));
 269                         return -EINVAL;
 270                 }
 271
 272                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 273                         if (tmp == NULL) {
 274                                 devices[i].bus = info->dev->bus->number;
 275                                 devices[i].devfn = info->dev->devfn;
 276                                 rcu_assign_pointer(devices[i].dev,
 277                                                    get_device(dev));
 278                                 return 1;
 279                         }
 280                 BUG_ON(i >= devices_cnt);
 281         }
 282
 283         return 0;
 284 }
 285
 286 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 287                           struct dmar_dev_scope *devices, int count)
 288 {
 289         int index;
 290         struct device *tmp;
 291
 292         if (info->seg != segment)
 293                 return 0;
 294
 295         for_each_active_dev_scope(devices, count, index, tmp)
 296                 if (tmp == &info->dev->dev) {
 297                         RCU_INIT_POINTER(devices[index].dev, NULL);
 298                         synchronize_rcu();
 299                         put_device(tmp);
 300                         return 1;
 301                 }
 302
 303         return 0;
 304 }
 305
 306 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 307 {
 308         int ret = 0;
 309         struct dmar_drhd_unit *dmaru;
 310         struct acpi_dmar_hardware_unit *drhd;
 311
 312         for_each_drhd_unit(dmaru) {
 313                 if (dmaru->include_all)
 314                         continue;
 315
 316                 drhd = container_of(dmaru->hdr,
 317                                     struct acpi_dmar_hardware_unit, header);
 318                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 319                                 ((void *)drhd) + drhd->header.length,
 320                                 dmaru->segment,
 321                                 dmaru->devices, dmaru->devices_cnt);
 322                 if (ret)
 323                         break;
 324         }
 325         if (ret >= 0)
 326                 ret = dmar_iommu_notify_scope_dev(info);
 327         if (ret < 0 && dmar_dev_scope_status == 0)
 328                 dmar_dev_scope_status = ret;
 329
 330         return ret;
 331 }
 332
 333 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 334 {
 335         struct dmar_drhd_unit *dmaru;
 336
 337         for_each_drhd_unit(dmaru)
 338                 if (dmar_remove_dev_scope(info, dmaru->segment,
 339                         dmaru->devices, dmaru->devices_cnt))
 340                         break;
 341         dmar_iommu_notify_scope_dev(info);
 342 }
 343
 344 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 345                                  unsigned long action, void *data)
 346 {
 347         struct pci_dev *pdev = to_pci_dev(data);
 348         struct dmar_pci_notify_info *info;
 349
 350         /* Only care about add/remove events for physical functions.
 351          * For VFs we actually do the lookup based on the corresponding
 352          * PF in device_to_iommu() anyway. */
 353         if (pdev->is_virtfn)
 354                 return NOTIFY_DONE;
 355         if (action != BUS_NOTIFY_ADD_DEVICE &&
 356             action != BUS_NOTIFY_REMOVED_DEVICE)
 357                 return NOTIFY_DONE;
 358
 359         info = dmar_alloc_pci_notify_info(pdev, action);
 360         if (!info)
 361                 return NOTIFY_DONE;
 362
 363         down_write(&dmar_global_lock);
 364         if (action == BUS_NOTIFY_ADD_DEVICE)
 365                 dmar_pci_bus_add_dev(info);
 366         else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 367                 dmar_pci_bus_del_dev(info);
 368         up_write(&dmar_global_lock);
 369
 370         dmar_free_pci_notify_info(info);
 371
 372         return NOTIFY_OK;
 373 }
 374
 375 static struct notifier_block dmar_pci_bus_nb = {
 376         .notifier_call = dmar_pci_bus_notifier,
 377         .priority = INT_MIN,
 378 };
 379
 380 static struct dmar_drhd_unit *
 381 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 382 {
 383         struct dmar_drhd_unit *dmaru;
 384
 385         list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
 386                 if (dmaru->segment == drhd->segment &&
 387                     dmaru->reg_base_addr == drhd->address)
 388                         return dmaru;
 389
 390         return NULL;
 391 }
 392
 393 /**
 394  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 395  * structure which uniquely represent one DMA remapping hardware unit
 396  * present in the platform
 397  */
 398 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 399 {
 400         struct acpi_dmar_hardware_unit *drhd;
 401         struct dmar_drhd_unit *dmaru;
 402         int ret;
 403
 404         drhd = (struct acpi_dmar_hardware_unit *)header;
 405         dmaru = dmar_find_dmaru(drhd);
 406         if (dmaru)
 407                 goto out;
 408
 409         dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 410         if (!dmaru)
 411                 return -ENOMEM;
 412
 413         /*
 414          * If header is allocated from slab by ACPI _DSM method, we need to
 415          * copy the content because the memory buffer will be freed on return.
 416          */
 417         dmaru->hdr = (void *)(dmaru + 1);
 418         memcpy(dmaru->hdr, header, header->length);
 419         dmaru->reg_base_addr = drhd->address;
 420         dmaru->segment = drhd->segment;
 421         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 422         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 423                                               ((void *)drhd) + drhd->header.length,
 424                                               &dmaru->devices_cnt);
 425         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 426                 kfree(dmaru);
 427                 return -ENOMEM;
 428         }
 429
 430         ret = alloc_iommu(dmaru);
 431         if (ret) {
 432                 dmar_free_dev_scope(&dmaru->devices,
 433                                     &dmaru->devices_cnt);
 434                 kfree(dmaru);
 435                 return ret;
 436         }
 437         dmar_register_drhd_unit(dmaru);
 438
 439 out:
 440         if (arg)
 441                 (*(int *)arg)++;
 442
 443         return 0;
 444 }
 445
 446 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 447 {
 448         if (dmaru->devices && dmaru->devices_cnt)
 449                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 450         if (dmaru->iommu)
 451                 free_iommu(dmaru->iommu);
 452         kfree(dmaru);
 453 }
 454
 455 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 456                                       void *arg)
 457 {
 458         struct acpi_dmar_andd *andd = (void *)header;
 459
 460         /* Check for NUL termination within the designated length */
 461         if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 462                 pr_warn(FW_BUG
 463                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 464                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 465                            dmi_get_system_info(DMI_BIOS_VENDOR),
 466                            dmi_get_system_info(DMI_BIOS_VERSION),
 467                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 468                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 469                 return -EINVAL;
 470         }
 471         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 472                 andd->device_name);
 473
 474         return 0;
 475 }
 476
 477 #ifdef CONFIG_ACPI_NUMA
 478 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 479 {
 480         struct acpi_dmar_rhsa *rhsa;
 481         struct dmar_drhd_unit *drhd;
 482
 483         rhsa = (struct acpi_dmar_rhsa *)header;
 484         for_each_drhd_unit(drhd) {
 485                 if (drhd->reg_base_addr == rhsa->base_address) {
 486                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 487
 488                         if (!node_online(node))
 489                                 node = -1;
 490                         drhd->iommu->node = node;
 491                         return 0;
 492                 }
 493         }
 494         pr_warn(FW_BUG
 495                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 496                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 497                 rhsa->base_address,
 498                 dmi_get_system_info(DMI_BIOS_VENDOR),
 499                 dmi_get_system_info(DMI_BIOS_VERSION),
 500                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 501         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 502
 503         return 0;
 504 }
 505 #else
 506 #define dmar_parse_one_rhsa             dmar_res_noop
 507 #endif
 508
 509 static void
 510 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 511 {
 512         struct acpi_dmar_hardware_unit *drhd;
 513         struct acpi_dmar_reserved_memory *rmrr;
 514         struct acpi_dmar_atsr *atsr;
 515         struct acpi_dmar_rhsa *rhsa;
 516
 517         switch (header->type) {
 518         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 519                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 520                                     header);
 521                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 522                         (unsigned long long)drhd->address, drhd->flags);
 523                 break;
 524         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 525                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 526                                     header);
 527                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 528                         (unsigned long long)rmrr->base_address,
 529                         (unsigned long long)rmrr->end_address);
 530                 break;
 531         case ACPI_DMAR_TYPE_ROOT_ATS:
 532                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 533                 pr_info("ATSR flags: %#x\n", atsr->flags);
 534                 break;
 535         case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 536                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 537                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 538                        (unsigned long long)rhsa->base_address,
 539                        rhsa->proximity_domain);
 540                 break;
 541         case ACPI_DMAR_TYPE_NAMESPACE:
 542                 /* We don't print this here because we need to sanity-check
 543                    it first. So print it in dmar_parse_one_andd() instead. */
 544                 break;
 545         }
 546 }
 547
 548 /**
 549  * dmar_table_detect - checks to see if the platform supports DMAR devices
 550  */
 551 static int __init dmar_table_detect(void)
 552 {
 553         acpi_status status = AE_OK;
 554
 555         /* if we could find DMAR table, then there are DMAR devices */
 556         status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
 557
 558         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 559                 pr_warn("Unable to map DMAR\n");
 560                 status = AE_NOT_FOUND;
 561         }
 562
 563         return ACPI_SUCCESS(status) ? 0 : -ENOENT;
 564 }
 565
 566 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 567                                        size_t len, struct dmar_res_callback *cb)
 568 {
 569         struct acpi_dmar_header *iter, *next;
 570         struct acpi_dmar_header *end = ((void *)start) + len;
 571
 572         for (iter = start; iter < end; iter = next) {
 573                 next = (void *)iter + iter->length;
 574                 if (iter->length == 0) {
 575                         /* Avoid looping forever on bad ACPI tables */
 576                         pr_debug(FW_BUG "Invalid 0-length structure\n");
 577                         break;
 578                 } else if (next > end) {
 579                         /* Avoid passing table end */
 580                         pr_warn(FW_BUG "Record passes table end\n");
 581                         return -EINVAL;
 582                 }
 583
 584                 if (cb->print_entry)
 585                         dmar_table_print_dmar_entry(iter);
 586
 587                 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 588                         /* continue for forward compatibility */
 589                         pr_debug("Unknown DMAR structure type %d\n",
 590                                  iter->type);
 591                 } else if (cb->cb[iter->type]) {
 592                         int ret;
 593
 594                         ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 595                         if (ret)
 596                                 return ret;
 597                 } else if (!cb->ignore_unhandled) {
 598                         pr_warn("No handler for DMAR structure type %d\n",
 599                                 iter->type);
 600                         return -EINVAL;
 601                 }
 602         }
 603
 604         return 0;
 605 }
 606
 607 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 608                                        struct dmar_res_callback *cb)
 609 {
 610         return dmar_walk_remapping_entries((void *)(dmar + 1),
 611                         dmar->header.length - sizeof(*dmar), cb);
 612 }
 613
 614 /**
 615  * parse_dmar_table - parses the DMA reporting table
 616  */
 617 static int __init
 618 parse_dmar_table(void)
 619 {
 620         struct acpi_table_dmar *dmar;
 621         int drhd_count = 0;
 622         int ret;
 623         struct dmar_res_callback cb = {
 624                 .print_entry = true,
 625                 .ignore_unhandled = true,
 626                 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 627                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 628                 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 629                 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 630                 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 631                 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 632         };
 633
 634         /*
 635          * Do it again, earlier dmar_tbl mapping could be mapped with
 636          * fixed map.
 637          */
 638         dmar_table_detect();
 639
 640         /*
 641          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 642          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 643          */
 644         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 645
 646         dmar = (struct acpi_table_dmar *)dmar_tbl;
 647         if (!dmar)
 648                 return -ENODEV;
 649
 650         if (dmar->width < PAGE_SHIFT - 1) {
 651                 pr_warn("Invalid DMAR haw\n");
 652                 return -EINVAL;
 653         }
 654
 655         pr_info("Host address width %d\n", dmar->width + 1);
 656         ret = dmar_walk_dmar_table(dmar, &cb);
 657         if (ret == 0 && drhd_count == 0)
 658                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 659
 660         return ret;
 661 }
 662
 663 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 664                                  int cnt, struct pci_dev *dev)
 665 {
 666         int index;
 667         struct device *tmp;
 668
 669         while (dev) {
 670                 for_each_active_dev_scope(devices, cnt, index, tmp)
 671                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 672                                 return 1;
 673
 674                 /* Check our parent */
 675                 dev = dev->bus->self;
 676         }
 677
 678         return 0;
 679 }
 680
 681 struct dmar_drhd_unit *
 682 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 683 {
 684         struct dmar_drhd_unit *dmaru;
 685         struct acpi_dmar_hardware_unit *drhd;
 686
 687         dev = pci_physfn(dev);
 688
 689         rcu_read_lock();
 690         for_each_drhd_unit(dmaru) {
 691                 drhd = container_of(dmaru->hdr,
 692                                     struct acpi_dmar_hardware_unit,
 693                                     header);
 694
 695                 if (dmaru->include_all &&
 696                     drhd->segment == pci_domain_nr(dev->bus))
 697                         goto out;
 698
 699                 if (dmar_pci_device_match(dmaru->devices,
 700                                           dmaru->devices_cnt, dev))
 701                         goto out;
 702         }
 703         dmaru = NULL;
 704 out:
 705         rcu_read_unlock();
 706
 707         return dmaru;
 708 }
 709
 710 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 711                                               struct acpi_device *adev)
 712 {
 713         struct dmar_drhd_unit *dmaru;
 714         struct acpi_dmar_hardware_unit *drhd;
 715         struct acpi_dmar_device_scope *scope;
 716         struct device *tmp;
 717         int i;
 718         struct acpi_dmar_pci_path *path;
 719
 720         for_each_drhd_unit(dmaru) {
 721                 drhd = container_of(dmaru->hdr,
 722                                     struct acpi_dmar_hardware_unit,
 723                                     header);
 724
 725                 for (scope = (void *)(drhd + 1);
 726                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 727                      scope = ((void *)scope) + scope->length) {
 728                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 729                                 continue;
 730                         if (scope->enumeration_id != device_number)
 731                                 continue;
 732
 733                         path = (void *)(scope + 1);
 734                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 735                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 736                                 scope->bus, path->device, path->function);
 737                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 738                                 if (tmp == NULL) {
 739                                         dmaru->devices[i].bus = scope->bus;
 740                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 741                                                                             path->function);
 742                                         rcu_assign_pointer(dmaru->devices[i].dev,
 743                                                            get_device(&adev->dev));
 744                                         return;
 745                                 }
 746                         BUG_ON(i >= dmaru->devices_cnt);
 747                 }
 748         }
 749         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 750                 device_number, dev_name(&adev->dev));
 751 }
 752
 753 static int __init dmar_acpi_dev_scope_init(void)
 754 {
 755         struct acpi_dmar_andd *andd;
 756
 757         if (dmar_tbl == NULL)
 758                 return -ENODEV;
 759
 760         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 761              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 762              andd = ((void *)andd) + andd->header.length) {
 763                 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 764                         acpi_handle h;
 765                         struct acpi_device *adev;
 766
 767                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 768                                                           andd->device_name,
 769                                                           &h))) {
 770                                 pr_err("Failed to find handle for ACPI object %s\n",
 771                                        andd->device_name);
 772                                 continue;
 773                         }
 774                         if (acpi_bus_get_device(h, &adev)) {
 775                                 pr_err("Failed to get device for ACPI object %s\n",
 776                                        andd->device_name);
 777                                 continue;
 778                         }
 779                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 780                 }
 781         }
 782         return 0;
 783 }
 784
 785 int __init dmar_dev_scope_init(void)
 786 {
 787         struct pci_dev *dev = NULL;
 788         struct dmar_pci_notify_info *info;
 789
 790         if (dmar_dev_scope_status != 1)
 791                 return dmar_dev_scope_status;
 792
 793         if (list_empty(&dmar_drhd_units)) {
 794                 dmar_dev_scope_status = -ENODEV;
 795         } else {
 796                 dmar_dev_scope_status = 0;
 797
 798                 dmar_acpi_dev_scope_init();
 799
 800                 for_each_pci_dev(dev) {
 801                         if (dev->is_virtfn)
 802                                 continue;
 803
 804                         info = dmar_alloc_pci_notify_info(dev,
 805                                         BUS_NOTIFY_ADD_DEVICE);
 806                         if (!info) {
 807                                 return dmar_dev_scope_status;
 808                         } else {
 809                                 dmar_pci_bus_add_dev(info);
 810                                 dmar_free_pci_notify_info(info);
 811                         }
 812                 }
 813         }
 814
 815         return dmar_dev_scope_status;
 816 }
 817
 818 void __init dmar_register_bus_notifier(void)
 819 {
 820         bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 821 }
 822
 823
 824 int __init dmar_table_init(void)
 825 {
 826         static int dmar_table_initialized;
 827         int ret;
 828
 829         if (dmar_table_initialized == 0) {
 830                 ret = parse_dmar_table();
 831                 if (ret < 0) {
 832                         if (ret != -ENODEV)
 833                                 pr_info("Parse DMAR table failure.\n");
 834                 } else  if (list_empty(&dmar_drhd_units)) {
 835                         pr_info("No DMAR devices found\n");
 836                         ret = -ENODEV;
 837                 }
 838
 839                 if (ret < 0)
 840                         dmar_table_initialized = ret;
 841                 else
 842                         dmar_table_initialized = 1;
 843         }
 844
 845         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 846 }
 847
 848 static void warn_invalid_dmar(u64 addr, const char *message)
 849 {
 850         pr_warn_once(FW_BUG
 851                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 852                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 853                 addr, message,
 854                 dmi_get_system_info(DMI_BIOS_VENDOR),
 855                 dmi_get_system_info(DMI_BIOS_VERSION),
 856                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 857         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 858 }
 859
 860 static int __ref
 861 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 862 {
 863         struct acpi_dmar_hardware_unit *drhd;
 864         void __iomem *addr;
 865         u64 cap, ecap;
 866
 867         drhd = (void *)entry;
 868         if (!drhd->address) {
 869                 warn_invalid_dmar(0, "");
 870                 return -EINVAL;
 871         }
 872
 873         if (arg)
 874                 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 875         else
 876                 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 877         if (!addr) {
 878                 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 879                 return -EINVAL;
 880         }
 881
 882         cap = dmar_readq(addr + DMAR_CAP_REG);
 883         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 884
 885         if (arg)
 886                 iounmap(addr);
 887         else
 888                 early_iounmap(addr, VTD_PAGE_SIZE);
 889
 890         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 891                 warn_invalid_dmar(drhd->address, " returns all ones");
 892                 return -EINVAL;
 893         }
 894
 895         return 0;
 896 }
 897
 898 int __init detect_intel_iommu(void)
 899 {
 900         int ret;
 901         struct dmar_res_callback validate_drhd_cb = {
 902                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 903                 .ignore_unhandled = true,
 904         };
 905
 906         down_write(&dmar_global_lock);
 907         ret = dmar_table_detect();
 908         if (!ret)
 909                 ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 910                                            &validate_drhd_cb);
 911         if (!ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 912                 iommu_detected = 1;
 913                 /* Make sure ACS will be enabled */
 914                 pci_request_acs();
 915         }
 916
 917 #ifdef CONFIG_X86
 918         if (!ret)
 919                 x86_init.iommu.iommu_init = intel_iommu_init;
 920 #endif
 921
 922         if (dmar_tbl) {
 923                 acpi_put_table(dmar_tbl);
 924                 dmar_tbl = NULL;
 925         }
 926         up_write(&dmar_global_lock);
 927
 928         return ret ? ret : 1;
 929 }
 930
 931 static void unmap_iommu(struct intel_iommu *iommu)
 932 {
 933         iounmap(iommu->reg);
 934         release_mem_region(iommu->reg_phys, iommu->reg_size);
 935 }
 936
 937 /**
 938  * map_iommu: map the iommu's registers
 939  * @iommu: the iommu to map
 940  * @phys_addr: the physical address of the base resgister
 941  *
 942  * Memory map the iommu's registers.  Start w/ a single page, and
 943  * possibly expand if that turns out to be insufficent.
 944  */
 945 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 946 {
 947         int map_size, err=0;
 948
 949         iommu->reg_phys = phys_addr;
 950         iommu->reg_size = VTD_PAGE_SIZE;
 951
 952         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 953                 pr_err("Can't reserve memory\n");
 954                 err = -EBUSY;
 955                 goto out;
 956         }
 957
 958         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 959         if (!iommu->reg) {
 960                 pr_err("Can't map the region\n");
 961                 err = -ENOMEM;
 962                 goto release;
 963         }
 964
 965         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 966         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 967
 968         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 969                 err = -EINVAL;
 970                 warn_invalid_dmar(phys_addr, " returns all ones");
 971                 goto unmap;
 972         }
 973
 974         /* the registers might be more than one page */
 975         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 976                          cap_max_fault_reg_offset(iommu->cap));
 977         map_size = VTD_PAGE_ALIGN(map_size);
 978         if (map_size > iommu->reg_size) {
 979                 iounmap(iommu->reg);
 980                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 981                 iommu->reg_size = map_size;
 982                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 983                                         iommu->name)) {
 984                         pr_err("Can't reserve memory\n");
 985                         err = -EBUSY;
 986                         goto out;
 987                 }
 988                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 989                 if (!iommu->reg) {
 990                         pr_err("Can't map the region\n");
 991                         err = -ENOMEM;
 992                         goto release;
 993                 }
 994         }
 995         err = 0;
 996         goto out;
 997
 998 unmap:
 999         iounmap(iommu->reg);
1000 release:
1001         release_mem_region(iommu->reg_phys, iommu->reg_size);
1002 out:
1003         return err;
1004 }
1005
1006 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
1007 {
1008         iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
1009                                             DMAR_UNITS_SUPPORTED);
1010         if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
1011                 iommu->seq_id = -1;
1012         } else {
1013                 set_bit(iommu->seq_id, dmar_seq_ids);
1014                 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1015         }
1016
1017         return iommu->seq_id;
1018 }
1019
1020 static void dmar_free_seq_id(struct intel_iommu *iommu)
1021 {
1022         if (iommu->seq_id >= 0) {
1023                 clear_bit(iommu->seq_id, dmar_seq_ids);
1024                 iommu->seq_id = -1;
1025         }
1026 }
1027
1028 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1029 {
1030         struct intel_iommu *iommu;
1031         u32 ver, sts;
1032         int agaw = 0;
1033         int msagaw = 0;
1034         int err;
1035
1036         if (!drhd->reg_base_addr) {
1037                 warn_invalid_dmar(0, "");
1038                 return -EINVAL;
1039         }
1040
1041         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1042         if (!iommu)
1043                 return -ENOMEM;
1044
1045         if (dmar_alloc_seq_id(iommu) < 0) {
1046                 pr_err("Failed to allocate seq_id\n");
1047                 err = -ENOSPC;
1048                 goto error;
1049         }
1050
1051         err = map_iommu(iommu, drhd->reg_base_addr);
1052         if (err) {
1053                 pr_err("Failed to map %s\n", iommu->name);
1054                 goto error_free_seq_id;
1055         }
1056
1057         err = -EINVAL;
1058         agaw = iommu_calculate_agaw(iommu);
1059         if (agaw < 0) {
1060                 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1061                         iommu->seq_id);
1062                 goto err_unmap;
1063         }
1064         msagaw = iommu_calculate_max_sagaw(iommu);
1065         if (msagaw < 0) {
1066                 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1067                         iommu->seq_id);
1068                 goto err_unmap;
1069         }
1070         iommu->agaw = agaw;
1071         iommu->msagaw = msagaw;
1072         iommu->segment = drhd->segment;
1073
1074         iommu->node = -1;
1075
1076         ver = readl(iommu->reg + DMAR_VER_REG);
1077         pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1078                 iommu->name,
1079                 (unsigned long long)drhd->reg_base_addr,
1080                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1081                 (unsigned long long)iommu->cap,
1082                 (unsigned long long)iommu->ecap);
1083
1084         /* Reflect status in gcmd */
1085         sts = readl(iommu->reg + DMAR_GSTS_REG);
1086         if (sts & DMA_GSTS_IRES)
1087                 iommu->gcmd |= DMA_GCMD_IRE;
1088         if (sts & DMA_GSTS_TES)
1089                 iommu->gcmd |= DMA_GCMD_TE;
1090         if (sts & DMA_GSTS_QIES)
1091                 iommu->gcmd |= DMA_GCMD_QIE;
1092
1093         raw_spin_lock_init(&iommu->register_lock);
1094
1095         if (intel_iommu_enabled) {
1096                 err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1097                                              intel_iommu_groups,
1098                                              "%s", iommu->name);
1099                 if (err)
1100                         goto err_unmap;
1101
1102                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
1103
1104                 err = iommu_device_register(&iommu->iommu);
1105                 if (err)
1106                         goto err_unmap;
1107         }
1108
1109         drhd->iommu = iommu;
1110
1111         return 0;
1112
1113 err_unmap:
1114         unmap_iommu(iommu);
1115 error_free_seq_id:
1116         dmar_free_seq_id(iommu);
1117 error:
1118         kfree(iommu);
1119         return err;
1120 }
1121
1122 static void free_iommu(struct intel_iommu *iommu)
1123 {
1124         if (intel_iommu_enabled) {
1125                 iommu_device_unregister(&iommu->iommu);
1126                 iommu_device_sysfs_remove(&iommu->iommu);
1127         }
1128
1129         if (iommu->irq) {
1130                 if (iommu->pr_irq) {
1131                         free_irq(iommu->pr_irq, iommu);
1132                         dmar_free_hwirq(iommu->pr_irq);
1133                         iommu->pr_irq = 0;
1134                 }
1135                 free_irq(iommu->irq, iommu);
1136                 dmar_free_hwirq(iommu->irq);
1137                 iommu->irq = 0;
1138         }
1139
1140         if (iommu->qi) {
1141                 free_page((unsigned long)iommu->qi->desc);
1142                 kfree(iommu->qi->desc_status);
1143                 kfree(iommu->qi);
1144         }
1145
1146         if (iommu->reg)
1147                 unmap_iommu(iommu);
1148
1149         dmar_free_seq_id(iommu);
1150         kfree(iommu);
1151 }
1152
1153 /*
1154  * Reclaim all the submitted descriptors which have completed its work.
1155  */
1156 static inline void reclaim_free_desc(struct q_inval *qi)
1157 {
1158         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1159                qi->desc_status[qi->free_tail] == QI_ABORT) {
1160                 qi->desc_status[qi->free_tail] = QI_FREE;
1161                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1162                 qi->free_cnt++;
1163         }
1164 }
1165
1166 static int qi_check_fault(struct intel_iommu *iommu, int index)
1167 {
1168         u32 fault;
1169         int head, tail;
1170         struct q_inval *qi = iommu->qi;
1171         int wait_index = (index + 1) % QI_LENGTH;
1172
1173         if (qi->desc_status[wait_index] == QI_ABORT)
1174                 return -EAGAIN;
1175
1176         fault = readl(iommu->reg + DMAR_FSTS_REG);
1177
1178         /*
1179          * If IQE happens, the head points to the descriptor associated
1180          * with the error. No new descriptors are fetched until the IQE
1181          * is cleared.
1182          */
1183         if (fault & DMA_FSTS_IQE) {
1184                 head = readl(iommu->reg + DMAR_IQH_REG);
1185                 if ((head >> DMAR_IQ_SHIFT) == index) {
1186                         pr_err("VT-d detected invalid descriptor: "
1187                                 "low=%llx, high=%llx\n",
1188                                 (unsigned long long)qi->desc[index].low,
1189                                 (unsigned long long)qi->desc[index].high);
1190                         memcpy(&qi->desc[index], &qi->desc[wait_index],
1191                                         sizeof(struct qi_desc));
1192                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1193                         return -EINVAL;
1194                 }
1195         }
1196
1197         /*
1198          * If ITE happens, all pending wait_desc commands are aborted.
1199          * No new descriptors are fetched until the ITE is cleared.
1200          */
1201         if (fault & DMA_FSTS_ITE) {
1202                 head = readl(iommu->reg + DMAR_IQH_REG);
1203                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1204                 head |= 1;
1205                 tail = readl(iommu->reg + DMAR_IQT_REG);
1206                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1207
1208                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1209
1210                 do {
1211                         if (qi->desc_status[head] == QI_IN_USE)
1212                                 qi->desc_status[head] = QI_ABORT;
1213                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1214                 } while (head != tail);
1215
1216                 if (qi->desc_status[wait_index] == QI_ABORT)
1217                         return -EAGAIN;
1218         }
1219
1220         if (fault & DMA_FSTS_ICE)
1221                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1222
1223         return 0;
1224 }
1225
1226 /*
1227  * Submit the queued invalidation descriptor to the remapping
1228  * hardware unit and wait for its completion.
1229  */
1230 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1231 {
1232         int rc;
1233         struct q_inval *qi = iommu->qi;
1234         struct qi_desc *hw, wait_desc;
1235         int wait_index, index;
1236         unsigned long flags;
1237
1238         if (!qi)
1239                 return 0;
1240
1241         hw = qi->desc;
1242
1243 restart:
1244         rc = 0;
1245
1246         raw_spin_lock_irqsave(&qi->q_lock, flags);
1247         while (qi->free_cnt < 3) {
1248                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1249                 cpu_relax();
1250                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1251         }
1252
1253         index = qi->free_head;
1254         wait_index = (index + 1) % QI_LENGTH;
1255
1256         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1257
1258         hw[index] = *desc;
1259
1260         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1261                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1262         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1263
1264         hw[wait_index] = wait_desc;
1265
1266         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1267         qi->free_cnt -= 2;
1268
1269         /*
1270          * update the HW tail register indicating the presence of
1271          * new descriptors.
1272          */
1273         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1274
1275         while (qi->desc_status[wait_index] != QI_DONE) {
1276                 /*
1277                  * We will leave the interrupts disabled, to prevent interrupt
1278                  * context to queue another cmd while a cmd is already submitted
1279                  * and waiting for completion on this cpu. This is to avoid
1280                  * a deadlock where the interrupt context can wait indefinitely
1281                  * for free slots in the queue.
1282                  */
1283                 rc = qi_check_fault(iommu, index);
1284                 if (rc)
1285                         break;
1286
1287                 raw_spin_unlock(&qi->q_lock);
1288                 cpu_relax();
1289                 raw_spin_lock(&qi->q_lock);
1290         }
1291
1292         qi->desc_status[index] = QI_DONE;
1293
1294         reclaim_free_desc(qi);
1295         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1296
1297         if (rc == -EAGAIN)
1298                 goto restart;
1299
1300         return rc;
1301 }
1302
1303 /*
1304  * Flush the global interrupt entry cache.
1305  */
1306 void qi_global_iec(struct intel_iommu *iommu)
1307 {
1308         struct qi_desc desc;
1309
1310         desc.low = QI_IEC_TYPE;
1311         desc.high = 0;
1312
1313         /* should never fail */
1314         qi_submit_sync(&desc, iommu);
1315 }
1316
1317 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1318                       u64 type)
1319 {
1320         struct qi_desc desc;
1321
1322         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1323                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1324         desc.high = 0;
1325
1326         qi_submit_sync(&desc, iommu);
1327 }
1328
1329 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1330                     unsigned int size_order, u64 type)
1331 {
1332         u8 dw = 0, dr = 0;
1333
1334         struct qi_desc desc;
1335         int ih = 0;
1336
1337         if (cap_write_drain(iommu->cap))
1338                 dw = 1;
1339
1340         if (cap_read_drain(iommu->cap))
1341                 dr = 1;
1342
1343         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1344                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1345         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1346                 | QI_IOTLB_AM(size_order);
1347
1348         qi_submit_sync(&desc, iommu);
1349 }
1350
1351 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1352                         u16 qdep, u64 addr, unsigned mask)
1353 {
1354         struct qi_desc desc;
1355
1356         if (mask) {
1357                 addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1358                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1359         } else
1360                 desc.high = QI_DEV_IOTLB_ADDR(addr);
1361
1362         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1363                 qdep = 0;
1364
1365         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1366                    QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1367
1368         qi_submit_sync(&desc, iommu);
1369 }
1370
1371 /*
1372  * Disable Queued Invalidation interface.
1373  */
1374 void dmar_disable_qi(struct intel_iommu *iommu)
1375 {
1376         unsigned long flags;
1377         u32 sts;
1378         cycles_t start_time = get_cycles();
1379
1380         if (!ecap_qis(iommu->ecap))
1381                 return;
1382
1383         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1384
1385         sts =  readl(iommu->reg + DMAR_GSTS_REG);
1386         if (!(sts & DMA_GSTS_QIES))
1387                 goto end;
1388
1389         /*
1390          * Give a chance to HW to complete the pending invalidation requests.
1391          */
1392         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1393                 readl(iommu->reg + DMAR_IQH_REG)) &&
1394                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1395                 cpu_relax();
1396
1397         iommu->gcmd &= ~DMA_GCMD_QIE;
1398         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1399
1400         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1401                       !(sts & DMA_GSTS_QIES), sts);
1402 end:
1403         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1404 }
1405
1406 /*
1407  * Enable queued invalidation.
1408  */
1409 static void __dmar_enable_qi(struct intel_iommu *iommu)
1410 {
1411         u32 sts;
1412         unsigned long flags;
1413         struct q_inval *qi = iommu->qi;
1414
1415         qi->free_head = qi->free_tail = 0;
1416         qi->free_cnt = QI_LENGTH;
1417
1418         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1419
1420         /* write zero to the tail reg */
1421         writel(0, iommu->reg + DMAR_IQT_REG);
1422
1423         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1424
1425         iommu->gcmd |= DMA_GCMD_QIE;
1426         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1427
1428         /* Make sure hardware complete it */
1429         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1430
1431         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1432 }
1433
1434 /*
1435  * Enable Queued Invalidation interface. This is a must to support
1436  * interrupt-remapping. Also used by DMA-remapping, which replaces
1437  * register based IOTLB invalidation.
1438  */
1439 int dmar_enable_qi(struct intel_iommu *iommu)
1440 {
1441         struct q_inval *qi;
1442         struct page *desc_page;
1443
1444         if (!ecap_qis(iommu->ecap))
1445                 return -ENOENT;
1446
1447         /*
1448          * queued invalidation is already setup and enabled.
1449          */
1450         if (iommu->qi)
1451                 return 0;
1452
1453         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1454         if (!iommu->qi)
1455                 return -ENOMEM;
1456
1457         qi = iommu->qi;
1458
1459
1460         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1461         if (!desc_page) {
1462                 kfree(qi);
1463                 iommu->qi = NULL;
1464                 return -ENOMEM;
1465         }
1466
1467         qi->desc = page_address(desc_page);
1468
1469         qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1470         if (!qi->desc_status) {
1471                 free_page((unsigned long) qi->desc);
1472                 kfree(qi);
1473                 iommu->qi = NULL;
1474                 return -ENOMEM;
1475         }
1476
1477         raw_spin_lock_init(&qi->q_lock);
1478
1479         __dmar_enable_qi(iommu);
1480
1481         return 0;
1482 }
1483
1484 /* iommu interrupt handling. Most stuff are MSI-like. */
1485
1486 enum faulttype {
1487         DMA_REMAP,
1488         INTR_REMAP,
1489         UNKNOWN,
1490 };
1491
1492 static const char *dma_remap_fault_reasons[] =
1493 {
1494         "Software",
1495         "Present bit in root entry is clear",
1496         "Present bit in context entry is clear",
1497         "Invalid context entry",
1498         "Access beyond MGAW",
1499         "PTE Write access is not set",
1500         "PTE Read access is not set",
1501         "Next page table ptr is invalid",
1502         "Root table address invalid",
1503         "Context table ptr is invalid",
1504         "non-zero reserved fields in RTP",
1505         "non-zero reserved fields in CTP",
1506         "non-zero reserved fields in PTE",
1507         "PCE for translation request specifies blocking",
1508 };
1509
1510 static const char *irq_remap_fault_reasons[] =
1511 {
1512         "Detected reserved fields in the decoded interrupt-remapped request",
1513         "Interrupt index exceeded the interrupt-remapping table size",
1514         "Present field in the IRTE entry is clear",
1515         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1516         "Detected reserved fields in the IRTE entry",
1517         "Blocked a compatibility format interrupt request",
1518         "Blocked an interrupt request due to source-id verification failure",
1519 };
1520
1521 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1522 {
1523         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1524                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1525                 *fault_type = INTR_REMAP;
1526                 return irq_remap_fault_reasons[fault_reason - 0x20];
1527         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1528                 *fault_type = DMA_REMAP;
1529                 return dma_remap_fault_reasons[fault_reason];
1530         } else {
1531                 *fault_type = UNKNOWN;
1532                 return "Unknown";
1533         }
1534 }
1535
1536
1537 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1538 {
1539         if (iommu->irq == irq)
1540                 return DMAR_FECTL_REG;
1541         else if (iommu->pr_irq == irq)
1542                 return DMAR_PECTL_REG;
1543         else
1544                 BUG();
1545 }
1546
1547 void dmar_msi_unmask(struct irq_data *data)
1548 {
1549         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1550         int reg = dmar_msi_reg(iommu, data->irq);
1551         unsigned long flag;
1552
1553         /* unmask it */
1554         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1555         writel(0, iommu->reg + reg);
1556         /* Read a reg to force flush the post write */
1557         readl(iommu->reg + reg);
1558         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1559 }
1560
1561 void dmar_msi_mask(struct irq_data *data)
1562 {
1563         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1564         int reg = dmar_msi_reg(iommu, data->irq);
1565         unsigned long flag;
1566
1567         /* mask it */
1568         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1569         writel(DMA_FECTL_IM, iommu->reg + reg);
1570         /* Read a reg to force flush the post write */
1571         readl(iommu->reg + reg);
1572         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1573 }
1574
1575 void dmar_msi_write(int irq, struct msi_msg *msg)
1576 {
1577         struct intel_iommu *iommu = irq_get_handler_data(irq);
1578         int reg = dmar_msi_reg(iommu, irq);
1579         unsigned long flag;
1580
1581         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1582         writel(msg->data, iommu->reg + reg + 4);
1583         writel(msg->address_lo, iommu->reg + reg + 8);
1584         writel(msg->address_hi, iommu->reg + reg + 12);
1585         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1586 }
1587
1588 void dmar_msi_read(int irq, struct msi_msg *msg)
1589 {
1590         struct intel_iommu *iommu = irq_get_handler_data(irq);
1591         int reg = dmar_msi_reg(iommu, irq);
1592         unsigned long flag;
1593
1594         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1595         msg->data = readl(iommu->reg + reg + 4);
1596         msg->address_lo = readl(iommu->reg + reg + 8);
1597         msg->address_hi = readl(iommu->reg + reg + 12);
1598         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1599 }
1600
1601 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1602                 u8 fault_reason, u16 source_id, unsigned long long addr)
1603 {
1604         const char *reason;
1605         int fault_type;
1606
1607         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1608
1609         if (fault_type == INTR_REMAP)
1610                 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1611                         source_id >> 8, PCI_SLOT(source_id & 0xFF),
1612                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1613                         fault_reason, reason);
1614         else
1615                 pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
1616                        type ? "DMA Read" : "DMA Write",
1617                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
1618                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1619         return 0;
1620 }
1621
1622 #define PRIMARY_FAULT_REG_LEN (16)
1623 irqreturn_t dmar_fault(int irq, void *dev_id)
1624 {
1625         struct intel_iommu *iommu = dev_id;
1626         int reg, fault_index;
1627         u32 fault_status;
1628         unsigned long flag;
1629         static DEFINE_RATELIMIT_STATE(rs,
1630                                       DEFAULT_RATELIMIT_INTERVAL,
1631                                       DEFAULT_RATELIMIT_BURST);
1632
1633         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1634         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1635         if (fault_status && __ratelimit(&rs))
1636                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1637
1638         /* TBD: ignore advanced fault log currently */
1639         if (!(fault_status & DMA_FSTS_PPF))
1640                 goto unlock_exit;
1641
1642         fault_index = dma_fsts_fault_record_index(fault_status);
1643         reg = cap_fault_reg_offset(iommu->cap);
1644         while (1) {
1645                 /* Disable printing, simply clear the fault when ratelimited */
1646                 bool ratelimited = !__ratelimit(&rs);
1647                 u8 fault_reason;
1648                 u16 source_id;
1649                 u64 guest_addr;
1650                 int type;
1651                 u32 data;
1652
1653                 /* highest 32 bits */
1654                 data = readl(iommu->reg + reg +
1655                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1656                 if (!(data & DMA_FRCD_F))
1657                         break;
1658
1659                 if (!ratelimited) {
1660                         fault_reason = dma_frcd_fault_reason(data);
1661                         type = dma_frcd_type(data);
1662
1663                         data = readl(iommu->reg + reg +
1664                                      fault_index * PRIMARY_FAULT_REG_LEN + 8);
1665                         source_id = dma_frcd_source_id(data);
1666
1667                         guest_addr = dmar_readq(iommu->reg + reg +
1668                                         fault_index * PRIMARY_FAULT_REG_LEN);
1669                         guest_addr = dma_frcd_page_addr(guest_addr);
1670                 }
1671
1672                 /* clear the fault */
1673                 writel(DMA_FRCD_F, iommu->reg + reg +
1674                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1675
1676                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1677
1678                 if (!ratelimited)
1679                         dmar_fault_do_one(iommu, type, fault_reason,
1680                                           source_id, guest_addr);
1681
1682                 fault_index++;
1683                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1684                         fault_index = 0;
1685                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1686         }
1687
1688         writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
1689                iommu->reg + DMAR_FSTS_REG);
1690
1691 unlock_exit:
1692         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1693         return IRQ_HANDLED;
1694 }
1695
1696 int dmar_set_interrupt(struct intel_iommu *iommu)
1697 {
1698         int irq, ret;
1699
1700         /*
1701          * Check if the fault interrupt is already initialized.
1702          */
1703         if (iommu->irq)
1704                 return 0;
1705
1706         irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1707         if (irq > 0) {
1708                 iommu->irq = irq;
1709         } else {
1710                 pr_err("No free IRQ vectors\n");
1711                 return -EINVAL;
1712         }
1713
1714         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1715         if (ret)
1716                 pr_err("Can't request irq\n");
1717         return ret;
1718 }
1719
1720 int __init enable_drhd_fault_handling(void)
1721 {
1722         struct dmar_drhd_unit *drhd;
1723         struct intel_iommu *iommu;
1724
1725         /*
1726          * Enable fault control interrupt.
1727          */
1728         for_each_iommu(iommu, drhd) {
1729                 u32 fault_status;
1730                 int ret = dmar_set_interrupt(iommu);
1731
1732                 if (ret) {
1733                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1734                                (unsigned long long)drhd->reg_base_addr, ret);
1735                         return -1;
1736                 }
1737
1738                 /*
1739                  * Clear any previous faults.
1740                  */
1741                 dmar_fault(iommu->irq, iommu);
1742                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1743                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1744         }
1745
1746         return 0;
1747 }
1748
1749 /*
1750  * Re-enable Queued Invalidation interface.
1751  */
1752 int dmar_reenable_qi(struct intel_iommu *iommu)
1753 {
1754         if (!ecap_qis(iommu->ecap))
1755                 return -ENOENT;
1756
1757         if (!iommu->qi)
1758                 return -ENOENT;
1759
1760         /*
1761          * First disable queued invalidation.
1762          */
1763         dmar_disable_qi(iommu);
1764         /*
1765          * Then enable queued invalidation again. Since there is no pending
1766          * invalidation requests now, it's safe to re-enable queued
1767          * invalidation.
1768          */
1769         __dmar_enable_qi(iommu);
1770
1771         return 0;
1772 }
1773
1774 /*
1775  * Check interrupt remapping support in DMAR table description.
1776  */
1777 int __init dmar_ir_support(void)
1778 {
1779         struct acpi_table_dmar *dmar;
1780         dmar = (struct acpi_table_dmar *)dmar_tbl;
1781         if (!dmar)
1782                 return 0;
1783         return dmar->flags & 0x1;
1784 }
1785
1786 /* Check whether DMAR units are in use */
1787 static inline bool dmar_in_use(void)
1788 {
1789         return irq_remapping_enabled || intel_iommu_enabled;
1790 }
1791
1792 static int __init dmar_free_unused_resources(void)
1793 {
1794         struct dmar_drhd_unit *dmaru, *dmaru_n;
1795
1796         if (dmar_in_use())
1797                 return 0;
1798
1799         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1800                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1801
1802         down_write(&dmar_global_lock);
1803         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1804                 list_del(&dmaru->list);
1805                 dmar_free_drhd(dmaru);
1806         }
1807         up_write(&dmar_global_lock);
1808
1809         return 0;
1810 }
1811
1812 late_initcall(dmar_free_unused_resources);
1813 IOMMU_INIT_POST(detect_intel_iommu);
1814
1815 /*
1816  * DMAR Hotplug Support
1817  * For more details, please refer to Intel(R) Virtualization Technology
1818  * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1819  * "Remapping Hardware Unit Hot Plug".
1820  */
1821 static guid_t dmar_hp_guid =
1822         GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
1823                   0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
1824
1825 /*
1826  * Currently there's only one revision and BIOS will not check the revision id,
1827  * so use 0 for safety.
1828  */
1829 #define DMAR_DSM_REV_ID                 0
1830 #define DMAR_DSM_FUNC_DRHD              1
1831 #define DMAR_DSM_FUNC_ATSR              2
1832 #define DMAR_DSM_FUNC_RHSA              3
1833
1834 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1835 {
1836         return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
1837 }
1838
1839 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1840                                   dmar_res_handler_t handler, void *arg)
1841 {
1842         int ret = -ENODEV;
1843         union acpi_object *obj;
1844         struct acpi_dmar_header *start;
1845         struct dmar_res_callback callback;
1846         static int res_type[] = {
1847                 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1848                 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1849                 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1850         };
1851
1852         if (!dmar_detect_dsm(handle, func))
1853                 return 0;
1854
1855         obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
1856                                       func, NULL, ACPI_TYPE_BUFFER);
1857         if (!obj)
1858                 return -ENODEV;
1859
1860         memset(&callback, 0, sizeof(callback));
1861         callback.cb[res_type[func]] = handler;
1862         callback.arg[res_type[func]] = arg;
1863         start = (struct acpi_dmar_header *)obj->buffer.pointer;
1864         ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1865
1866         ACPI_FREE(obj);
1867
1868         return ret;
1869 }
1870
1871 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1872 {
1873         int ret;
1874         struct dmar_drhd_unit *dmaru;
1875
1876         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1877         if (!dmaru)
1878                 return -ENODEV;
1879
1880         ret = dmar_ir_hotplug(dmaru, true);
1881         if (ret == 0)
1882                 ret = dmar_iommu_hotplug(dmaru, true);
1883
1884         return ret;
1885 }
1886
1887 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1888 {
1889         int i, ret;
1890         struct device *dev;
1891         struct dmar_drhd_unit *dmaru;
1892
1893         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1894         if (!dmaru)
1895                 return 0;
1896
1897         /*
1898          * All PCI devices managed by this unit should have been destroyed.
1899          */
1900         if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
1901                 for_each_active_dev_scope(dmaru->devices,
1902                                           dmaru->devices_cnt, i, dev)
1903                         return -EBUSY;
1904         }
1905
1906         ret = dmar_ir_hotplug(dmaru, false);
1907         if (ret == 0)
1908                 ret = dmar_iommu_hotplug(dmaru, false);
1909
1910         return ret;
1911 }
1912
1913 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1914 {
1915         struct dmar_drhd_unit *dmaru;
1916
1917         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1918         if (dmaru) {
1919                 list_del_rcu(&dmaru->list);
1920                 synchronize_rcu();
1921                 dmar_free_drhd(dmaru);
1922         }
1923
1924         return 0;
1925 }
1926
1927 static int dmar_hotplug_insert(acpi_handle handle)
1928 {
1929         int ret;
1930         int drhd_count = 0;
1931
1932         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1933                                      &dmar_validate_one_drhd, (void *)1);
1934         if (ret)
1935                 goto out;
1936
1937         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1938                                      &dmar_parse_one_drhd, (void *)&drhd_count);
1939         if (ret == 0 && drhd_count == 0) {
1940                 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1941                 goto out;
1942         } else if (ret) {
1943                 goto release_drhd;
1944         }
1945
1946         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1947                                      &dmar_parse_one_rhsa, NULL);
1948         if (ret)
1949                 goto release_drhd;
1950
1951         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1952                                      &dmar_parse_one_atsr, NULL);
1953         if (ret)
1954                 goto release_atsr;
1955
1956         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1957                                      &dmar_hp_add_drhd, NULL);
1958         if (!ret)
1959                 return 0;
1960
1961         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1962                                &dmar_hp_remove_drhd, NULL);
1963 release_atsr:
1964         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1965                                &dmar_release_one_atsr, NULL);
1966 release_drhd:
1967         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1968                                &dmar_hp_release_drhd, NULL);
1969 out:
1970         return ret;
1971 }
1972
1973 static int dmar_hotplug_remove(acpi_handle handle)
1974 {
1975         int ret;
1976
1977         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1978                                      &dmar_check_one_atsr, NULL);
1979         if (ret)
1980                 return ret;
1981
1982         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1983                                      &dmar_hp_remove_drhd, NULL);
1984         if (ret == 0) {
1985                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1986                                                &dmar_release_one_atsr, NULL));
1987                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1988                                                &dmar_hp_release_drhd, NULL));
1989         } else {
1990                 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1991                                        &dmar_hp_add_drhd, NULL);
1992         }
1993
1994         return ret;
1995 }
1996
1997 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1998                                        void *context, void **retval)
1999 {
2000         acpi_handle *phdl = retval;
2001
2002         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2003                 *phdl = handle;
2004                 return AE_CTRL_TERMINATE;
2005         }
2006
2007         return AE_OK;
2008 }
2009
2010 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2011 {
2012         int ret;
2013         acpi_handle tmp = NULL;
2014         acpi_status status;
2015
2016         if (!dmar_in_use())
2017                 return 0;
2018
2019         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2020                 tmp = handle;
2021         } else {
2022                 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2023                                              ACPI_UINT32_MAX,
2024                                              dmar_get_dsm_handle,
2025                                              NULL, NULL, &tmp);
2026                 if (ACPI_FAILURE(status)) {
2027                         pr_warn("Failed to locate _DSM method.\n");
2028                         return -ENXIO;
2029                 }
2030         }
2031         if (tmp == NULL)
2032                 return 0;
2033
2034         down_write(&dmar_global_lock);
2035         if (insert)
2036                 ret = dmar_hotplug_insert(tmp);
2037         else
2038                 ret = dmar_hotplug_remove(tmp);
2039         up_write(&dmar_global_lock);
2040
2041         return ret;
2042 }
2043
2044 int dmar_device_add(acpi_handle handle)
2045 {
2046         return dmar_device_hotplug(handle, true);
2047 }
2048
2049 int dmar_device_remove(acpi_handle handle)
2050 {
2051         return dmar_device_hotplug(handle, false);
2052 }