drivers/iommu/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
  30
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/iova.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/timer.h>
  36 #include <linux/irq.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/slab.h>
  41 #include <asm/irq_remapping.h>
  42 #include <asm/iommu_table.h>
  43
  44 /* No locks are needed as DMA remapping hardware unit
  45  * list is constructed at boot time and hotplug of
  46  * these units are not supported by the architecture.
  47  */
  48 LIST_HEAD(dmar_drhd_units);
  49
  50 struct acpi_table_header * __initdata dmar_tbl;
  51 static acpi_size dmar_tbl_size;
  52
  53 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  54 {
  55         /*
  56          * add INCLUDE_ALL at the tail, so scan the list will find it at
  57          * the very end.
  58          */
  59         if (drhd->include_all)
  60                 list_add_tail(&drhd->list, &dmar_drhd_units);
  61         else
  62                 list_add(&drhd->list, &dmar_drhd_units);
  63 }
  64
  65 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  66                                            struct pci_dev **dev, u16 segment)
  67 {
  68         struct pci_bus *bus;
  69         struct pci_dev *pdev = NULL;
  70         struct acpi_dmar_pci_path *path;
  71         int count;
  72
  73         bus = pci_find_bus(segment, scope->bus);
  74         path = (struct acpi_dmar_pci_path *)(scope + 1);
  75         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  76                 / sizeof(struct acpi_dmar_pci_path);
  77
  78         while (count) {
  79                 if (pdev)
  80                         pci_dev_put(pdev);
  81                 /*
  82                  * Some BIOSes list non-exist devices in DMAR table, just
  83                  * ignore it
  84                  */
  85                 if (!bus) {
  86                         pr_warn("Device scope bus [%d] not found\n", scope->bus);
  87                         break;
  88                 }
  89                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  90                 if (!pdev) {
  91                         /* warning will be printed below */
  92                         break;
  93                 }
  94                 path ++;
  95                 count --;
  96                 bus = pdev->subordinate;
  97         }
  98         if (!pdev) {
  99                 pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
 100                         segment, scope->bus, path->dev, path->fn);
 101                 *dev = NULL;
 102                 return 0;
 103         }
 104         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 105                         pdev->subordinate) || (scope->entry_type == \
 106                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 107                 pci_dev_put(pdev);
 108                 pr_warn("Device scope type does not match for %s\n",
 109                         pci_name(pdev));
 110                 return -EINVAL;
 111         }
 112         *dev = pdev;
 113         return 0;
 114 }
 115
 116 int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 117                                 struct pci_dev ***devices, u16 segment)
 118 {
 119         struct acpi_dmar_device_scope *scope;
 120         void * tmp = start;
 121         int index;
 122         int ret;
 123
 124         *cnt = 0;
 125         while (start < end) {
 126                 scope = start;
 127                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 128                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 129                         (*cnt)++;
 130                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
 131                         pr_warn("Unsupported device scope\n");
 132                 }
 133                 start += scope->length;
 134         }
 135         if (*cnt == 0)
 136                 return 0;
 137
 138         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 139         if (!*devices)
 140                 return -ENOMEM;
 141
 142         start = tmp;
 143         index = 0;
 144         while (start < end) {
 145                 scope = start;
 146                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 147                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 148                         ret = dmar_parse_one_dev_scope(scope,
 149                                 &(*devices)[index], segment);
 150                         if (ret) {
 151                                 kfree(*devices);
 152                                 return ret;
 153                         }
 154                         index ++;
 155                 }
 156                 start += scope->length;
 157         }
 158
 159         return 0;
 160 }
 161
 162 /**
 163  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 164  * structure which uniquely represent one DMA remapping hardware unit
 165  * present in the platform
 166  */
 167 static int __init
 168 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 169 {
 170         struct acpi_dmar_hardware_unit *drhd;
 171         struct dmar_drhd_unit *dmaru;
 172         int ret = 0;
 173
 174         drhd = (struct acpi_dmar_hardware_unit *)header;
 175         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 176         if (!dmaru)
 177                 return -ENOMEM;
 178
 179         dmaru->hdr = header;
 180         dmaru->reg_base_addr = drhd->address;
 181         dmaru->segment = drhd->segment;
 182         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 183
 184         ret = alloc_iommu(dmaru);
 185         if (ret) {
 186                 kfree(dmaru);
 187                 return ret;
 188         }
 189         dmar_register_drhd_unit(dmaru);
 190         return 0;
 191 }
 192
 193 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 194 {
 195         struct acpi_dmar_hardware_unit *drhd;
 196         int ret = 0;
 197
 198         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 199
 200         if (dmaru->include_all)
 201                 return 0;
 202
 203         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 204                                 ((void *)drhd) + drhd->header.length,
 205                                 &dmaru->devices_cnt, &dmaru->devices,
 206                                 drhd->segment);
 207         if (ret) {
 208                 list_del(&dmaru->list);
 209                 kfree(dmaru);
 210         }
 211         return ret;
 212 }
 213
 214 #ifdef CONFIG_ACPI_NUMA
 215 static int __init
 216 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 217 {
 218         struct acpi_dmar_rhsa *rhsa;
 219         struct dmar_drhd_unit *drhd;
 220
 221         rhsa = (struct acpi_dmar_rhsa *)header;
 222         for_each_drhd_unit(drhd) {
 223                 if (drhd->reg_base_addr == rhsa->base_address) {
 224                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 225
 226                         if (!node_online(node))
 227                                 node = -1;
 228                         drhd->iommu->node = node;
 229                         return 0;
 230                 }
 231         }
 232         WARN_TAINT(
 233                 1, TAINT_FIRMWARE_WORKAROUND,
 234                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 235                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 236                 drhd->reg_base_addr,
 237                 dmi_get_system_info(DMI_BIOS_VENDOR),
 238                 dmi_get_system_info(DMI_BIOS_VERSION),
 239                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 240
 241         return 0;
 242 }
 243 #endif
 244
 245 static void __init
 246 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 247 {
 248         struct acpi_dmar_hardware_unit *drhd;
 249         struct acpi_dmar_reserved_memory *rmrr;
 250         struct acpi_dmar_atsr *atsr;
 251         struct acpi_dmar_rhsa *rhsa;
 252
 253         switch (header->type) {
 254         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 255                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 256                                     header);
 257                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 258                         (unsigned long long)drhd->address, drhd->flags);
 259                 break;
 260         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 261                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 262                                     header);
 263                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 264                         (unsigned long long)rmrr->base_address,
 265                         (unsigned long long)rmrr->end_address);
 266                 break;
 267         case ACPI_DMAR_TYPE_ATSR:
 268                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 269                 pr_info("ATSR flags: %#x\n", atsr->flags);
 270                 break;
 271         case ACPI_DMAR_HARDWARE_AFFINITY:
 272                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 273                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 274                        (unsigned long long)rhsa->base_address,
 275                        rhsa->proximity_domain);
 276                 break;
 277         }
 278 }
 279
 280 /**
 281  * dmar_table_detect - checks to see if the platform supports DMAR devices
 282  */
 283 static int __init dmar_table_detect(void)
 284 {
 285         acpi_status status = AE_OK;
 286
 287         /* if we could find DMAR table, then there are DMAR devices */
 288         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 289                                 (struct acpi_table_header **)&dmar_tbl,
 290                                 &dmar_tbl_size);
 291
 292         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 293                 pr_warn("Unable to map DMAR\n");
 294                 status = AE_NOT_FOUND;
 295         }
 296
 297         return (ACPI_SUCCESS(status) ? 1 : 0);
 298 }
 299
 300 /**
 301  * parse_dmar_table - parses the DMA reporting table
 302  */
 303 static int __init
 304 parse_dmar_table(void)
 305 {
 306         struct acpi_table_dmar *dmar;
 307         struct acpi_dmar_header *entry_header;
 308         int ret = 0;
 309
 310         /*
 311          * Do it again, earlier dmar_tbl mapping could be mapped with
 312          * fixed map.
 313          */
 314         dmar_table_detect();
 315
 316         /*
 317          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 318          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 319          */
 320         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 321
 322         dmar = (struct acpi_table_dmar *)dmar_tbl;
 323         if (!dmar)
 324                 return -ENODEV;
 325
 326         if (dmar->width < PAGE_SHIFT - 1) {
 327                 pr_warn("Invalid DMAR haw\n");
 328                 return -EINVAL;
 329         }
 330
 331         pr_info("Host address width %d\n", dmar->width + 1);
 332
 333         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 334         while (((unsigned long)entry_header) <
 335                         (((unsigned long)dmar) + dmar_tbl->length)) {
 336                 /* Avoid looping forever on bad ACPI tables */
 337                 if (entry_header->length == 0) {
 338                         pr_warn("Invalid 0-length structure\n");
 339                         ret = -EINVAL;
 340                         break;
 341                 }
 342
 343                 dmar_table_print_dmar_entry(entry_header);
 344
 345                 switch (entry_header->type) {
 346                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 347                         ret = dmar_parse_one_drhd(entry_header);
 348                         break;
 349                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 350                         ret = dmar_parse_one_rmrr(entry_header);
 351                         break;
 352                 case ACPI_DMAR_TYPE_ATSR:
 353                         ret = dmar_parse_one_atsr(entry_header);
 354                         break;
 355                 case ACPI_DMAR_HARDWARE_AFFINITY:
 356 #ifdef CONFIG_ACPI_NUMA
 357                         ret = dmar_parse_one_rhsa(entry_header);
 358 #endif
 359                         break;
 360                 default:
 361                         pr_warn("Unknown DMAR structure type %d\n",
 362                                 entry_header->type);
 363                         ret = 0; /* for forward compatibility */
 364                         break;
 365                 }
 366                 if (ret)
 367                         break;
 368
 369                 entry_header = ((void *)entry_header + entry_header->length);
 370         }
 371         return ret;
 372 }
 373
 374 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 375                           struct pci_dev *dev)
 376 {
 377         int index;
 378
 379         while (dev) {
 380                 for (index = 0; index < cnt; index++)
 381                         if (dev == devices[index])
 382                                 return 1;
 383
 384                 /* Check our parent */
 385                 dev = dev->bus->self;
 386         }
 387
 388         return 0;
 389 }
 390
 391 struct dmar_drhd_unit *
 392 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 393 {
 394         struct dmar_drhd_unit *dmaru = NULL;
 395         struct acpi_dmar_hardware_unit *drhd;
 396
 397         dev = pci_physfn(dev);
 398
 399         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 400                 drhd = container_of(dmaru->hdr,
 401                                     struct acpi_dmar_hardware_unit,
 402                                     header);
 403
 404                 if (dmaru->include_all &&
 405                     drhd->segment == pci_domain_nr(dev->bus))
 406                         return dmaru;
 407
 408                 if (dmar_pci_device_match(dmaru->devices,
 409                                           dmaru->devices_cnt, dev))
 410                         return dmaru;
 411         }
 412
 413         return NULL;
 414 }
 415
 416 int __init dmar_dev_scope_init(void)
 417 {
 418         static int dmar_dev_scope_initialized;
 419         struct dmar_drhd_unit *drhd, *drhd_n;
 420         int ret = -ENODEV;
 421
 422         if (dmar_dev_scope_initialized)
 423                 return dmar_dev_scope_initialized;
 424
 425         if (list_empty(&dmar_drhd_units))
 426                 goto fail;
 427
 428         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 429                 ret = dmar_parse_dev(drhd);
 430                 if (ret)
 431                         goto fail;
 432         }
 433
 434         ret = dmar_parse_rmrr_atsr_dev();
 435         if (ret)
 436                 goto fail;
 437
 438         dmar_dev_scope_initialized = 1;
 439         return 0;
 440
 441 fail:
 442         dmar_dev_scope_initialized = ret;
 443         return ret;
 444 }
 445
 446
 447 int __init dmar_table_init(void)
 448 {
 449         static int dmar_table_initialized;
 450         int ret;
 451
 452         if (dmar_table_initialized)
 453                 return 0;
 454
 455         dmar_table_initialized = 1;
 456
 457         ret = parse_dmar_table();
 458         if (ret) {
 459                 if (ret != -ENODEV)
 460                         pr_info("parse DMAR table failure.\n");
 461                 return ret;
 462         }
 463
 464         if (list_empty(&dmar_drhd_units)) {
 465                 pr_info("No DMAR devices found\n");
 466                 return -ENODEV;
 467         }
 468
 469         return 0;
 470 }
 471
 472 static void warn_invalid_dmar(u64 addr, const char *message)
 473 {
 474         WARN_TAINT_ONCE(
 475                 1, TAINT_FIRMWARE_WORKAROUND,
 476                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 477                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 478                 addr, message,
 479                 dmi_get_system_info(DMI_BIOS_VENDOR),
 480                 dmi_get_system_info(DMI_BIOS_VERSION),
 481                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 482 }
 483
 484 int __init check_zero_address(void)
 485 {
 486         struct acpi_table_dmar *dmar;
 487         struct acpi_dmar_header *entry_header;
 488         struct acpi_dmar_hardware_unit *drhd;
 489
 490         dmar = (struct acpi_table_dmar *)dmar_tbl;
 491         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 492
 493         while (((unsigned long)entry_header) <
 494                         (((unsigned long)dmar) + dmar_tbl->length)) {
 495                 /* Avoid looping forever on bad ACPI tables */
 496                 if (entry_header->length == 0) {
 497                         pr_warn("Invalid 0-length structure\n");
 498                         return 0;
 499                 }
 500
 501                 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 502                         void __iomem *addr;
 503                         u64 cap, ecap;
 504
 505                         drhd = (void *)entry_header;
 506                         if (!drhd->address) {
 507                                 warn_invalid_dmar(0, "");
 508                                 goto failed;
 509                         }
 510
 511                         addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 512                         if (!addr ) {
 513                                 printk("IOMMU: can't validate: %llx\n", drhd->address);
 514                                 goto failed;
 515                         }
 516                         cap = dmar_readq(addr + DMAR_CAP_REG);
 517                         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 518                         early_iounmap(addr, VTD_PAGE_SIZE);
 519                         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 520                                 warn_invalid_dmar(drhd->address,
 521                                                   " returns all ones");
 522                                 goto failed;
 523                         }
 524                 }
 525
 526                 entry_header = ((void *)entry_header + entry_header->length);
 527         }
 528         return 1;
 529
 530 failed:
 531         return 0;
 532 }
 533
 534 int __init detect_intel_iommu(void)
 535 {
 536         int ret;
 537
 538         ret = dmar_table_detect();
 539         if (ret)
 540                 ret = check_zero_address();
 541         {
 542                 struct acpi_table_dmar *dmar;
 543
 544                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 545
 546                 if (ret && irq_remapping_enabled && cpu_has_x2apic &&
 547                     dmar->flags & 0x1)
 548                         pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
 549
 550                 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 551                         iommu_detected = 1;
 552                         /* Make sure ACS will be enabled */
 553                         pci_request_acs();
 554                 }
 555
 556 #ifdef CONFIG_X86
 557                 if (ret)
 558                         x86_init.iommu.iommu_init = intel_iommu_init;
 559 #endif
 560         }
 561         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 562         dmar_tbl = NULL;
 563
 564         return ret ? 1 : -ENODEV;
 565 }
 566
 567
 568 static void unmap_iommu(struct intel_iommu *iommu)
 569 {
 570         iounmap(iommu->reg);
 571         release_mem_region(iommu->reg_phys, iommu->reg_size);
 572 }
 573
 574 /**
 575  * map_iommu: map the iommu's registers
 576  * @iommu: the iommu to map
 577  * @phys_addr: the physical address of the base resgister
 578  *
 579  * Memory map the iommu's registers.  Start w/ a single page, and
 580  * possibly expand if that turns out to be insufficent.
 581  */
 582 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 583 {
 584         int map_size, err=0;
 585
 586         iommu->reg_phys = phys_addr;
 587         iommu->reg_size = VTD_PAGE_SIZE;
 588
 589         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 590                 pr_err("IOMMU: can't reserve memory\n");
 591                 err = -EBUSY;
 592                 goto out;
 593         }
 594
 595         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 596         if (!iommu->reg) {
 597                 pr_err("IOMMU: can't map the region\n");
 598                 err = -ENOMEM;
 599                 goto release;
 600         }
 601
 602         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 603         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 604
 605         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 606                 err = -EINVAL;
 607                 warn_invalid_dmar(phys_addr, " returns all ones");
 608                 goto unmap;
 609         }
 610
 611         /* the registers might be more than one page */
 612         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 613                          cap_max_fault_reg_offset(iommu->cap));
 614         map_size = VTD_PAGE_ALIGN(map_size);
 615         if (map_size > iommu->reg_size) {
 616                 iounmap(iommu->reg);
 617                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 618                 iommu->reg_size = map_size;
 619                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 620                                         iommu->name)) {
 621                         pr_err("IOMMU: can't reserve memory\n");
 622                         err = -EBUSY;
 623                         goto out;
 624                 }
 625                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 626                 if (!iommu->reg) {
 627                         pr_err("IOMMU: can't map the region\n");
 628                         err = -ENOMEM;
 629                         goto release;
 630                 }
 631         }
 632         err = 0;
 633         goto out;
 634
 635 unmap:
 636         iounmap(iommu->reg);
 637 release:
 638         release_mem_region(iommu->reg_phys, iommu->reg_size);
 639 out:
 640         return err;
 641 }
 642
 643 int alloc_iommu(struct dmar_drhd_unit *drhd)
 644 {
 645         struct intel_iommu *iommu;
 646         u32 ver;
 647         static int iommu_allocated = 0;
 648         int agaw = 0;
 649         int msagaw = 0;
 650         int err;
 651
 652         if (!drhd->reg_base_addr) {
 653                 warn_invalid_dmar(0, "");
 654                 return -EINVAL;
 655         }
 656
 657         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 658         if (!iommu)
 659                 return -ENOMEM;
 660
 661         iommu->seq_id = iommu_allocated++;
 662         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 663
 664         err = map_iommu(iommu, drhd->reg_base_addr);
 665         if (err) {
 666                 pr_err("IOMMU: failed to map %s\n", iommu->name);
 667                 goto error;
 668         }
 669
 670         err = -EINVAL;
 671         agaw = iommu_calculate_agaw(iommu);
 672         if (agaw < 0) {
 673                 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
 674                         iommu->seq_id);
 675                 goto err_unmap;
 676         }
 677         msagaw = iommu_calculate_max_sagaw(iommu);
 678         if (msagaw < 0) {
 679                 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 680                         iommu->seq_id);
 681                 goto err_unmap;
 682         }
 683         iommu->agaw = agaw;
 684         iommu->msagaw = msagaw;
 685
 686         iommu->node = -1;
 687
 688         ver = readl(iommu->reg + DMAR_VER_REG);
 689         pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 690                 iommu->seq_id,
 691                 (unsigned long long)drhd->reg_base_addr,
 692                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 693                 (unsigned long long)iommu->cap,
 694                 (unsigned long long)iommu->ecap);
 695
 696         raw_spin_lock_init(&iommu->register_lock);
 697
 698         drhd->iommu = iommu;
 699         return 0;
 700
 701  err_unmap:
 702         unmap_iommu(iommu);
 703  error:
 704         kfree(iommu);
 705         return err;
 706 }
 707
 708 void free_iommu(struct intel_iommu *iommu)
 709 {
 710         if (!iommu)
 711                 return;
 712
 713         free_dmar_iommu(iommu);
 714
 715         if (iommu->reg)
 716                 unmap_iommu(iommu);
 717
 718         kfree(iommu);
 719 }
 720
 721 /*
 722  * Reclaim all the submitted descriptors which have completed its work.
 723  */
 724 static inline void reclaim_free_desc(struct q_inval *qi)
 725 {
 726         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 727                qi->desc_status[qi->free_tail] == QI_ABORT) {
 728                 qi->desc_status[qi->free_tail] = QI_FREE;
 729                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 730                 qi->free_cnt++;
 731         }
 732 }
 733
 734 static int qi_check_fault(struct intel_iommu *iommu, int index)
 735 {
 736         u32 fault;
 737         int head, tail;
 738         struct q_inval *qi = iommu->qi;
 739         int wait_index = (index + 1) % QI_LENGTH;
 740
 741         if (qi->desc_status[wait_index] == QI_ABORT)
 742                 return -EAGAIN;
 743
 744         fault = readl(iommu->reg + DMAR_FSTS_REG);
 745
 746         /*
 747          * If IQE happens, the head points to the descriptor associated
 748          * with the error. No new descriptors are fetched until the IQE
 749          * is cleared.
 750          */
 751         if (fault & DMA_FSTS_IQE) {
 752                 head = readl(iommu->reg + DMAR_IQH_REG);
 753                 if ((head >> DMAR_IQ_SHIFT) == index) {
 754                         pr_err("VT-d detected invalid descriptor: "
 755                                 "low=%llx, high=%llx\n",
 756                                 (unsigned long long)qi->desc[index].low,
 757                                 (unsigned long long)qi->desc[index].high);
 758                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 759                                         sizeof(struct qi_desc));
 760                         __iommu_flush_cache(iommu, &qi->desc[index],
 761                                         sizeof(struct qi_desc));
 762                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 763                         return -EINVAL;
 764                 }
 765         }
 766
 767         /*
 768          * If ITE happens, all pending wait_desc commands are aborted.
 769          * No new descriptors are fetched until the ITE is cleared.
 770          */
 771         if (fault & DMA_FSTS_ITE) {
 772                 head = readl(iommu->reg + DMAR_IQH_REG);
 773                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 774                 head |= 1;
 775                 tail = readl(iommu->reg + DMAR_IQT_REG);
 776                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 777
 778                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 779
 780                 do {
 781                         if (qi->desc_status[head] == QI_IN_USE)
 782                                 qi->desc_status[head] = QI_ABORT;
 783                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 784                 } while (head != tail);
 785
 786                 if (qi->desc_status[wait_index] == QI_ABORT)
 787                         return -EAGAIN;
 788         }
 789
 790         if (fault & DMA_FSTS_ICE)
 791                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 792
 793         return 0;
 794 }
 795
 796 /*
 797  * Submit the queued invalidation descriptor to the remapping
 798  * hardware unit and wait for its completion.
 799  */
 800 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 801 {
 802         int rc;
 803         struct q_inval *qi = iommu->qi;
 804         struct qi_desc *hw, wait_desc;
 805         int wait_index, index;
 806         unsigned long flags;
 807
 808         if (!qi)
 809                 return 0;
 810
 811         hw = qi->desc;
 812
 813 restart:
 814         rc = 0;
 815
 816         raw_spin_lock_irqsave(&qi->q_lock, flags);
 817         while (qi->free_cnt < 3) {
 818                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 819                 cpu_relax();
 820                 raw_spin_lock_irqsave(&qi->q_lock, flags);
 821         }
 822
 823         index = qi->free_head;
 824         wait_index = (index + 1) % QI_LENGTH;
 825
 826         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 827
 828         hw[index] = *desc;
 829
 830         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 831                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 832         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 833
 834         hw[wait_index] = wait_desc;
 835
 836         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 837         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 838
 839         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 840         qi->free_cnt -= 2;
 841
 842         /*
 843          * update the HW tail register indicating the presence of
 844          * new descriptors.
 845          */
 846         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 847
 848         while (qi->desc_status[wait_index] != QI_DONE) {
 849                 /*
 850                  * We will leave the interrupts disabled, to prevent interrupt
 851                  * context to queue another cmd while a cmd is already submitted
 852                  * and waiting for completion on this cpu. This is to avoid
 853                  * a deadlock where the interrupt context can wait indefinitely
 854                  * for free slots in the queue.
 855                  */
 856                 rc = qi_check_fault(iommu, index);
 857                 if (rc)
 858                         break;
 859
 860                 raw_spin_unlock(&qi->q_lock);
 861                 cpu_relax();
 862                 raw_spin_lock(&qi->q_lock);
 863         }
 864
 865         qi->desc_status[index] = QI_DONE;
 866
 867         reclaim_free_desc(qi);
 868         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 869
 870         if (rc == -EAGAIN)
 871                 goto restart;
 872
 873         return rc;
 874 }
 875
 876 /*
 877  * Flush the global interrupt entry cache.
 878  */
 879 void qi_global_iec(struct intel_iommu *iommu)
 880 {
 881         struct qi_desc desc;
 882
 883         desc.low = QI_IEC_TYPE;
 884         desc.high = 0;
 885
 886         /* should never fail */
 887         qi_submit_sync(&desc, iommu);
 888 }
 889
 890 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 891                       u64 type)
 892 {
 893         struct qi_desc desc;
 894
 895         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 896                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 897         desc.high = 0;
 898
 899         qi_submit_sync(&desc, iommu);
 900 }
 901
 902 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 903                     unsigned int size_order, u64 type)
 904 {
 905         u8 dw = 0, dr = 0;
 906
 907         struct qi_desc desc;
 908         int ih = 0;
 909
 910         if (cap_write_drain(iommu->cap))
 911                 dw = 1;
 912
 913         if (cap_read_drain(iommu->cap))
 914                 dr = 1;
 915
 916         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 917                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 918         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 919                 | QI_IOTLB_AM(size_order);
 920
 921         qi_submit_sync(&desc, iommu);
 922 }
 923
 924 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 925                         u64 addr, unsigned mask)
 926 {
 927         struct qi_desc desc;
 928
 929         if (mask) {
 930                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
 931                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 932                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 933         } else
 934                 desc.high = QI_DEV_IOTLB_ADDR(addr);
 935
 936         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 937                 qdep = 0;
 938
 939         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 940                    QI_DIOTLB_TYPE;
 941
 942         qi_submit_sync(&desc, iommu);
 943 }
 944
 945 /*
 946  * Disable Queued Invalidation interface.
 947  */
 948 void dmar_disable_qi(struct intel_iommu *iommu)
 949 {
 950         unsigned long flags;
 951         u32 sts;
 952         cycles_t start_time = get_cycles();
 953
 954         if (!ecap_qis(iommu->ecap))
 955                 return;
 956
 957         raw_spin_lock_irqsave(&iommu->register_lock, flags);
 958
 959         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 960         if (!(sts & DMA_GSTS_QIES))
 961                 goto end;
 962
 963         /*
 964          * Give a chance to HW to complete the pending invalidation requests.
 965          */
 966         while ((readl(iommu->reg + DMAR_IQT_REG) !=
 967                 readl(iommu->reg + DMAR_IQH_REG)) &&
 968                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 969                 cpu_relax();
 970
 971         iommu->gcmd &= ~DMA_GCMD_QIE;
 972         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 973
 974         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 975                       !(sts & DMA_GSTS_QIES), sts);
 976 end:
 977         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
 978 }
 979
 980 /*
 981  * Enable queued invalidation.
 982  */
 983 static void __dmar_enable_qi(struct intel_iommu *iommu)
 984 {
 985         u32 sts;
 986         unsigned long flags;
 987         struct q_inval *qi = iommu->qi;
 988
 989         qi->free_head = qi->free_tail = 0;
 990         qi->free_cnt = QI_LENGTH;
 991
 992         raw_spin_lock_irqsave(&iommu->register_lock, flags);
 993
 994         /* write zero to the tail reg */
 995         writel(0, iommu->reg + DMAR_IQT_REG);
 996
 997         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 998
 999         iommu->gcmd |= DMA_GCMD_QIE;
1000         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1001
1002         /* Make sure hardware complete it */
1003         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1004
1005         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1006 }
1007
1008 /*
1009  * Enable Queued Invalidation interface. This is a must to support
1010  * interrupt-remapping. Also used by DMA-remapping, which replaces
1011  * register based IOTLB invalidation.
1012  */
1013 int dmar_enable_qi(struct intel_iommu *iommu)
1014 {
1015         struct q_inval *qi;
1016         struct page *desc_page;
1017
1018         if (!ecap_qis(iommu->ecap))
1019                 return -ENOENT;
1020
1021         /*
1022          * queued invalidation is already setup and enabled.
1023          */
1024         if (iommu->qi)
1025                 return 0;
1026
1027         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1028         if (!iommu->qi)
1029                 return -ENOMEM;
1030
1031         qi = iommu->qi;
1032
1033
1034         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1035         if (!desc_page) {
1036                 kfree(qi);
1037                 iommu->qi = 0;
1038                 return -ENOMEM;
1039         }
1040
1041         qi->desc = page_address(desc_page);
1042
1043         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1044         if (!qi->desc_status) {
1045                 free_page((unsigned long) qi->desc);
1046                 kfree(qi);
1047                 iommu->qi = 0;
1048                 return -ENOMEM;
1049         }
1050
1051         qi->free_head = qi->free_tail = 0;
1052         qi->free_cnt = QI_LENGTH;
1053
1054         raw_spin_lock_init(&qi->q_lock);
1055
1056         __dmar_enable_qi(iommu);
1057
1058         return 0;
1059 }
1060
1061 /* iommu interrupt handling. Most stuff are MSI-like. */
1062
1063 enum faulttype {
1064         DMA_REMAP,
1065         INTR_REMAP,
1066         UNKNOWN,
1067 };
1068
1069 static const char *dma_remap_fault_reasons[] =
1070 {
1071         "Software",
1072         "Present bit in root entry is clear",
1073         "Present bit in context entry is clear",
1074         "Invalid context entry",
1075         "Access beyond MGAW",
1076         "PTE Write access is not set",
1077         "PTE Read access is not set",
1078         "Next page table ptr is invalid",
1079         "Root table address invalid",
1080         "Context table ptr is invalid",
1081         "non-zero reserved fields in RTP",
1082         "non-zero reserved fields in CTP",
1083         "non-zero reserved fields in PTE",
1084 };
1085
1086 static const char *irq_remap_fault_reasons[] =
1087 {
1088         "Detected reserved fields in the decoded interrupt-remapped request",
1089         "Interrupt index exceeded the interrupt-remapping table size",
1090         "Present field in the IRTE entry is clear",
1091         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1092         "Detected reserved fields in the IRTE entry",
1093         "Blocked a compatibility format interrupt request",
1094         "Blocked an interrupt request due to source-id verification failure",
1095 };
1096
1097 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1098
1099 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1100 {
1101         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1102                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1103                 *fault_type = INTR_REMAP;
1104                 return irq_remap_fault_reasons[fault_reason - 0x20];
1105         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1106                 *fault_type = DMA_REMAP;
1107                 return dma_remap_fault_reasons[fault_reason];
1108         } else {
1109                 *fault_type = UNKNOWN;
1110                 return "Unknown";
1111         }
1112 }
1113
1114 void dmar_msi_unmask(struct irq_data *data)
1115 {
1116         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1117         unsigned long flag;
1118
1119         /* unmask it */
1120         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1121         writel(0, iommu->reg + DMAR_FECTL_REG);
1122         /* Read a reg to force flush the post write */
1123         readl(iommu->reg + DMAR_FECTL_REG);
1124         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1125 }
1126
1127 void dmar_msi_mask(struct irq_data *data)
1128 {
1129         unsigned long flag;
1130         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1131
1132         /* mask it */
1133         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1134         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1135         /* Read a reg to force flush the post write */
1136         readl(iommu->reg + DMAR_FECTL_REG);
1137         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1138 }
1139
1140 void dmar_msi_write(int irq, struct msi_msg *msg)
1141 {
1142         struct intel_iommu *iommu = irq_get_handler_data(irq);
1143         unsigned long flag;
1144
1145         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1146         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1147         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1148         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1149         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1150 }
1151
1152 void dmar_msi_read(int irq, struct msi_msg *msg)
1153 {
1154         struct intel_iommu *iommu = irq_get_handler_data(irq);
1155         unsigned long flag;
1156
1157         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1158         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1159         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1160         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1161         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1162 }
1163
1164 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1165                 u8 fault_reason, u16 source_id, unsigned long long addr)
1166 {
1167         const char *reason;
1168         int fault_type;
1169
1170         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1171
1172         if (fault_type == INTR_REMAP)
1173                 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1174                        "fault index %llx\n"
1175                         "INTR-REMAP:[fault reason %02d] %s\n",
1176                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1177                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1178                         fault_reason, reason);
1179         else
1180                 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1181                        "fault addr %llx \n"
1182                        "DMAR:[fault reason %02d] %s\n",
1183                        (type ? "DMA Read" : "DMA Write"),
1184                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1185                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1186         return 0;
1187 }
1188
1189 #define PRIMARY_FAULT_REG_LEN (16)
1190 irqreturn_t dmar_fault(int irq, void *dev_id)
1191 {
1192         struct intel_iommu *iommu = dev_id;
1193         int reg, fault_index;
1194         u32 fault_status;
1195         unsigned long flag;
1196
1197         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1198         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1199         if (fault_status)
1200                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1201
1202         /* TBD: ignore advanced fault log currently */
1203         if (!(fault_status & DMA_FSTS_PPF))
1204                 goto clear_rest;
1205
1206         fault_index = dma_fsts_fault_record_index(fault_status);
1207         reg = cap_fault_reg_offset(iommu->cap);
1208         while (1) {
1209                 u8 fault_reason;
1210                 u16 source_id;
1211                 u64 guest_addr;
1212                 int type;
1213                 u32 data;
1214
1215                 /* highest 32 bits */
1216                 data = readl(iommu->reg + reg +
1217                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1218                 if (!(data & DMA_FRCD_F))
1219                         break;
1220
1221                 fault_reason = dma_frcd_fault_reason(data);
1222                 type = dma_frcd_type(data);
1223
1224                 data = readl(iommu->reg + reg +
1225                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1226                 source_id = dma_frcd_source_id(data);
1227
1228                 guest_addr = dmar_readq(iommu->reg + reg +
1229                                 fault_index * PRIMARY_FAULT_REG_LEN);
1230                 guest_addr = dma_frcd_page_addr(guest_addr);
1231                 /* clear the fault */
1232                 writel(DMA_FRCD_F, iommu->reg + reg +
1233                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1234
1235                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1236
1237                 dmar_fault_do_one(iommu, type, fault_reason,
1238                                 source_id, guest_addr);
1239
1240                 fault_index++;
1241                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1242                         fault_index = 0;
1243                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1244         }
1245 clear_rest:
1246         /* clear all the other faults */
1247         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1248         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1249
1250         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1251         return IRQ_HANDLED;
1252 }
1253
1254 int dmar_set_interrupt(struct intel_iommu *iommu)
1255 {
1256         int irq, ret;
1257
1258         /*
1259          * Check if the fault interrupt is already initialized.
1260          */
1261         if (iommu->irq)
1262                 return 0;
1263
1264         irq = create_irq();
1265         if (!irq) {
1266                 pr_err("IOMMU: no free vectors\n");
1267                 return -EINVAL;
1268         }
1269
1270         irq_set_handler_data(irq, iommu);
1271         iommu->irq = irq;
1272
1273         ret = arch_setup_dmar_msi(irq);
1274         if (ret) {
1275                 irq_set_handler_data(irq, NULL);
1276                 iommu->irq = 0;
1277                 destroy_irq(irq);
1278                 return ret;
1279         }
1280
1281         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1282         if (ret)
1283                 pr_err("IOMMU: can't request irq\n");
1284         return ret;
1285 }
1286
1287 int __init enable_drhd_fault_handling(void)
1288 {
1289         struct dmar_drhd_unit *drhd;
1290
1291         /*
1292          * Enable fault control interrupt.
1293          */
1294         for_each_drhd_unit(drhd) {
1295                 int ret;
1296                 struct intel_iommu *iommu = drhd->iommu;
1297                 ret = dmar_set_interrupt(iommu);
1298
1299                 if (ret) {
1300                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1301                                (unsigned long long)drhd->reg_base_addr, ret);
1302                         return -1;
1303                 }
1304
1305                 /*
1306                  * Clear any previous faults.
1307                  */
1308                 dmar_fault(iommu->irq, iommu);
1309         }
1310
1311         return 0;
1312 }
1313
1314 /*
1315  * Re-enable Queued Invalidation interface.
1316  */
1317 int dmar_reenable_qi(struct intel_iommu *iommu)
1318 {
1319         if (!ecap_qis(iommu->ecap))
1320                 return -ENOENT;
1321
1322         if (!iommu->qi)
1323                 return -ENOENT;
1324
1325         /*
1326          * First disable queued invalidation.
1327          */
1328         dmar_disable_qi(iommu);
1329         /*
1330          * Then enable queued invalidation again. Since there is no pending
1331          * invalidation requests now, it's safe to re-enable queued
1332          * invalidation.
1333          */
1334         __dmar_enable_qi(iommu);
1335
1336         return 0;
1337 }
1338
1339 /*
1340  * Check interrupt remapping support in DMAR table description.
1341  */
1342 int __init dmar_ir_support(void)
1343 {
1344         struct acpi_table_dmar *dmar;
1345         dmar = (struct acpi_table_dmar *)dmar_tbl;
1346         if (!dmar)
1347                 return 0;
1348         return dmar->flags & 0x1;
1349 }
1350 IOMMU_INIT_POST(detect_intel_iommu);