iommu/amd: Add support for X2APIC IOMMU interrupts

AMD IOMMU requires IntCapXT registers to be setup in order to generate
its own interrupts (for Event Log, PPR Log, and GA Log) with 32-bit
APIC destination ID. Without this support, AMD IOMMU MSI interrupts
will not be routed correctly when booting the system in X2APIC mode.

Cc: Joerg Roedel <joro@8bytes.org>
Fixes: 90fcffd9cf5e ('iommu/amd: Add support for IOMMU XT mode')
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index eb104c7..4413aa6 100644 (file)
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -23,6 +23,8 @@
 #include <linux/mem_encrypt.h>
 #include <asm/pci-direct.h>
 #include <asm/iommu.h>
+#include <asm/apic.h>
+#include <asm/msidef.h>
 #include <asm/gart.h>
 #include <asm/x86_init.h>
 #include <asm/iommu_table.h>
@@ -1920,6 +1922,90 @@ static int iommu_setup_msi(struct amd_iommu *iommu)
        return 0;
 }
 
+#define XT_INT_DEST_MODE(x)    (((x) & 0x1ULL) << 2)
+#define XT_INT_DEST_LO(x)      (((x) & 0xFFFFFFULL) << 8)
+#define XT_INT_VEC(x)          (((x) & 0xFFULL) << 32)
+#define XT_INT_DEST_HI(x)      ((((x) >> 24) & 0xFFULL) << 56)
+
+/**
+ * Setup the IntCapXT registers with interrupt routing information
+ * based on the PCI MSI capability block registers, accessed via
+ * MMIO MSI address low/hi and MSI data registers.
+ */
+static void iommu_update_intcapxt(struct amd_iommu *iommu)
+{
+       u64 val;
+       u32 addr_lo = readl(iommu->mmio_base + MMIO_MSI_ADDR_LO_OFFSET);
+       u32 addr_hi = readl(iommu->mmio_base + MMIO_MSI_ADDR_HI_OFFSET);
+       u32 data    = readl(iommu->mmio_base + MMIO_MSI_DATA_OFFSET);
+       bool dm     = (addr_lo >> MSI_ADDR_DEST_MODE_SHIFT) & 0x1;
+       u32 dest    = ((addr_lo >> MSI_ADDR_DEST_ID_SHIFT) & 0xFF);
+
+       if (x2apic_enabled())
+               dest |= MSI_ADDR_EXT_DEST_ID(addr_hi);
+
+       val = XT_INT_VEC(data & 0xFF) |
+             XT_INT_DEST_MODE(dm) |
+             XT_INT_DEST_LO(dest) |
+             XT_INT_DEST_HI(dest);
+
+       /**
+        * Current IOMMU implemtation uses the same IRQ for all
+        * 3 IOMMU interrupts.
+        */
+       writeq(val, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
+       writeq(val, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);
+       writeq(val, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);
+}
+
+static void _irq_notifier_notify(struct irq_affinity_notify *notify,
+                                const cpumask_t *mask)
+{
+       struct amd_iommu *iommu;
+
+       for_each_iommu(iommu) {
+               if (iommu->dev->irq == notify->irq) {
+                       iommu_update_intcapxt(iommu);
+                       break;
+               }
+       }
+}
+
+static void _irq_notifier_release(struct kref *ref)
+{
+}
+
+static int iommu_init_intcapxt(struct amd_iommu *iommu)
+{
+       int ret;
+       struct irq_affinity_notify *notify = &iommu->intcapxt_notify;
+
+       /**
+        * IntCapXT requires XTSup=1, which can be inferred
+        * amd_iommu_xt_mode.
+        */
+       if (amd_iommu_xt_mode != IRQ_REMAP_X2APIC_MODE)
+               return 0;
+
+       /**
+        * Also, we need to setup notifier to update the IntCapXT registers
+        * whenever the irq affinity is changed from user-space.
+        */
+       notify->irq = iommu->dev->irq;
+       notify->notify = _irq_notifier_notify,
+       notify->release = _irq_notifier_release,
+       ret = irq_set_affinity_notifier(iommu->dev->irq, notify);
+       if (ret) {
+               pr_err("Failed to register irq affinity notifier (devid=%#x, irq %d)\n",
+                      iommu->devid, iommu->dev->irq);
+               return ret;
+       }
+
+       iommu_update_intcapxt(iommu);
+       iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
+       return ret;
+}
+
 static int iommu_init_msi(struct amd_iommu *iommu)
 {
        int ret;
@@ -1936,6 +2022,10 @@ static int iommu_init_msi(struct amd_iommu *iommu)
                return ret;
 
 enable_faults:
+       ret = iommu_init_intcapxt(iommu);
+       if (ret)
+               return ret;
+
        iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
 
        if (iommu->ppr_log != NULL)

diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 52c35d5..64edd5a 100644 (file)
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -60,6 +60,12 @@
 #define MMIO_PPR_LOG_OFFSET    0x0038
 #define MMIO_GA_LOG_BASE_OFFSET        0x00e0
 #define MMIO_GA_LOG_TAIL_OFFSET        0x00e8
+#define MMIO_MSI_ADDR_LO_OFFSET        0x015C
+#define MMIO_MSI_ADDR_HI_OFFSET        0x0160
+#define MMIO_MSI_DATA_OFFSET   0x0164
+#define MMIO_INTCAPXT_EVT_OFFSET       0x0170
+#define MMIO_INTCAPXT_PPR_OFFSET       0x0178
+#define MMIO_INTCAPXT_GALOG_OFFSET     0x0180
 #define MMIO_CMD_HEAD_OFFSET   0x2000
 #define MMIO_CMD_TAIL_OFFSET   0x2008
 #define MMIO_EVT_HEAD_OFFSET   0x2010
@@ -150,6 +156,7 @@
 #define CONTROL_GALOG_EN        0x1CULL
 #define CONTROL_GAINT_EN        0x1DULL
 #define CONTROL_XT_EN           0x32ULL
+#define CONTROL_INTCAPXT_EN     0x33ULL
 
 #define CTRL_INV_TO_MASK       (7 << CONTROL_INV_TIMEOUT)
 #define CTRL_INV_TO_NONE       0
@@ -592,6 +599,8 @@ struct amd_iommu {
        /* DebugFS Info */
        struct dentry *debugfs;
 #endif
+       /* IRQ notifier for IntCapXT interrupt */
+       struct irq_affinity_notify intcapxt_notify;
 };
 
 static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)