#include "amd_iommu_types.h"
#include "irq_remapping.h"
+#define AMD_IOMMU_MAPPING_ERROR 0
+
#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
#define LOOP_TIMEOUT 100000
LIST_HEAD(hpet_map);
LIST_HEAD(acpihid_map);
- #define FLUSH_QUEUE_SIZE 256
-
- struct flush_queue_entry {
- unsigned long iova_pfn;
- unsigned long pages;
- struct dma_ops_domain *dma_dom;
- };
-
- struct flush_queue {
- spinlock_t lock;
- unsigned next;
- struct flush_queue_entry *entries;
- };
-
- static DEFINE_PER_CPU(struct flush_queue, flush_queue);
-
- static atomic_t queue_timer_on;
- static struct timer_list queue_timer;
-
/*
* Domain for untranslated devices - only allocated
* if iommu=pt passed on kernel cmd line.
PPR completions */
u32 errata; /* Bitmap for errata to apply */
bool use_vapic; /* Enable device to use vapic mode */
+
+ struct ratelimit_state rs; /* Ratelimit IOPF messages */
};
/*
static int protection_domain_init(struct protection_domain *domain);
static void detach_device(struct device *dev);
+ #define FLUSH_QUEUE_SIZE 256
+
+ struct flush_queue_entry {
+ unsigned long iova_pfn;
+ unsigned long pages;
+ u64 counter; /* Flush counter when this entry was added to the queue */
+ };
+
+ struct flush_queue {
+ struct flush_queue_entry *entries;
+ unsigned head, tail;
+ spinlock_t lock;
+ };
+
/*
* Data container for a dma_ops specific protection domain
*/
/* IOVA RB-Tree */
struct iova_domain iovad;
+
+ struct flush_queue __percpu *flush_queue;
+
+ /*
+ * We need two counter here to be race-free wrt. IOTLB flushing and
+ * adding entries to the flush queue.
+ *
+ * The flush_start_cnt is incremented _before_ the IOTLB flush starts.
+ * New entries added to the flush ring-buffer get their 'counter' value
+ * from here. This way we can make sure that entries added to the queue
+ * (or other per-cpu queues of the same domain) while the TLB is about
+ * to be flushed are not considered to be flushed already.
+ */
+ atomic64_t flush_start_cnt;
+
+ /*
+ * The flush_finish_cnt is incremented when an IOTLB flush is complete.
+ * This value is always smaller than flush_start_cnt. The queue_add
+ * function frees all IOVAs that have a counter value smaller than
+ * flush_finish_cnt. This makes sure that we only free IOVAs that are
+ * flushed out of the IOTLB of the domain.
+ */
+ atomic64_t flush_finish_cnt;
+
+ /*
+ * Timer to make sure we don't keep IOVAs around unflushed
+ * for too long
+ */
+ struct timer_list flush_timer;
+ atomic_t flush_timer_on;
};
static struct iova_domain reserved_iova_ranges;
list_add_tail(&dev_data->dev_data_list, &dev_data_list);
spin_unlock_irqrestore(&dev_data_list_lock, flags);
+ ratelimit_default_init(&dev_data->rs);
+
return dev_data;
}
pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
}
+ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id,
+ u64 address, int flags)
+ {
+ struct iommu_dev_data *dev_data = NULL;
+ struct pci_dev *pdev;
+
+ pdev = pci_get_bus_and_slot(PCI_BUS_NUM(devid), devid & 0xff);
+ if (pdev)
+ dev_data = get_dev_data(&pdev->dev);
+
+ if (dev_data && __ratelimit(&dev_data->rs)) {
+ dev_err(&pdev->dev, "AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%016llx flags=0x%04x]\n",
+ domain_id, address, flags);
+ } else if (printk_ratelimit()) {
+ pr_err("AMD-Vi: Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n",
+ PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+ domain_id, address, flags);
+ }
+
+ if (pdev)
+ pci_dev_put(pdev);
+ }
+
static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
{
int type, devid, domid, flags;
goto retry;
}
- printk(KERN_ERR "AMD-Vi: Event logged [");
+ if (type == EVENT_TYPE_IO_FAULT) {
+ amd_iommu_report_page_fault(devid, domid, address, flags);
+ return;
+ } else {
+ printk(KERN_ERR "AMD-Vi: Event logged [");
+ }
switch (type) {
case EVENT_TYPE_ILL_DEV:
address, flags);
dump_dte_entry(devid);
break;
- case EVENT_TYPE_IO_FAULT:
- printk("IO_PAGE_FAULT device=%02x:%02x.%x "
- "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
- PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
- domid, address, flags);
- break;
case EVENT_TYPE_DEV_TAB_ERR:
printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
"address=0x%016llx flags=0x%04x]\n",
}
static void copy_cmd_to_buffer(struct amd_iommu *iommu,
- struct iommu_cmd *cmd,
- u32 tail)
+ struct iommu_cmd *cmd)
{
u8 *target;
- target = iommu->cmd_buf + tail;
- tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
+ target = iommu->cmd_buf + iommu->cmd_buf_tail;
+
+ iommu->cmd_buf_tail += sizeof(*cmd);
+ iommu->cmd_buf_tail %= CMD_BUFFER_SIZE;
/* Copy command to buffer */
memcpy(target, cmd, sizeof(*cmd));
/* Tell the IOMMU about it */
- writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+ writel(iommu->cmd_buf_tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
}
static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
struct iommu_cmd *cmd,
bool sync)
{
- u32 left, tail, head, next_tail;
+ unsigned int count = 0;
+ u32 left, next_tail;
+ next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
again:
-
- head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
- tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
- next_tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
- left = (head - next_tail) % CMD_BUFFER_SIZE;
+ left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;
if (left <= 0x20) {
- struct iommu_cmd sync_cmd;
- int ret;
-
- iommu->cmd_sem = 0;
+ /* Skip udelay() the first time around */
+ if (count++) {
+ if (count == LOOP_TIMEOUT) {
+ pr_err("AMD-Vi: Command buffer timeout\n");
+ return -EIO;
+ }
- build_completion_wait(&sync_cmd, (u64)&iommu->cmd_sem);
- copy_cmd_to_buffer(iommu, &sync_cmd, tail);
+ udelay(1);
+ }
- if ((ret = wait_on_sem(&iommu->cmd_sem)) != 0)
- return ret;
+ /* Update head and recheck remaining space */
+ iommu->cmd_buf_head = readl(iommu->mmio_base +
+ MMIO_CMD_HEAD_OFFSET);
goto again;
}
- copy_cmd_to_buffer(iommu, cmd, tail);
+ copy_cmd_to_buffer(iommu, cmd);
- /* We need to sync now to make sure all commands are processed */
+ /* Do we need to make sure all commands are processed? */
iommu->need_sync = sync;
return 0;
free_page((unsigned long)domain->gcr3_tbl);
}
+ static void dma_ops_domain_free_flush_queue(struct dma_ops_domain *dom)
+ {
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue;
+
+ queue = per_cpu_ptr(dom->flush_queue, cpu);
+ kfree(queue->entries);
+ }
+
+ free_percpu(dom->flush_queue);
+
+ dom->flush_queue = NULL;
+ }
+
+ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
+ {
+ int cpu;
+
+ atomic64_set(&dom->flush_start_cnt, 0);
+ atomic64_set(&dom->flush_finish_cnt, 0);
+
+ dom->flush_queue = alloc_percpu(struct flush_queue);
+ if (!dom->flush_queue)
+ return -ENOMEM;
+
+ /* First make sure everything is cleared */
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue;
+
+ queue = per_cpu_ptr(dom->flush_queue, cpu);
+ queue->head = 0;
+ queue->tail = 0;
+ queue->entries = NULL;
+ }
+
+ /* Now start doing the allocation */
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue;
+
+ queue = per_cpu_ptr(dom->flush_queue, cpu);
+ queue->entries = kzalloc(FLUSH_QUEUE_SIZE * sizeof(*queue->entries),
+ GFP_KERNEL);
+ if (!queue->entries) {
+ dma_ops_domain_free_flush_queue(dom);
+ return -ENOMEM;
+ }
+
+ spin_lock_init(&queue->lock);
+ }
+
+ return 0;
+ }
+
+ static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
+ {
+ atomic64_inc(&dom->flush_start_cnt);
+ domain_flush_tlb(&dom->domain);
+ domain_flush_complete(&dom->domain);
+ atomic64_inc(&dom->flush_finish_cnt);
+ }
+
+ static inline bool queue_ring_full(struct flush_queue *queue)
+ {
+ assert_spin_locked(&queue->lock);
+
+ return (((queue->tail + 1) % FLUSH_QUEUE_SIZE) == queue->head);
+ }
+
+ #define queue_ring_for_each(i, q) \
+ for (i = (q)->head; i != (q)->tail; i = (i + 1) % FLUSH_QUEUE_SIZE)
+
+ static inline unsigned queue_ring_add(struct flush_queue *queue)
+ {
+ unsigned idx = queue->tail;
+
+ assert_spin_locked(&queue->lock);
+ queue->tail = (idx + 1) % FLUSH_QUEUE_SIZE;
+
+ return idx;
+ }
+
+ static inline void queue_ring_remove_head(struct flush_queue *queue)
+ {
+ assert_spin_locked(&queue->lock);
+ queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
+ }
+
+ static void queue_ring_free_flushed(struct dma_ops_domain *dom,
+ struct flush_queue *queue)
+ {
+ u64 counter = atomic64_read(&dom->flush_finish_cnt);
+ int idx;
+
+ queue_ring_for_each(idx, queue) {
+ /*
+ * This assumes that counter values in the ring-buffer are
+ * monotonously rising.
+ */
+ if (queue->entries[idx].counter >= counter)
+ break;
+
+ free_iova_fast(&dom->iovad,
+ queue->entries[idx].iova_pfn,
+ queue->entries[idx].pages);
+
+ queue_ring_remove_head(queue);
+ }
+ }
+
+ static void queue_add(struct dma_ops_domain *dom,
+ unsigned long address, unsigned long pages)
+ {
+ struct flush_queue *queue;
+ unsigned long flags;
+ int idx;
+
+ pages = __roundup_pow_of_two(pages);
+ address >>= PAGE_SHIFT;
+
+ queue = get_cpu_ptr(dom->flush_queue);
+ spin_lock_irqsave(&queue->lock, flags);
+
+ /*
+ * First remove the enries from the ring-buffer that are already
+ * flushed to make the below queue_ring_full() check less likely
+ */
+ queue_ring_free_flushed(dom, queue);
+
+ /*
+ * When ring-queue is full, flush the entries from the IOTLB so
+ * that we can free all entries with queue_ring_free_flushed()
+ * below.
+ */
+ if (queue_ring_full(queue)) {
+ dma_ops_domain_flush_tlb(dom);
+ queue_ring_free_flushed(dom, queue);
+ }
+
+ idx = queue_ring_add(queue);
+
+ queue->entries[idx].iova_pfn = address;
+ queue->entries[idx].pages = pages;
+ queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt);
+
+ spin_unlock_irqrestore(&queue->lock, flags);
+
+ if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
+ mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
+
+ put_cpu_ptr(dom->flush_queue);
+ }
+
+ static void queue_flush_timeout(unsigned long data)
+ {
+ struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
+ int cpu;
+
+ atomic_set(&dom->flush_timer_on, 0);
+
+ dma_ops_domain_flush_tlb(dom);
+
+ for_each_possible_cpu(cpu) {
+ struct flush_queue *queue;
+ unsigned long flags;
+
+ queue = per_cpu_ptr(dom->flush_queue, cpu);
+ spin_lock_irqsave(&queue->lock, flags);
+ queue_ring_free_flushed(dom, queue);
+ spin_unlock_irqrestore(&queue->lock, flags);
+ }
+ }
+
/*
* Free a domain, only used if something went wrong in the
* allocation path and we need to free an already allocated page table
del_domain_from_list(&dom->domain);
+ if (timer_pending(&dom->flush_timer))
+ del_timer(&dom->flush_timer);
+
+ dma_ops_domain_free_flush_queue(dom);
+
put_iova_domain(&dom->iovad);
free_pagetable(&dom->domain);
/* Initialize reserved ranges */
copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
+ if (dma_ops_domain_alloc_flush_queue(dma_dom))
+ goto free_dma_dom;
+
+ setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
+ (unsigned long)dma_dom);
+
+ atomic_set(&dma_dom->flush_timer_on, 0);
+
add_domain_to_list(&dma_dom->domain);
return dma_dom;
flags |= tmp;
}
- flags &= ~(0xffffUL);
+
+ flags &= ~(DTE_FLAG_SA | 0xffffULL);
flags |= domain->id;
amd_iommu_dev_table[devid].data[1] = flags;
*
*****************************************************************************/
- static void __queue_flush(struct flush_queue *queue)
- {
- struct protection_domain *domain;
- unsigned long flags;
- int idx;
-
- /* First flush TLB of all known domains */
- spin_lock_irqsave(&amd_iommu_pd_lock, flags);
- list_for_each_entry(domain, &amd_iommu_pd_list, list)
- domain_flush_tlb(domain);
- spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
-
- /* Wait until flushes have completed */
- domain_flush_complete(NULL);
-
- for (idx = 0; idx < queue->next; ++idx) {
- struct flush_queue_entry *entry;
-
- entry = queue->entries + idx;
-
- free_iova_fast(&entry->dma_dom->iovad,
- entry->iova_pfn,
- entry->pages);
-
- /* Not really necessary, just to make sure we catch any bugs */
- entry->dma_dom = NULL;
- }
-
- queue->next = 0;
- }
-
- static void queue_flush_all(void)
- {
- int cpu;
-
- for_each_possible_cpu(cpu) {
- struct flush_queue *queue;
- unsigned long flags;
-
- queue = per_cpu_ptr(&flush_queue, cpu);
- spin_lock_irqsave(&queue->lock, flags);
- if (queue->next > 0)
- __queue_flush(queue);
- spin_unlock_irqrestore(&queue->lock, flags);
- }
- }
-
- static void queue_flush_timeout(unsigned long unsused)
- {
- atomic_set(&queue_timer_on, 0);
- queue_flush_all();
- }
-
- static void queue_add(struct dma_ops_domain *dma_dom,
- unsigned long address, unsigned long pages)
- {
- struct flush_queue_entry *entry;
- struct flush_queue *queue;
- unsigned long flags;
- int idx;
-
- pages = __roundup_pow_of_two(pages);
- address >>= PAGE_SHIFT;
-
- queue = get_cpu_ptr(&flush_queue);
- spin_lock_irqsave(&queue->lock, flags);
-
- if (queue->next == FLUSH_QUEUE_SIZE)
- __queue_flush(queue);
-
- idx = queue->next++;
- entry = queue->entries + idx;
-
- entry->iova_pfn = address;
- entry->pages = pages;
- entry->dma_dom = dma_dom;
-
- spin_unlock_irqrestore(&queue->lock, flags);
-
- if (atomic_cmpxchg(&queue_timer_on, 0, 1) == 0)
- mod_timer(&queue_timer, jiffies + msecs_to_jiffies(10));
-
- put_cpu_ptr(&flush_queue);
- }
-
-
/*
* In the dma_ops path we only have the struct device. This function
* finds the corresponding IOMMU, the protection domain and the
paddr &= PAGE_MASK;
address = dma_ops_alloc_iova(dev, dma_dom, pages, dma_mask);
- if (address == DMA_ERROR_CODE)
+ if (address == AMD_IOMMU_MAPPING_ERROR)
goto out;
prot = dir2prot(direction);
dma_ops_free_iova(dma_dom, address, pages);
- return DMA_ERROR_CODE;
+ return AMD_IOMMU_MAPPING_ERROR;
}
/*
if (PTR_ERR(domain) == -EINVAL)
return (dma_addr_t)paddr;
else if (IS_ERR(domain))
- return DMA_ERROR_CODE;
+ return AMD_IOMMU_MAPPING_ERROR;
dma_mask = *dev->dma_mask;
dma_dom = to_dma_ops_domain(domain);
npages = sg_num_pages(dev, sglist, nelems);
address = dma_ops_alloc_iova(dev, dma_dom, npages, dma_mask);
- if (address == DMA_ERROR_CODE)
+ if (address == AMD_IOMMU_MAPPING_ERROR)
goto out_err;
prot = dir2prot(direction);
*dma_addr = __map_single(dev, dma_dom, page_to_phys(page),
size, DMA_BIDIRECTIONAL, dma_mask);
- if (*dma_addr == DMA_ERROR_CODE)
+ if (*dma_addr == AMD_IOMMU_MAPPING_ERROR)
goto out_free;
return page_address(page);
*/
static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{
+ if (!x86_dma_supported(dev, mask))
+ return 0;
return check_device(dev);
}
+static int amd_iommu_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+ return dma_addr == AMD_IOMMU_MAPPING_ERROR;
+}
+
static const struct dma_map_ops amd_iommu_dma_ops = {
.alloc = alloc_coherent,
.free = free_coherent,
.map_sg = map_sg,
.unmap_sg = unmap_sg,
.dma_supported = amd_iommu_dma_supported,
+ .mapping_error = amd_iommu_mapping_error,
};
static int init_reserved_iova_ranges(void)
int __init amd_iommu_init_api(void)
{
- int ret, cpu, err = 0;
+ int ret, err = 0;
ret = iova_cache_get();
if (ret)
if (ret)
return ret;
- for_each_possible_cpu(cpu) {
- struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu);
-
- queue->entries = kzalloc(FLUSH_QUEUE_SIZE *
- sizeof(*queue->entries),
- GFP_KERNEL);
- if (!queue->entries)
- goto out_put_iova;
-
- spin_lock_init(&queue->lock);
- }
-
err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
if (err)
return err;
err = bus_set_iommu(&platform_bus_type, &amd_iommu_ops);
if (err)
return err;
- return 0;
-
- out_put_iova:
- for_each_possible_cpu(cpu) {
- struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu);
-
- kfree(queue->entries);
- }
- return -ENOMEM;
+ return 0;
}
int __init amd_iommu_init_dma_ops(void)
{
- setup_timer(&queue_timer, queue_flush_timeout, 0);
- atomic_set(&queue_timer_on, 0);
-
swiotlb = iommu_pass_through ? 1 : 0;
iommu_detected = 1;
switch (dom->type) {
case IOMMU_DOMAIN_DMA:
- /*
- * First make sure the domain is no longer referenced from the
- * flush queue
- */
- queue_flush_all();
-
/* Now release the domain */
dma_dom = to_dma_ops_domain(domain);
dma_ops_domain_free(dma_dom);
irte_info->index);
}
- static struct irq_domain_ops amd_ir_domain_ops = {
+ static const struct irq_domain_ops amd_ir_domain_ops = {
.alloc = irq_remapping_alloc,
.free = irq_remapping_free,
.activate = irq_remapping_activate,
}
static struct irq_chip amd_ir_chip = {
- .irq_ack = ir_ack_apic_edge,
- .irq_set_affinity = amd_ir_set_affinity,
- .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity,
- .irq_compose_msi_msg = ir_compose_msi_msg,
+ .name = "AMD-IR",
+ .irq_ack = ir_ack_apic_edge,
+ .irq_set_affinity = amd_ir_set_affinity,
+ .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity,
+ .irq_compose_msi_msg = ir_compose_msi_msg,
};
int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
{
- iommu->ir_domain = irq_domain_add_tree(NULL, &amd_ir_domain_ops, iommu);
+ struct fwnode_handle *fn;
+
+ fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index);
+ if (!fn)
+ return -ENOMEM;
+ iommu->ir_domain = irq_domain_create_tree(fn, &amd_ir_domain_ops, iommu);
+ irq_domain_free_fwnode(fn);
if (!iommu->ir_domain)
return -ENOMEM;
iommu->ir_domain->parent = arch_get_ir_parent_domain();
- iommu->msi_domain = arch_create_msi_irq_domain(iommu->ir_domain);
-
+ iommu->msi_domain = arch_create_remap_msi_irq_domain(iommu->ir_domain,
+ "AMD-IR-MSI",
+ iommu->index);
return 0;
}
projects / platform / kernel / linux-rpi.git / commitdiff