config IRQ_GIC
bool
-config IRQ_CPU_OCTEON
- bool
-
config MIPS_BOARDS_GEN
bool
config CPU_CAVIUM_OCTEON
bool "Cavium Octeon processor"
depends on SYS_HAS_CPU_CAVIUM_OCTEON
- select IRQ_CPU
- select IRQ_CPU_OCTEON
select CPU_HAS_PREFETCH
select CPU_SUPPORTS_64BIT_KERNEL
select SYS_SUPPORTS_SMP
void __init board_setup(void)
{
unsigned long bcsr1, bcsr2;
- u32 pin_func;
bcsr1 = DB1000_BCSR_PHYS_ADDR;
bcsr2 = DB1000_BCSR_PHYS_ADDR + DB1000_BCSR_HEXLED_OFS;
- pin_func = 0;
-
#ifdef CONFIG_MIPS_DB1000
printk(KERN_INFO "AMD Alchemy Au1000/Db1000 Board\n");
#endif
/* Not valid for Au1550 */
#if defined(CONFIG_IRDA) && \
(defined(CONFIG_SOC_AU1000) || defined(CONFIG_SOC_AU1100))
- /* Set IRFIRSEL instead of GPIO15 */
- pin_func = au_readl(SYS_PINFUNC) | SYS_PF_IRF;
- au_writel(pin_func, SYS_PINFUNC);
- /* Power off until the driver is in use */
- bcsr_mod(BCSR_RESETS, BCSR_RESETS_IRDA_MODE_MASK,
- BCSR_RESETS_IRDA_MODE_OFF);
+ {
+ u32 pin_func;
+
+ /* Set IRFIRSEL instead of GPIO15 */
+ pin_func = au_readl(SYS_PINFUNC) | SYS_PF_IRF;
+ au_writel(pin_func, SYS_PINFUNC);
+ /* Power off until the driver is in use */
+ bcsr_mod(BCSR_RESETS, BCSR_RESETS_IRDA_MODE_MASK,
+ BCSR_RESETS_IRDA_MODE_OFF);
+ }
#endif
bcsr_write(BCSR_PCMCIA, 0); /* turn off PCMCIA power */
alchemy_gpio1_input_enable();
#ifdef CONFIG_MIPS_MIRAGE
- /* GPIO[20] is output */
- alchemy_gpio_direction_output(20, 0);
+ {
+ u32 pin_func;
- /* Set GPIO[210:208] instead of SSI_0 */
- pin_func = au_readl(SYS_PINFUNC) | SYS_PF_S0;
+ /* GPIO[20] is output */
+ alchemy_gpio_direction_output(20, 0);
- /* Set GPIO[215:211] for LEDs */
- pin_func |= 5 << 2;
+ /* Set GPIO[210:208] instead of SSI_0 */
+ pin_func = au_readl(SYS_PINFUNC) | SYS_PF_S0;
- /* Set GPIO[214:213] for more LEDs */
- pin_func |= 5 << 12;
+ /* Set GPIO[215:211] for LEDs */
+ pin_func |= 5 << 2;
- /* Set GPIO[207:200] instead of PCMCIA/LCD */
- pin_func |= SYS_PF_LCD | SYS_PF_PC;
- au_writel(pin_func, SYS_PINFUNC);
+ /* Set GPIO[214:213] for more LEDs */
+ pin_func |= 5 << 12;
- /*
- * Enable speaker amplifier. This should
- * be part of the audio driver.
- */
- alchemy_gpio_direction_output(209, 1);
+ /* Set GPIO[207:200] instead of PCMCIA/LCD */
+ pin_func |= SYS_PF_LCD | SYS_PF_PC;
+ au_writel(pin_func, SYS_PINFUNC);
- pm_power_off = mirage_power_off;
- _machine_halt = mirage_power_off;
- _machine_restart = (void(*)(char *))mips_softreset;
+ /*
+ * Enable speaker amplifier. This should
+ * be part of the audio driver.
+ */
+ alchemy_gpio_direction_output(209, 1);
+
+ pm_power_off = mirage_power_off;
+ _machine_halt = mirage_power_off;
+ _machine_restart = (void(*)(char *))mips_softreset;
+ }
#endif
#ifdef CONFIG_MIPS_BOSPORUS
prom_init_cmdline();
memsize_str = prom_getenv("memsize");
- if (!memsize_str)
+ if (!memsize_str || strict_strtoul(memsize_str, 0, &memsize))
memsize = 0x04000000;
- else
- strict_strtoul(memsize_str, 0, &memsize);
+
add_memory_region(0, memsize, BOOT_MEM_RAM);
}
int main(int argc, char *argv[])
{
+ unsigned long long vmlinux_size, vmlinux_load_addr, vmlinuz_load_addr;
struct stat sb;
- uint64_t vmlinux_size, vmlinux_load_addr, vmlinuz_load_addr;
if (argc != 3) {
fprintf(stderr, "Usage: %s <pathname> <vmlinux_load_addr>\n",
-config CAVIUM_OCTEON_SPECIFIC_OPTIONS
- bool "Enable Octeon specific options"
- depends on CPU_CAVIUM_OCTEON
- default "y"
+if CPU_CAVIUM_OCTEON
config CAVIUM_CN63XXP1
bool "Enable CN63XXP1 errata worarounds"
- depends on CAVIUM_OCTEON_SPECIFIC_OPTIONS
default "n"
help
The CN63XXP1 chip requires build time workarounds to
config CAVIUM_OCTEON_2ND_KERNEL
bool "Build the kernel to be used as a 2nd kernel on the same chip"
- depends on CAVIUM_OCTEON_SPECIFIC_OPTIONS
default "n"
help
This option configures this kernel to be linked at a different
config CAVIUM_OCTEON_HW_FIX_UNALIGNED
bool "Enable hardware fixups of unaligned loads and stores"
- depends on CAVIUM_OCTEON_SPECIFIC_OPTIONS
default "y"
help
Configure the Octeon hardware to automatically fix unaligned loads
config CAVIUM_OCTEON_CVMSEG_SIZE
int "Number of L1 cache lines reserved for CVMSEG memory"
- depends on CAVIUM_OCTEON_SPECIFIC_OPTIONS
range 0 54
default 1
help
config CAVIUM_OCTEON_LOCK_L2
bool "Lock often used kernel code in the L2"
- depends on CAVIUM_OCTEON_SPECIFIC_OPTIONS
default "y"
help
Enable locking parts of the kernel into the L2 cache.
config ARCH_SPARSEMEM_ENABLE
def_bool y
select SPARSEMEM_STATIC
- depends on CPU_CAVIUM_OCTEON
config CAVIUM_OCTEON_HELPER
def_bool y
config SWIOTLB
def_bool y
- depends on CPU_CAVIUM_OCTEON
select IOMMU_HELPER
select NEED_SG_DMA_LENGTH
+
+
+endif # CPU_CAVIUM_OCTEON
#define SMP_CACHE_SHIFT L1_CACHE_SHIFT
#define SMP_CACHE_BYTES L1_CACHE_BYTES
-#define __read_mostly __attribute__((__section__(".data.read_mostly")))
+#define __read_mostly __attribute__((__section__(".data..read_mostly")))
#endif /* _ASM_CACHE_H */
#ifndef __ASM_CEVT_R4K_H
#define __ASM_CEVT_R4K_H
+#include <linux/clockchips.h>
+#include <asm/time.h>
+
DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device);
void mips_event_handler(struct clock_event_device *dev);
static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep)
{
+ flush_tlb_mm(vma->vm_mm);
}
static inline int huge_pte_none(pte_t pte)
char kernel_crc[CRC_LEN];
/* 228-235: Unused at present */
char reserved1[8];
- /* 236-239: CRC32 of header excluding tagVersion */
+ /* 236-239: CRC32 of header excluding last 20 bytes */
char header_crc[CRC_LEN];
/* 240-255: Unused at present */
char reserved2[16];
*/
int vdma_remap(unsigned long laddr, unsigned long paddr, unsigned long size)
{
- int first, pages, npages;
+ int first, pages;
if (laddr > 0xffffff) {
if (vdma_debug)
return -EINVAL; /* invalid physical address */
}
- npages = pages =
- (((paddr & (VDMA_PAGESIZE - 1)) + size) >> 12) + 1;
+ pages = (((paddr & (VDMA_PAGESIZE - 1)) + size) >> 12) + 1;
first = laddr >> 12;
if (vdma_debug)
printk("vdma_remap: first=%x, pages=%x\n", first, pages);
static void jz4740_dma_chan_irq(struct jz4740_dma_chan *dma)
{
- uint32_t status;
-
- status = jz4740_dma_read(JZ_REG_DMA_STATUS_CTRL(dma->id));
+ (void) jz4740_dma_read(JZ_REG_DMA_STATUS_CTRL(dma->id));
jz4740_dma_write_mask(JZ_REG_DMA_STATUS_CTRL(dma->id), 0,
JZ_DMA_STATUS_CTRL_ENABLE | JZ_DMA_STATUS_CTRL_TRANSFER_DONE);
static struct clock_event_device jz4740_clockevent = {
.name = "jz4740-timer",
- .features = CLOCK_EVT_FEAT_PERIODIC,
+ .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
.set_next_event = jz4740_clockevent_set_next,
.set_mode = jz4740_clockevent_set_mode,
.rating = 200,
{
writel(BIT(16), jz4740_timer_base + JZ_REG_TIMER_STOP_CLEAR);
}
+EXPORT_SYMBOL_GPL(jz4740_timer_enable_watchdog);
void jz4740_timer_disable_watchdog(void)
{
writel(BIT(16), jz4740_timer_base + JZ_REG_TIMER_STOP_SET);
}
+EXPORT_SYMBOL_GPL(jz4740_timer_disable_watchdog);
void __init jz4740_timer_init(void)
{
#define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */
#define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */
+#define JUMP_RANGE_MASK ((1UL << 28) - 1)
#define INSN_NOP 0x00000000 /* nop */
#define INSN_JAL(addr) \
/* jal (ftrace_caller + 8), jump over the first two instruction */
buf = (u32 *)&insn_jal_ftrace_caller;
- uasm_i_jal(&buf, (FTRACE_ADDR + 8));
+ uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* j ftrace_graph_caller */
buf = (u32 *)&insn_j_ftrace_graph_caller;
- uasm_i_j(&buf, (unsigned long)ftrace_graph_caller);
+ uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK);
#endif
}
secure_computing(regs->regs[2]);
if (unlikely(current->audit_context) && entryexit)
- audit_syscall_exit(AUDITSC_RESULT(regs->regs[2]),
- regs->regs[2]);
+ audit_syscall_exit(AUDITSC_RESULT(regs->regs[7]),
+ -regs->regs[2]);
if (!(current->ptrace & PT_PTRACED))
goto out;
sys sys_ioprio_get 2 /* 4315 */
sys sys_utimensat 4
sys sys_signalfd 3
- sys sys_ni_syscall 0
+ sys sys_ni_syscall 0 /* was timerfd */
sys sys_eventfd 1
sys sys_fallocate 6 /* 4320 */
sys sys_timerfd_create 2
PTR sys_ioprio_get
PTR sys_utimensat /* 5275 */
PTR sys_signalfd
- PTR sys_ni_syscall
+ PTR sys_ni_syscall /* was timerfd */
PTR sys_eventfd
PTR sys_fallocate
PTR sys_timerfd_create /* 5280 */
PTR sys_ioprio_get
PTR compat_sys_utimensat
PTR compat_sys_signalfd /* 6280 */
- PTR sys_ni_syscall
+ PTR sys_ni_syscall /* was timerfd */
PTR sys_eventfd
PTR sys_fallocate
PTR sys_timerfd_create
PTR sys_ioprio_get /* 4315 */
PTR compat_sys_utimensat
PTR compat_sys_signalfd
- PTR sys_ni_syscall
+ PTR sys_ni_syscall /* was timerfd */
PTR sys_eventfd
PTR sys32_fallocate /* 4320 */
PTR sys_timerfd_create
INIT_TASK_DATA(PAGE_SIZE)
NOSAVE_DATA
CACHELINE_ALIGNED_DATA(1 << CONFIG_MIPS_L1_CACHE_SHIFT)
+ READ_MOSTLY_DATA(1 << CONFIG_MIPS_L1_CACHE_SHIFT)
DATA_DATA
CONSTRUCTORS
}
#define parse_even_earlier(res, option, p) \
do { \
- int ret; \
+ unsigned int tmp __maybe_unused; \
+ \
if (strncmp(option, (char *)p, strlen(option)) == 0) \
- ret = strict_strtol((char *)p + strlen(option"="), 10, &res); \
+ tmp = strict_strtol((char *)p + strlen(option"="), 10, &res); \
} while (0)
void __init prom_init_env(void)
unsigned long flags, addr, begin, end, pow2;
unsigned int config = read_c0_config();
struct cpuinfo_mips *c = ¤t_cpu_data;
- int tmp;
if (config & CONF_SC)
return 0;
/* Now search for the wrap around point. */
pow2 = (128 * 1024);
- tmp = 0;
for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
cache_op(Index_Load_Tag_SD, addr);
__asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
struct uasm_reloc *r = relocs;
u32 *f;
unsigned int final_len;
- struct mips_huge_tlb_info htlb_info;
- enum vmalloc64_mode vmalloc_mode;
+ struct mips_huge_tlb_info htlb_info __maybe_unused;
+ enum vmalloc64_mode vmalloc_mode __maybe_unused;
memset(tlb_handler, 0, sizeof(tlb_handler));
memset(labels, 0, sizeof(labels));
void __init prom_init(void)
{
- int result;
-
prom_argc = fw_arg0;
_prom_argv = (int *) fw_arg1;
_prom_envp = (int *) fw_arg2;
#ifdef CONFIG_SERIAL_8250_CONSOLE
console_config();
#endif
- /* Early detection of CMP support */
- result = gcmp_probe(GCMP_BASE_ADDR, GCMP_ADDRSPACE_SZ);
-
#ifdef CONFIG_MIPS_CMP
- if (result)
+ /* Early detection of CMP support */
+ if (gcmp_probe(GCMP_BASE_ADDR, GCMP_ADDRSPACE_SZ))
register_smp_ops(&cmp_smp_ops);
+ else
#endif
#ifdef CONFIG_MIPS_MT_SMP
-#ifdef CONFIG_MIPS_CMP
- if (!result)
register_smp_ops(&vsmp_smp_ops);
-#else
- register_smp_ops(&vsmp_smp_ops);
-#endif
#endif
#ifdef CONFIG_MIPS_MT_SMTC
register_smp_ops(&msmtc_smp_ops);
static inline int mips_pcibios_iack(void)
{
int irq;
- u32 dummy;
/*
* Determine highest priority pending interrupt by performing
BONITO_PCIMAP_CFG = 0x20000;
/* Flush Bonito register block */
- dummy = BONITO_PCIMAP_CFG;
+ (void) BONITO_PCIMAP_CFG;
iob(); /* sync */
irq = __raw_readl((u32 *)_pcictrl_bonito_pcicfg);
static struct irq_chip msp_per_irq_controller = {
.name = "MSP_PER",
- .irq_enable = unmask_per_irq.
+ .irq_enable = unmask_per_irq,
.irq_disable = mask_per_irq,
.irq_ack = msp_per_irq_ack,
#ifdef CONFIG_SMP
0:
PTR_L t1, PBE_ADDRESS(t0) /* source */
PTR_L t2, PBE_ORIG_ADDRESS(t0) /* destination */
- PTR_ADDIU t3, t1, PAGE_SIZE
+ PTR_ADDU t3, t1, PAGE_SIZE
1:
REG_L t8, (t1)
REG_S t8, (t2)
*/
static int __init sgiseeq_devinit(void)
{
- unsigned int tmp;
+ unsigned int pbdma __maybe_unused;
int res, i;
eth0_pd.hpc = hpc3c0;
/* Second HPC is missing? */
if (ip22_is_fullhouse() ||
- get_dbe(tmp, (unsigned int *)&hpc3c1->pbdma[1]))
+ get_dbe(pbdma, (unsigned int *)&hpc3c1->pbdma[1]))
return 0;
sgimc->giopar |= SGIMC_GIOPAR_MASTEREXP1 | SGIMC_GIOPAR_EXP164 |
static unsigned long dosample(void)
{
u32 ct0, ct1;
- u8 msb, lsb;
+ u8 msb;
/* Start the counter. */
sgint->tcword = (SGINT_TCWORD_CNT2 | SGINT_TCWORD_CALL |
/* Latch and spin until top byte of counter2 is zero */
do {
writeb(SGINT_TCWORD_CNT2 | SGINT_TCWORD_CLAT, &sgint->tcword);
- lsb = readb(&sgint->tcnt2);
+ (void) readb(&sgint->tcnt2);
msb = readb(&sgint->tcnt2);
ct1 = read_c0_count();
} while (msb);
unsigned long xtalk_addr, size_t size)
{
nasid_t nasid = COMPACT_TO_NASID_NODEID(cnode);
- volatile hubreg_t junk;
unsigned i;
/* use small-window mapping if possible */
* after we write it.
*/
IIO_ITTE_PUT(nasid, i, HUB_PIO_MAP_TO_MEM, widget, xtalk_addr);
- junk = HUB_L(IIO_ITTE_GET(nasid, i));
+ (void) HUB_L(IIO_ITTE_GET(nasid, i));
return NODE_BWIN_BASE(nasid, widget) + (xtalk_addr % BWIN_SIZE);
}
static __init void set_ktext_source(nasid_t client_nasid, nasid_t server_nasid)
{
- cnodeid_t client_cnode;
kern_vars_t *kvp;
- client_cnode = NASID_TO_COMPACT_NODEID(client_nasid);
-
kvp = &hub_data(client_nasid)->kern_vars;
KERN_VARS_ADDR(client_nasid) = (unsigned long)kvp;
static __init unsigned long dosample(void)
{
u32 ct0, ct1;
- volatile u8 msb, lsb;
+ volatile u8 msb;
/* Start the counter. */
outb_p(0x34, 0x43);
/* Latch and spin until top byte of counter0 is zero */
do {
outb(0x00, 0x43);
- lsb = inb(0x40);
+ (void) inb(0x40);
msb = inb(0x40);
ct1 = read_c0_count();
} while (msb);
bool "AMD IOMMU support"
select SWIOTLB
select PCI_MSI
+ select PCI_IOV
depends on X86_64 && PCI && ACPI
---help---
With this option you can enable support for AMD IOMMU hardware in
#ifndef _ASM_X86_AMD_IOMMU_PROTO_H
#define _ASM_X86_AMD_IOMMU_PROTO_H
-struct amd_iommu;
+#include <asm/amd_iommu_types.h>
extern int amd_iommu_init_dma_ops(void);
extern int amd_iommu_init_passthrough(void);
+extern irqreturn_t amd_iommu_int_thread(int irq, void *data);
extern irqreturn_t amd_iommu_int_handler(int irq, void *data);
-extern void amd_iommu_flush_all_domains(void);
-extern void amd_iommu_flush_all_devices(void);
extern void amd_iommu_apply_erratum_63(u16 devid);
extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu);
extern int amd_iommu_init_devices(void);
(pdev->device == PCI_DEVICE_ID_RD890_IOMMU);
}
+static inline bool iommu_feature(struct amd_iommu *iommu, u64 f)
+{
+ if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))
+ return false;
+
+ return !!(iommu->features & f);
+}
+
#endif /* _ASM_X86_AMD_IOMMU_PROTO_H */
#define MMIO_CONTROL_OFFSET 0x0018
#define MMIO_EXCL_BASE_OFFSET 0x0020
#define MMIO_EXCL_LIMIT_OFFSET 0x0028
+#define MMIO_EXT_FEATURES 0x0030
#define MMIO_CMD_HEAD_OFFSET 0x2000
#define MMIO_CMD_TAIL_OFFSET 0x2008
#define MMIO_EVT_HEAD_OFFSET 0x2010
#define MMIO_EVT_TAIL_OFFSET 0x2018
#define MMIO_STATUS_OFFSET 0x2020
+
+/* Extended Feature Bits */
+#define FEATURE_PREFETCH (1ULL<<0)
+#define FEATURE_PPR (1ULL<<1)
+#define FEATURE_X2APIC (1ULL<<2)
+#define FEATURE_NX (1ULL<<3)
+#define FEATURE_GT (1ULL<<4)
+#define FEATURE_IA (1ULL<<6)
+#define FEATURE_GA (1ULL<<7)
+#define FEATURE_HE (1ULL<<8)
+#define FEATURE_PC (1ULL<<9)
+
/* MMIO status bits */
#define MMIO_STATUS_COM_WAIT_INT_MASK 0x04
/* command specific defines */
#define CMD_COMPL_WAIT 0x01
#define CMD_INV_DEV_ENTRY 0x02
-#define CMD_INV_IOMMU_PAGES 0x03
+#define CMD_INV_IOMMU_PAGES 0x03
+#define CMD_INV_IOTLB_PAGES 0x04
+#define CMD_INV_ALL 0x08
#define CMD_COMPL_WAIT_STORE_MASK 0x01
#define CMD_COMPL_WAIT_INT_MASK 0x02
#define IOMMU_PTE_IR (1ULL << 61)
#define IOMMU_PTE_IW (1ULL << 62)
+#define DTE_FLAG_IOTLB 0x01
+
#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
#define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK))
/* IOMMU capabilities */
#define IOMMU_CAP_IOTLB 24
#define IOMMU_CAP_NPCACHE 26
+#define IOMMU_CAP_EFR 27
#define MAX_DOMAIN_ID 65536
/* global flag if IOMMUs cache non-present entries */
extern bool amd_iommu_np_cache;
+/* Only true if all IOMMUs support device IOTLBs */
+extern bool amd_iommu_iotlb_sup;
/*
* Make iterating over all IOMMUs easier
/* flags read from acpi table */
u8 acpi_flags;
+ /* Extended features */
+ u64 features;
+
/*
* Capability pointer. There could be more than one IOMMU per PCI
* device function if there are more than one AMD IOMMU capability
/* if one, we need to send a completion wait command */
bool need_sync;
- /* becomes true if a command buffer reset is running */
- bool reset_in_progress;
-
/* default dma_ops domain for that IOMMU */
struct dma_ops_domain *default_dom;
*/
#include <linux/pci.h>
+#include <linux/pci-ats.h>
#include <linux/bitmap.h>
#include <linux/slab.h>
#include <linux/debugfs.h>
#include <linux/dma-mapping.h>
#include <linux/iommu-helper.h>
#include <linux/iommu.h>
+#include <linux/delay.h>
#include <asm/proto.h>
#include <asm/iommu.h>
#include <asm/gart.h>
#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
-#define EXIT_LOOP_COUNT 10000000
+#define LOOP_TIMEOUT 100000
static DEFINE_RWLOCK(amd_iommu_devtable_lock);
u32 data[4];
};
-static void reset_iommu_command_buffer(struct amd_iommu *iommu);
static void update_domain(struct protection_domain *domain);
/****************************************************************************
break;
case EVENT_TYPE_ILL_CMD:
printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
- iommu->reset_in_progress = true;
- reset_iommu_command_buffer(iommu);
dump_command(address);
break;
case EVENT_TYPE_CMD_HARD_ERR:
spin_unlock_irqrestore(&iommu->lock, flags);
}
-irqreturn_t amd_iommu_int_handler(int irq, void *data)
+irqreturn_t amd_iommu_int_thread(int irq, void *data)
{
struct amd_iommu *iommu;
return IRQ_HANDLED;
}
+irqreturn_t amd_iommu_int_handler(int irq, void *data)
+{
+ return IRQ_WAKE_THREAD;
+}
+
/****************************************************************************
*
* IOMMU command queuing functions
*
****************************************************************************/
-/*
- * Writes the command to the IOMMUs command buffer and informs the
- * hardware about the new command. Must be called with iommu->lock held.
- */
-static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
+static int wait_on_sem(volatile u64 *sem)
+{
+ int i = 0;
+
+ while (*sem == 0 && i < LOOP_TIMEOUT) {
+ udelay(1);
+ i += 1;
+ }
+
+ if (i == LOOP_TIMEOUT) {
+ pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static void copy_cmd_to_buffer(struct amd_iommu *iommu,
+ struct iommu_cmd *cmd,
+ u32 tail)
{
- u32 tail, head;
u8 *target;
- WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
- tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
target = iommu->cmd_buf + tail;
- memcpy_toio(target, cmd, sizeof(*cmd));
- tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
- head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
- if (tail == head)
- return -ENOMEM;
+ tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
+
+ /* Copy command to buffer */
+ memcpy(target, cmd, sizeof(*cmd));
+
+ /* Tell the IOMMU about it */
writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+}
- return 0;
+static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
+{
+ WARN_ON(address & 0x7ULL);
+
+ memset(cmd, 0, sizeof(*cmd));
+ cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
+ cmd->data[1] = upper_32_bits(__pa(address));
+ cmd->data[2] = 1;
+ CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
+}
+
+static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
+{
+ memset(cmd, 0, sizeof(*cmd));
+ cmd->data[0] = devid;
+ CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
+}
+
+static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
+ size_t size, u16 domid, int pde)
+{
+ u64 pages;
+ int s;
+
+ pages = iommu_num_pages(address, size, PAGE_SIZE);
+ s = 0;
+
+ if (pages > 1) {
+ /*
+ * If we have to flush more than one page, flush all
+ * TLB entries for this domain
+ */
+ address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
+ s = 1;
+ }
+
+ address &= PAGE_MASK;
+
+ memset(cmd, 0, sizeof(*cmd));
+ cmd->data[1] |= domid;
+ cmd->data[2] = lower_32_bits(address);
+ cmd->data[3] = upper_32_bits(address);
+ CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
+ if (s) /* size bit - we flush more than one 4kb page */
+ cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
+ if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
+ cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
+}
+
+static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
+ u64 address, size_t size)
+{
+ u64 pages;
+ int s;
+
+ pages = iommu_num_pages(address, size, PAGE_SIZE);
+ s = 0;
+
+ if (pages > 1) {
+ /*
+ * If we have to flush more than one page, flush all
+ * TLB entries for this domain
+ */
+ address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
+ s = 1;
+ }
+
+ address &= PAGE_MASK;
+
+ memset(cmd, 0, sizeof(*cmd));
+ cmd->data[0] = devid;
+ cmd->data[0] |= (qdep & 0xff) << 24;
+ cmd->data[1] = devid;
+ cmd->data[2] = lower_32_bits(address);
+ cmd->data[3] = upper_32_bits(address);
+ CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
+ if (s)
+ cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
+}
+
+static void build_inv_all(struct iommu_cmd *cmd)
+{
+ memset(cmd, 0, sizeof(*cmd));
+ CMD_SET_TYPE(cmd, CMD_INV_ALL);
}
/*
- * General queuing function for commands. Takes iommu->lock and calls
- * __iommu_queue_command().
+ * Writes the command to the IOMMUs command buffer and informs the
+ * hardware about the new command.
*/
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
{
+ u32 left, tail, head, next_tail;
unsigned long flags;
- int ret;
+ WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
+
+again:
spin_lock_irqsave(&iommu->lock, flags);
- ret = __iommu_queue_command(iommu, cmd);
- if (!ret)
- iommu->need_sync = true;
- spin_unlock_irqrestore(&iommu->lock, flags);
- return ret;
-}
+ head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
+ tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+ next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
+ left = (head - next_tail) % iommu->cmd_buf_size;
-/*
- * This function waits until an IOMMU has completed a completion
- * wait command
- */
-static void __iommu_wait_for_completion(struct amd_iommu *iommu)
-{
- int ready = 0;
- unsigned status = 0;
- unsigned long i = 0;
+ if (left <= 2) {
+ struct iommu_cmd sync_cmd;
+ volatile u64 sem = 0;
+ int ret;
- INC_STATS_COUNTER(compl_wait);
+ build_completion_wait(&sync_cmd, (u64)&sem);
+ copy_cmd_to_buffer(iommu, &sync_cmd, tail);
- while (!ready && (i < EXIT_LOOP_COUNT)) {
- ++i;
- /* wait for the bit to become one */
- status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
- ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
+ spin_unlock_irqrestore(&iommu->lock, flags);
+
+ if ((ret = wait_on_sem(&sem)) != 0)
+ return ret;
+
+ goto again;
}
- /* set bit back to zero */
- status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
- writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
+ copy_cmd_to_buffer(iommu, cmd, tail);
+
+ /* We need to sync now to make sure all commands are processed */
+ iommu->need_sync = true;
+
+ spin_unlock_irqrestore(&iommu->lock, flags);
- if (unlikely(i == EXIT_LOOP_COUNT))
- iommu->reset_in_progress = true;
+ return 0;
}
/*
* This function queues a completion wait command into the command
* buffer of an IOMMU
*/
-static int __iommu_completion_wait(struct amd_iommu *iommu)
+static int iommu_completion_wait(struct amd_iommu *iommu)
{
struct iommu_cmd cmd;
+ volatile u64 sem = 0;
+ int ret;
- memset(&cmd, 0, sizeof(cmd));
- cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
- CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
+ if (!iommu->need_sync)
+ return 0;
- return __iommu_queue_command(iommu, &cmd);
+ build_completion_wait(&cmd, (u64)&sem);
+
+ ret = iommu_queue_command(iommu, &cmd);
+ if (ret)
+ return ret;
+
+ return wait_on_sem(&sem);
}
-/*
- * This function is called whenever we need to ensure that the IOMMU has
- * completed execution of all commands we sent. It sends a
- * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
- * us about that by writing a value to a physical address we pass with
- * the command.
- */
-static int iommu_completion_wait(struct amd_iommu *iommu)
+static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
{
- int ret = 0;
- unsigned long flags;
+ struct iommu_cmd cmd;
- spin_lock_irqsave(&iommu->lock, flags);
+ build_inv_dte(&cmd, devid);
- if (!iommu->need_sync)
- goto out;
+ return iommu_queue_command(iommu, &cmd);
+}
- ret = __iommu_completion_wait(iommu);
+static void iommu_flush_dte_all(struct amd_iommu *iommu)
+{
+ u32 devid;
- iommu->need_sync = false;
+ for (devid = 0; devid <= 0xffff; ++devid)
+ iommu_flush_dte(iommu, devid);
- if (ret)
- goto out;
-
- __iommu_wait_for_completion(iommu);
+ iommu_completion_wait(iommu);
+}
-out:
- spin_unlock_irqrestore(&iommu->lock, flags);
+/*
+ * This function uses heavy locking and may disable irqs for some time. But
+ * this is no issue because it is only called during resume.
+ */
+static void iommu_flush_tlb_all(struct amd_iommu *iommu)
+{
+ u32 dom_id;
- if (iommu->reset_in_progress)
- reset_iommu_command_buffer(iommu);
+ for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
+ struct iommu_cmd cmd;
+ build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
+ dom_id, 1);
+ iommu_queue_command(iommu, &cmd);
+ }
- return 0;
+ iommu_completion_wait(iommu);
}
-static void iommu_flush_complete(struct protection_domain *domain)
+static void iommu_flush_all(struct amd_iommu *iommu)
{
- int i;
+ struct iommu_cmd cmd;
- for (i = 0; i < amd_iommus_present; ++i) {
- if (!domain->dev_iommu[i])
- continue;
+ build_inv_all(&cmd);
- /*
- * Devices of this domain are behind this IOMMU
- * We need to wait for completion of all commands.
- */
- iommu_completion_wait(amd_iommus[i]);
+ iommu_queue_command(iommu, &cmd);
+ iommu_completion_wait(iommu);
+}
+
+void iommu_flush_all_caches(struct amd_iommu *iommu)
+{
+ if (iommu_feature(iommu, FEATURE_IA)) {
+ iommu_flush_all(iommu);
+ } else {
+ iommu_flush_dte_all(iommu);
+ iommu_flush_tlb_all(iommu);
}
}
/*
- * Command send function for invalidating a device table entry
+ * Command send function for flushing on-device TLB
*/
-static int iommu_flush_device(struct device *dev)
+static int device_flush_iotlb(struct device *dev, u64 address, size_t size)
{
+ struct pci_dev *pdev = to_pci_dev(dev);
struct amd_iommu *iommu;
struct iommu_cmd cmd;
u16 devid;
+ int qdep;
+ qdep = pci_ats_queue_depth(pdev);
devid = get_device_id(dev);
iommu = amd_iommu_rlookup_table[devid];
- /* Build command */
- memset(&cmd, 0, sizeof(cmd));
- CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
- cmd.data[0] = devid;
+ build_inv_iotlb_pages(&cmd, devid, qdep, address, size);
return iommu_queue_command(iommu, &cmd);
}
-static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
- u16 domid, int pde, int s)
-{
- memset(cmd, 0, sizeof(*cmd));
- address &= PAGE_MASK;
- CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
- cmd->data[1] |= domid;
- cmd->data[2] = lower_32_bits(address);
- cmd->data[3] = upper_32_bits(address);
- if (s) /* size bit - we flush more than one 4kb page */
- cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
- if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
- cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
-}
-
/*
- * Generic command send function for invalidaing TLB entries
+ * Command send function for invalidating a device table entry
*/
-static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
- u64 address, u16 domid, int pde, int s)
+static int device_flush_dte(struct device *dev)
{
- struct iommu_cmd cmd;
+ struct amd_iommu *iommu;
+ struct pci_dev *pdev;
+ u16 devid;
int ret;
- __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
+ pdev = to_pci_dev(dev);
+ devid = get_device_id(dev);
+ iommu = amd_iommu_rlookup_table[devid];
- ret = iommu_queue_command(iommu, &cmd);
+ ret = iommu_flush_dte(iommu, devid);
+ if (ret)
+ return ret;
+
+ if (pci_ats_enabled(pdev))
+ ret = device_flush_iotlb(dev, 0, ~0UL);
return ret;
}
* It invalidates a single PTE if the range to flush is within a single
* page. Otherwise it flushes the whole TLB of the IOMMU.
*/
-static void __iommu_flush_pages(struct protection_domain *domain,
- u64 address, size_t size, int pde)
+static void __domain_flush_pages(struct protection_domain *domain,
+ u64 address, size_t size, int pde)
{
- int s = 0, i;
- unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE);
-
- address &= PAGE_MASK;
-
- if (pages > 1) {
- /*
- * If we have to flush more than one page, flush all
- * TLB entries for this domain
- */
- address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
- s = 1;
- }
+ struct iommu_dev_data *dev_data;
+ struct iommu_cmd cmd;
+ int ret = 0, i;
+ build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
for (i = 0; i < amd_iommus_present; ++i) {
if (!domain->dev_iommu[i])
* Devices of this domain are behind this IOMMU
* We need a TLB flush
*/
- iommu_queue_inv_iommu_pages(amd_iommus[i], address,
- domain->id, pde, s);
+ ret |= iommu_queue_command(amd_iommus[i], &cmd);
+ }
+
+ list_for_each_entry(dev_data, &domain->dev_list, list) {
+ struct pci_dev *pdev = to_pci_dev(dev_data->dev);
+
+ if (!pci_ats_enabled(pdev))
+ continue;
+
+ ret |= device_flush_iotlb(dev_data->dev, address, size);
}
- return;
+ WARN_ON(ret);
}
-static void iommu_flush_pages(struct protection_domain *domain,
- u64 address, size_t size)
+static void domain_flush_pages(struct protection_domain *domain,
+ u64 address, size_t size)
{
- __iommu_flush_pages(domain, address, size, 0);
+ __domain_flush_pages(domain, address, size, 0);
}
/* Flush the whole IO/TLB for a given protection domain */
-static void iommu_flush_tlb(struct protection_domain *domain)
+static void domain_flush_tlb(struct protection_domain *domain)
{
- __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
+ __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
}
/* Flush the whole IO/TLB for a given protection domain - including PDE */
-static void iommu_flush_tlb_pde(struct protection_domain *domain)
+static void domain_flush_tlb_pde(struct protection_domain *domain)
{
- __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
-}
-
-
-/*
- * This function flushes the DTEs for all devices in domain
- */
-static void iommu_flush_domain_devices(struct protection_domain *domain)
-{
- struct iommu_dev_data *dev_data;
- unsigned long flags;
-
- spin_lock_irqsave(&domain->lock, flags);
-
- list_for_each_entry(dev_data, &domain->dev_list, list)
- iommu_flush_device(dev_data->dev);
-
- spin_unlock_irqrestore(&domain->lock, flags);
+ __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
}
-static void iommu_flush_all_domain_devices(void)
+static void domain_flush_complete(struct protection_domain *domain)
{
- struct protection_domain *domain;
- unsigned long flags;
+ int i;
- spin_lock_irqsave(&amd_iommu_pd_lock, flags);
+ for (i = 0; i < amd_iommus_present; ++i) {
+ if (!domain->dev_iommu[i])
+ continue;
- list_for_each_entry(domain, &amd_iommu_pd_list, list) {
- iommu_flush_domain_devices(domain);
- iommu_flush_complete(domain);
+ /*
+ * Devices of this domain are behind this IOMMU
+ * We need to wait for completion of all commands.
+ */
+ iommu_completion_wait(amd_iommus[i]);
}
-
- spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
}
-void amd_iommu_flush_all_devices(void)
-{
- iommu_flush_all_domain_devices();
-}
/*
- * This function uses heavy locking and may disable irqs for some time. But
- * this is no issue because it is only called during resume.
+ * This function flushes the DTEs for all devices in domain
*/
-void amd_iommu_flush_all_domains(void)
+static void domain_flush_devices(struct protection_domain *domain)
{
- struct protection_domain *domain;
+ struct iommu_dev_data *dev_data;
unsigned long flags;
- spin_lock_irqsave(&amd_iommu_pd_lock, flags);
-
- list_for_each_entry(domain, &amd_iommu_pd_list, list) {
- spin_lock(&domain->lock);
- iommu_flush_tlb_pde(domain);
- iommu_flush_complete(domain);
- spin_unlock(&domain->lock);
- }
-
- spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
-}
-
-static void reset_iommu_command_buffer(struct amd_iommu *iommu)
-{
- pr_err("AMD-Vi: Resetting IOMMU command buffer\n");
-
- if (iommu->reset_in_progress)
- panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n");
+ spin_lock_irqsave(&domain->lock, flags);
- amd_iommu_reset_cmd_buffer(iommu);
- amd_iommu_flush_all_devices();
- amd_iommu_flush_all_domains();
+ list_for_each_entry(dev_data, &domain->dev_list, list)
+ device_flush_dte(dev_data->dev);
- iommu->reset_in_progress = false;
+ spin_unlock_irqrestore(&domain->lock, flags);
}
/****************************************************************************
return domain->flags & PD_DMA_OPS_MASK;
}
-static void set_dte_entry(u16 devid, struct protection_domain *domain)
+static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
{
u64 pte_root = virt_to_phys(domain->pt_root);
+ u32 flags = 0;
pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
<< DEV_ENTRY_MODE_SHIFT;
pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
- amd_iommu_dev_table[devid].data[2] = domain->id;
- amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
- amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
+ if (ats)
+ flags |= DTE_FLAG_IOTLB;
+
+ amd_iommu_dev_table[devid].data[3] |= flags;
+ amd_iommu_dev_table[devid].data[2] = domain->id;
+ amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
+ amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
}
static void clear_dte_entry(u16 devid)
{
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu;
+ struct pci_dev *pdev;
+ bool ats = false;
u16 devid;
devid = get_device_id(dev);
iommu = amd_iommu_rlookup_table[devid];
dev_data = get_dev_data(dev);
+ pdev = to_pci_dev(dev);
+
+ if (amd_iommu_iotlb_sup)
+ ats = pci_ats_enabled(pdev);
/* Update data structures */
dev_data->domain = domain;
list_add(&dev_data->list, &domain->dev_list);
- set_dte_entry(devid, domain);
+ set_dte_entry(devid, domain, ats);
/* Do reference counting */
domain->dev_iommu[iommu->index] += 1;
domain->dev_cnt += 1;
/* Flush the DTE entry */
- iommu_flush_device(dev);
+ device_flush_dte(dev);
}
static void do_detach(struct device *dev)
{
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu;
+ struct pci_dev *pdev;
u16 devid;
devid = get_device_id(dev);
iommu = amd_iommu_rlookup_table[devid];
dev_data = get_dev_data(dev);
+ pdev = to_pci_dev(dev);
/* decrease reference counters */
dev_data->domain->dev_iommu[iommu->index] -= 1;
clear_dte_entry(devid);
/* Flush the DTE entry */
- iommu_flush_device(dev);
+ device_flush_dte(dev);
}
/*
static int attach_device(struct device *dev,
struct protection_domain *domain)
{
+ struct pci_dev *pdev = to_pci_dev(dev);
unsigned long flags;
int ret;
+ if (amd_iommu_iotlb_sup)
+ pci_enable_ats(pdev, PAGE_SHIFT);
+
write_lock_irqsave(&amd_iommu_devtable_lock, flags);
ret = __attach_device(dev, domain);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
* left the caches in the IOMMU dirty. So we have to flush
* here to evict all dirty stuff.
*/
- iommu_flush_tlb_pde(domain);
+ domain_flush_tlb_pde(domain);
return ret;
}
*/
static void detach_device(struct device *dev)
{
+ struct pci_dev *pdev = to_pci_dev(dev);
unsigned long flags;
/* lock device table */
write_lock_irqsave(&amd_iommu_devtable_lock, flags);
__detach_device(dev);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+
+ if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev))
+ pci_disable_ats(pdev);
}
/*
goto out;
}
- iommu_flush_device(dev);
+ device_flush_dte(dev);
iommu_completion_wait(iommu);
out:
struct iommu_dev_data *dev_data;
list_for_each_entry(dev_data, &domain->dev_list, list) {
+ struct pci_dev *pdev = to_pci_dev(dev_data->dev);
u16 devid = get_device_id(dev_data->dev);
- set_dte_entry(devid, domain);
+ set_dte_entry(devid, domain, pci_ats_enabled(pdev));
}
}
return;
update_device_table(domain);
- iommu_flush_domain_devices(domain);
- iommu_flush_tlb_pde(domain);
+
+ domain_flush_devices(domain);
+ domain_flush_tlb_pde(domain);
domain->updated = false;
}
ADD_STATS_COUNTER(alloced_io_mem, size);
if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
- iommu_flush_tlb(&dma_dom->domain);
+ domain_flush_tlb(&dma_dom->domain);
dma_dom->need_flush = false;
} else if (unlikely(amd_iommu_np_cache))
- iommu_flush_pages(&dma_dom->domain, address, size);
+ domain_flush_pages(&dma_dom->domain, address, size);
out:
return address;
dma_ops_free_addresses(dma_dom, dma_addr, pages);
if (amd_iommu_unmap_flush || dma_dom->need_flush) {
- iommu_flush_pages(&dma_dom->domain, flush_addr, size);
+ domain_flush_pages(&dma_dom->domain, flush_addr, size);
dma_dom->need_flush = false;
}
}
if (addr == DMA_ERROR_CODE)
goto out;
- iommu_flush_complete(domain);
+ domain_flush_complete(domain);
out:
spin_unlock_irqrestore(&domain->lock, flags);
__unmap_single(domain->priv, dma_addr, size, dir);
- iommu_flush_complete(domain);
+ domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
}
goto unmap;
}
- iommu_flush_complete(domain);
+ domain_flush_complete(domain);
out:
spin_unlock_irqrestore(&domain->lock, flags);
s->dma_address = s->dma_length = 0;
}
- iommu_flush_complete(domain);
+ domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
}
goto out_free;
}
- iommu_flush_complete(domain);
+ domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
- iommu_flush_complete(domain);
+ domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
if (!iommu)
return;
- iommu_flush_device(dev);
+ device_flush_dte(dev);
iommu_completion_wait(iommu);
}
unmap_size = iommu_unmap_page(domain, iova, page_size);
mutex_unlock(&domain->api_lock);
- iommu_flush_tlb_pde(domain);
+ domain_flush_tlb_pde(domain);
return get_order(unmap_size);
}
/* IOMMUs have a non-present cache? */
bool amd_iommu_np_cache __read_mostly;
+bool amd_iommu_iotlb_sup __read_mostly = true;
/*
* The ACPI table parsing functions set this variable on an error
static u32 alias_table_size; /* size of the alias table */
static u32 rlookup_table_size; /* size if the rlookup table */
+/*
+ * This function flushes all internal caches of
+ * the IOMMU used by this driver.
+ */
+extern void iommu_flush_all_caches(struct amd_iommu *iommu);
+
static inline void update_last_devid(u16 devid)
{
if (devid > amd_iommu_last_bdf)
/* Function to enable the hardware */
static void iommu_enable(struct amd_iommu *iommu)
{
- printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx\n",
+ static const char * const feat_str[] = {
+ "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
+ "IA", "GA", "HE", "PC", NULL
+ };
+ int i;
+
+ printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx",
dev_name(&iommu->dev->dev), iommu->cap_ptr);
+ if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
+ printk(KERN_CONT " extended features: ");
+ for (i = 0; feat_str[i]; ++i)
+ if (iommu_feature(iommu, (1ULL << i)))
+ printk(KERN_CONT " %s", feat_str[i]);
+ }
+ printk(KERN_CONT "\n");
+
iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
}
static void __init init_iommu_from_pci(struct amd_iommu *iommu)
{
int cap_ptr = iommu->cap_ptr;
- u32 range, misc;
+ u32 range, misc, low, high;
int i, j;
pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
MMIO_GET_LD(range));
iommu->evt_msi_num = MMIO_MSI_NUM(misc);
+ if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
+ amd_iommu_iotlb_sup = false;
+
+ /* read extended feature bits */
+ low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
+ high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
+
+ iommu->features = ((u64)high << 32) | low;
+
if (!is_rd890_iommu(iommu->dev))
return;
if (pci_enable_msi(iommu->dev))
return 1;
- r = request_irq(iommu->dev->irq, amd_iommu_int_handler,
- IRQF_SAMPLE_RANDOM,
- "AMD-Vi",
- NULL);
+ r = request_threaded_irq(iommu->dev->irq,
+ amd_iommu_int_handler,
+ amd_iommu_int_thread,
+ 0, "AMD-Vi",
+ iommu->dev);
if (r) {
pci_disable_msi(iommu->dev);
iommu_set_exclusion_range(iommu);
iommu_init_msi(iommu);
iommu_enable(iommu);
+ iommu_flush_all_caches(iommu);
}
}
* we have to flush after the IOMMUs are enabled because a
* disabled IOMMU will never execute the commands we send
*/
- amd_iommu_flush_all_devices();
- amd_iommu_flush_all_domains();
+ for_each_iommu(iommu)
+ iommu_flush_all_caches(iommu);
}
static int amd_iommu_suspend(void)
struct iommu_table_entry *finish)
{
struct iommu_table_entry *p, *q, *x;
- char sym_p[KSYM_SYMBOL_LEN];
- char sym_q[KSYM_SYMBOL_LEN];
/* Simple cyclic dependency checker. */
for (p = start; p < finish; p++) {
q = find_dependents_of(start, finish, p);
x = find_dependents_of(start, finish, q);
if (p == x) {
- sprint_symbol(sym_p, (unsigned long)p->detect);
- sprint_symbol(sym_q, (unsigned long)q->detect);
-
- printk(KERN_ERR "CYCLIC DEPENDENCY FOUND! %s depends" \
- " on %s and vice-versa. BREAKING IT.\n",
- sym_p, sym_q);
+ printk(KERN_ERR "CYCLIC DEPENDENCY FOUND! %pS depends on %pS and vice-versa. BREAKING IT.\n",
+ p->detect, q->detect);
/* Heavy handed way..*/
x->depend = 0;
}
for (p = start; p < finish; p++) {
q = find_dependents_of(p, finish, p);
if (q && q > p) {
- sprint_symbol(sym_p, (unsigned long)p->detect);
- sprint_symbol(sym_q, (unsigned long)q->detect);
-
- printk(KERN_ERR "EXECUTION ORDER INVALID! %s "\
- "should be called before %s!\n",
- sym_p, sym_q);
+ printk(KERN_ERR "EXECUTION ORDER INVALID! %pS should be called before %pS!\n",
+ p->detect, q->detect);
}
}
}
#include <linux/syscore_ops.h>
#include <linux/tboot.h>
#include <linux/dmi.h>
+#include <linux/pci-ats.h>
#include <asm/cacheflush.h>
#include <asm/iommu.h>
#include "pci.h"
#include <linux/mutex.h>
#include <linux/string.h>
#include <linux/delay.h>
+#include <linux/pci-ats.h>
#include "pci.h"
#define VIRTFN_ID_LEN 16
u8 __iomem *mstate; /* VF Migration State Array */
};
-/* Address Translation Service */
-struct pci_ats {
- int pos; /* capability position */
- int stu; /* Smallest Translation Unit */
- int qdep; /* Invalidate Queue Depth */
- int ref_cnt; /* Physical Function reference count */
- unsigned int is_enabled:1; /* Enable bit is set */
-};
-
#ifdef CONFIG_PCI_IOV
extern int pci_iov_init(struct pci_dev *dev);
extern void pci_iov_release(struct pci_dev *dev);
extern void pci_restore_iov_state(struct pci_dev *dev);
extern int pci_iov_bus_range(struct pci_bus *bus);
-extern int pci_enable_ats(struct pci_dev *dev, int ps);
-extern void pci_disable_ats(struct pci_dev *dev);
-extern int pci_ats_queue_depth(struct pci_dev *dev);
-/**
- * pci_ats_enabled - query the ATS status
- * @dev: the PCI device
- *
- * Returns 1 if ATS capability is enabled, or 0 if not.
- */
-static inline int pci_ats_enabled(struct pci_dev *dev)
-{
- return dev->ats && dev->ats->is_enabled;
-}
#else
static inline int pci_iov_init(struct pci_dev *dev)
{
return 0;
}
-static inline int pci_enable_ats(struct pci_dev *dev, int ps)
-{
- return -ENODEV;
-}
-static inline void pci_disable_ats(struct pci_dev *dev)
-{
-}
-static inline int pci_ats_queue_depth(struct pci_dev *dev)
-{
- return -ENODEV;
-}
-static inline int pci_ats_enabled(struct pci_dev *dev)
-{
- return 0;
-}
#endif /* CONFIG_PCI_IOV */
static inline resource_size_t pci_resource_alignment(struct pci_dev *dev,
unsigned long group, group_offset;
int i, j, n, ret;
- for (i = 0; i < nitems; i += n) {
+ for (i = 0; i < nitems; i = j) {
group = nilfs_palloc_group(inode, entry_nrs[i], &group_offset);
ret = nilfs_palloc_get_desc_block(inode, group, 0, &desc_bh);
if (ret < 0)
XFS_LOOKUP_BATCH,
XFS_ICI_RECLAIM_TAG);
if (!nr_found) {
+ done = 1;
rcu_read_unlock();
break;
}
*/
STATIC void
xfs_ail_worker(
- struct work_struct *work)
+ struct work_struct *work)
{
- struct xfs_ail *ailp = container_of(to_delayed_work(work),
+ struct xfs_ail *ailp = container_of(to_delayed_work(work),
struct xfs_ail, xa_work);
- long tout;
- xfs_lsn_t target = ailp->xa_target;
- xfs_lsn_t lsn;
- xfs_log_item_t *lip;
- int flush_log, count, stuck;
- xfs_mount_t *mp = ailp->xa_mount;
+ xfs_mount_t *mp = ailp->xa_mount;
struct xfs_ail_cursor *cur = &ailp->xa_cursors;
- int push_xfsbufd = 0;
+ xfs_log_item_t *lip;
+ xfs_lsn_t lsn;
+ xfs_lsn_t target;
+ long tout = 10;
+ int flush_log = 0;
+ int stuck = 0;
+ int count = 0;
+ int push_xfsbufd = 0;
spin_lock(&ailp->xa_lock);
+ target = ailp->xa_target;
xfs_trans_ail_cursor_init(ailp, cur);
lip = xfs_trans_ail_cursor_first(ailp, cur, ailp->xa_last_pushed_lsn);
if (!lip || XFS_FORCED_SHUTDOWN(mp)) {
*/
xfs_trans_ail_cursor_done(ailp, cur);
spin_unlock(&ailp->xa_lock);
- ailp->xa_last_pushed_lsn = 0;
- return;
+ goto out_done;
}
XFS_STATS_INC(xs_push_ail);
* lots of contention on the AIL lists.
*/
lsn = lip->li_lsn;
- flush_log = stuck = count = 0;
- while ((XFS_LSN_CMP(lip->li_lsn, target) < 0)) {
+ while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
int lock_result;
/*
* If we can lock the item without sleeping, unlock the AIL
}
/* assume we have more work to do in a short while */
- tout = 10;
+out_done:
if (!count) {
/* We're past our target or empty, so idle */
ailp->xa_last_pushed_lsn = 0;
/*
- * Check for an updated push target before clearing the
- * XFS_AIL_PUSHING_BIT. If the target changed, we've got more
- * work to do. Wait a bit longer before starting that work.
+ * We clear the XFS_AIL_PUSHING_BIT first before checking
+ * whether the target has changed. If the target has changed,
+ * this pushes the requeue race directly onto the result of the
+ * atomic test/set bit, so we are guaranteed that either the
+ * the pusher that changed the target or ourselves will requeue
+ * the work (but not both).
*/
+ clear_bit(XFS_AIL_PUSHING_BIT, &ailp->xa_flags);
smp_rmb();
- if (ailp->xa_target == target) {
- clear_bit(XFS_AIL_PUSHING_BIT, &ailp->xa_flags);
+ if (XFS_LSN_CMP(ailp->xa_target, target) == 0 ||
+ test_and_set_bit(XFS_AIL_PUSHING_BIT, &ailp->xa_flags))
return;
- }
+
tout = 50;
} else if (XFS_LSN_CMP(lsn, target) >= 0) {
/*
* the XFS_AIL_PUSHING_BIT.
*/
smp_wmb();
- ailp->xa_target = threshold_lsn;
+ xfs_trans_ail_copy_lsn(ailp, &ailp->xa_target, &threshold_lsn);
if (!test_and_set_bit(XFS_AIL_PUSHING_BIT, &ailp->xa_flags))
queue_delayed_work(xfs_syncd_wq, &ailp->xa_work, 0);
}
--- /dev/null
+#ifndef LINUX_PCI_ATS_H
+#define LINUX_PCI_ATS_H
+
+/* Address Translation Service */
+struct pci_ats {
+ int pos; /* capability position */
+ int stu; /* Smallest Translation Unit */
+ int qdep; /* Invalidate Queue Depth */
+ int ref_cnt; /* Physical Function reference count */
+ unsigned int is_enabled:1; /* Enable bit is set */
+};
+
+#ifdef CONFIG_PCI_IOV
+
+extern int pci_enable_ats(struct pci_dev *dev, int ps);
+extern void pci_disable_ats(struct pci_dev *dev);
+extern int pci_ats_queue_depth(struct pci_dev *dev);
+/**
+ * pci_ats_enabled - query the ATS status
+ * @dev: the PCI device
+ *
+ * Returns 1 if ATS capability is enabled, or 0 if not.
+ */
+static inline int pci_ats_enabled(struct pci_dev *dev)
+{
+ return dev->ats && dev->ats->is_enabled;
+}
+
+#else /* CONFIG_PCI_IOV */
+
+static inline int pci_enable_ats(struct pci_dev *dev, int ps)
+{
+ return -ENODEV;
+}
+
+static inline void pci_disable_ats(struct pci_dev *dev)
+{
+}
+
+static inline int pci_ats_queue_depth(struct pci_dev *dev)
+{
+ return -ENODEV;
+}
+
+static inline int pci_ats_enabled(struct pci_dev *dev)
+{
+ return 0;
+}
+
+#endif /* CONFIG_PCI_IOV */
+
+#endif /* LINUX_PCI_ATS_H*/
return -ENOMEM;
}
-static int device_dma_allocations(struct device *dev)
+static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
{
struct dma_debug_entry *entry;
unsigned long flags;
for (i = 0; i < HASH_SIZE; ++i) {
spin_lock(&dma_entry_hash[i].lock);
list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
- if (entry->dev == dev)
+ if (entry->dev == dev) {
count += 1;
+ *out_entry = entry;
+ }
}
spin_unlock(&dma_entry_hash[i].lock);
}
static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
{
struct device *dev = data;
+ struct dma_debug_entry *uninitialized_var(entry);
int count;
if (global_disable)
switch (action) {
case BUS_NOTIFY_UNBOUND_DRIVER:
- count = device_dma_allocations(dev);
+ count = device_dma_allocations(dev, &entry);
if (count == 0)
break;
- err_printk(dev, NULL, "DMA-API: device driver has pending "
+ err_printk(dev, entry, "DMA-API: device driver has pending "
"DMA allocations while released from device "
- "[count=%d]\n", count);
+ "[count=%d]\n"
+ "One of leaked entries details: "
+ "[device address=0x%016llx] [size=%llu bytes] "
+ "[mapped with %s] [mapped as %s]\n",
+ count, entry->dev_addr, entry->size,
+ dir2name[entry->direction], type2name[entry->type]);
break;
default:
break;
err = c->trans_mod->request(c, req);
if (err < 0) {
- if (err != -ERESTARTSYS)
+ if (err != -ERESTARTSYS && err != -EFAULT)
c->status = Disconnected;
goto reterr;
}
int nr_pages, u8 rw)
{
uint32_t first_page_bytes = 0;
- uint32_t pdata_mapped_pages;
+ int32_t pdata_mapped_pages;
struct trans_rpage_info *rpinfo;
*pdata_off = (__force size_t)req->tc->pubuf & (PAGE_SIZE-1);
rpinfo = req->tc->private;
pdata_mapped_pages = get_user_pages_fast((unsigned long)req->tc->pubuf,
nr_pages, rw, &rpinfo->rp_data[0]);
+ if (pdata_mapped_pages <= 0)
+ return pdata_mapped_pages;
- if (pdata_mapped_pages < 0) {
- printk(KERN_ERR "get_user_pages_fast failed:%d udata:%p"
- "nr_pages:%d\n", pdata_mapped_pages,
- req->tc->pubuf, nr_pages);
- pdata_mapped_pages = 0;
- return -EIO;
- }
rpinfo->rp_nr_pages = pdata_mapped_pages;
if (*pdata_off) {
*pdata_len = first_page_bytes;
projects / platform / upstream / kernel-adaptation-pc.git / commitdiff