1 // SPDX-License-Identifier: GPL-2.0
3 * IOMMU API for ARM architected SMMUv3 implementations.
5 * Copyright (C) 2015 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver is powered by bad coffee and bombay mix.
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitops.h>
15 #include <linux/crash_dump.h>
16 #include <linux/delay.h>
17 #include <linux/err.h>
18 #include <linux/interrupt.h>
19 #include <linux/io-pgtable.h>
20 #include <linux/iopoll.h>
21 #include <linux/module.h>
22 #include <linux/msi.h>
24 #include <linux/of_address.h>
25 #include <linux/of_platform.h>
26 #include <linux/pci.h>
27 #include <linux/pci-ats.h>
28 #include <linux/platform_device.h>
30 #include "arm-smmu-v3.h"
31 #include "../../dma-iommu.h"
32 #include "../../iommu-sva.h"
34 static bool disable_bypass = true;
35 module_param(disable_bypass, bool, 0444);
36 MODULE_PARM_DESC(disable_bypass,
37 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
39 static bool disable_msipolling;
40 module_param(disable_msipolling, bool, 0444);
41 MODULE_PARM_DESC(disable_msipolling,
42 "Disable MSI-based polling for CMD_SYNC completion.");
44 enum arm_smmu_msi_index {
51 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
53 ARM_SMMU_EVTQ_IRQ_CFG0,
54 ARM_SMMU_EVTQ_IRQ_CFG1,
55 ARM_SMMU_EVTQ_IRQ_CFG2,
57 [GERROR_MSI_INDEX] = {
58 ARM_SMMU_GERROR_IRQ_CFG0,
59 ARM_SMMU_GERROR_IRQ_CFG1,
60 ARM_SMMU_GERROR_IRQ_CFG2,
63 ARM_SMMU_PRIQ_IRQ_CFG0,
64 ARM_SMMU_PRIQ_IRQ_CFG1,
65 ARM_SMMU_PRIQ_IRQ_CFG2,
69 struct arm_smmu_option_prop {
74 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
75 DEFINE_MUTEX(arm_smmu_asid_lock);
78 * Special value used by SVA when a process dies, to quiesce a CD without
81 struct arm_smmu_ctx_desc quiet_cd = { 0 };
83 static struct arm_smmu_option_prop arm_smmu_options[] = {
84 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
85 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
89 static void parse_driver_options(struct arm_smmu_device *smmu)
94 if (of_property_read_bool(smmu->dev->of_node,
95 arm_smmu_options[i].prop)) {
96 smmu->options |= arm_smmu_options[i].opt;
97 dev_notice(smmu->dev, "option %s\n",
98 arm_smmu_options[i].prop);
100 } while (arm_smmu_options[++i].opt);
103 /* Low-level queue manipulation functions */
104 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
106 u32 space, prod, cons;
108 prod = Q_IDX(q, q->prod);
109 cons = Q_IDX(q, q->cons);
111 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
112 space = (1 << q->max_n_shift) - (prod - cons);
119 static bool queue_full(struct arm_smmu_ll_queue *q)
121 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
122 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
125 static bool queue_empty(struct arm_smmu_ll_queue *q)
127 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
128 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
131 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
133 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
134 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
135 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
136 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
139 static void queue_sync_cons_out(struct arm_smmu_queue *q)
142 * Ensure that all CPU accesses (reads and writes) to the queue
143 * are complete before we update the cons pointer.
146 writel_relaxed(q->llq.cons, q->cons_reg);
149 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
151 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
152 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
155 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
157 struct arm_smmu_ll_queue *llq = &q->llq;
159 if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
162 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
163 Q_IDX(llq, llq->cons);
164 queue_sync_cons_out(q);
167 static int queue_sync_prod_in(struct arm_smmu_queue *q)
173 * We can't use the _relaxed() variant here, as we must prevent
174 * speculative reads of the queue before we have determined that
175 * prod has indeed moved.
177 prod = readl(q->prod_reg);
179 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
186 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
188 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
189 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
192 static void queue_poll_init(struct arm_smmu_device *smmu,
193 struct arm_smmu_queue_poll *qp)
197 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
198 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
201 static int queue_poll(struct arm_smmu_queue_poll *qp)
203 if (ktime_compare(ktime_get(), qp->timeout) > 0)
208 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
219 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
223 for (i = 0; i < n_dwords; ++i)
224 *dst++ = cpu_to_le64(*src++);
227 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
231 for (i = 0; i < n_dwords; ++i)
232 *dst++ = le64_to_cpu(*src++);
235 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
237 if (queue_empty(&q->llq))
240 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
241 queue_inc_cons(&q->llq);
242 queue_sync_cons_out(q);
246 /* High-level queue accessors */
247 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
249 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
250 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
252 switch (ent->opcode) {
253 case CMDQ_OP_TLBI_EL2_ALL:
254 case CMDQ_OP_TLBI_NSNH_ALL:
256 case CMDQ_OP_PREFETCH_CFG:
257 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
259 case CMDQ_OP_CFGI_CD:
260 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
262 case CMDQ_OP_CFGI_STE:
263 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
264 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
266 case CMDQ_OP_CFGI_CD_ALL:
267 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
269 case CMDQ_OP_CFGI_ALL:
270 /* Cover the entire SID range */
271 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
273 case CMDQ_OP_TLBI_NH_VA:
274 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
276 case CMDQ_OP_TLBI_EL2_VA:
277 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
278 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
279 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
280 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
281 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
282 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
283 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
285 case CMDQ_OP_TLBI_S2_IPA:
286 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
287 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
288 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
289 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
290 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
291 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
292 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
294 case CMDQ_OP_TLBI_NH_ASID:
295 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
297 case CMDQ_OP_TLBI_S12_VMALL:
298 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
300 case CMDQ_OP_TLBI_EL2_ASID:
301 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
303 case CMDQ_OP_ATC_INV:
304 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
305 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
306 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
307 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
308 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
309 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
311 case CMDQ_OP_PRI_RESP:
312 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
313 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
314 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
315 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
316 switch (ent->pri.resp) {
324 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
327 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
328 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
329 cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
331 case CMDQ_OP_CMD_SYNC:
332 if (ent->sync.msiaddr) {
333 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
334 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
336 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
338 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
339 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
348 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)
353 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
354 struct arm_smmu_queue *q, u32 prod)
356 struct arm_smmu_cmdq_ent ent = {
357 .opcode = CMDQ_OP_CMD_SYNC,
361 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
362 * payload, so the write will zero the entire command on that platform.
364 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
365 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
369 arm_smmu_cmdq_build_cmd(cmd, &ent);
372 static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
373 struct arm_smmu_queue *q)
375 static const char * const cerror_str[] = {
376 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
377 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
378 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
379 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
383 u64 cmd[CMDQ_ENT_DWORDS];
384 u32 cons = readl_relaxed(q->cons_reg);
385 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
386 struct arm_smmu_cmdq_ent cmd_sync = {
387 .opcode = CMDQ_OP_CMD_SYNC,
390 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
391 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
394 case CMDQ_ERR_CERROR_ABT_IDX:
395 dev_err(smmu->dev, "retrying command fetch\n");
397 case CMDQ_ERR_CERROR_NONE_IDX:
399 case CMDQ_ERR_CERROR_ATC_INV_IDX:
401 * ATC Invalidation Completion timeout. CONS is still pointing
402 * at the CMD_SYNC. Attempt to complete other pending commands
403 * by repeating the CMD_SYNC, though we might well end up back
404 * here since the ATC invalidation may still be pending.
407 case CMDQ_ERR_CERROR_ILL_IDX:
413 * We may have concurrent producers, so we need to be careful
414 * not to touch any of the shadow cmdq state.
416 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
417 dev_err(smmu->dev, "skipping command in error state:\n");
418 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
419 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
421 /* Convert the erroneous command into a CMD_SYNC */
422 arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
424 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
427 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
429 __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q);
433 * Command queue locking.
434 * This is a form of bastardised rwlock with the following major changes:
436 * - The only LOCK routines are exclusive_trylock() and shared_lock().
437 * Neither have barrier semantics, and instead provide only a control
440 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
441 * fails if the caller appears to be the last lock holder (yes, this is
442 * racy). All successful UNLOCK routines have RELEASE semantics.
444 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
449 * We can try to avoid the cmpxchg() loop by simply incrementing the
450 * lock counter. When held in exclusive state, the lock counter is set
451 * to INT_MIN so these increments won't hurt as the value will remain
454 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
458 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
459 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
462 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
464 (void)atomic_dec_return_release(&cmdq->lock);
467 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
469 if (atomic_read(&cmdq->lock) == 1)
472 arm_smmu_cmdq_shared_unlock(cmdq);
476 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
479 local_irq_save(flags); \
480 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
482 local_irq_restore(flags); \
486 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
488 atomic_set_release(&cmdq->lock, 0); \
489 local_irq_restore(flags); \
494 * Command queue insertion.
495 * This is made fiddly by our attempts to achieve some sort of scalability
496 * since there is one queue shared amongst all of the CPUs in the system. If
497 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
498 * then you'll *love* this monstrosity.
500 * The basic idea is to split the queue up into ranges of commands that are
501 * owned by a given CPU; the owner may not have written all of the commands
502 * itself, but is responsible for advancing the hardware prod pointer when
503 * the time comes. The algorithm is roughly:
505 * 1. Allocate some space in the queue. At this point we also discover
506 * whether the head of the queue is currently owned by another CPU,
507 * or whether we are the owner.
509 * 2. Write our commands into our allocated slots in the queue.
511 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
513 * 4. If we are an owner:
514 * a. Wait for the previous owner to finish.
515 * b. Mark the queue head as unowned, which tells us the range
516 * that we are responsible for publishing.
517 * c. Wait for all commands in our owned range to become valid.
518 * d. Advance the hardware prod pointer.
519 * e. Tell the next owner we've finished.
521 * 5. If we are inserting a CMD_SYNC (we may or may not have been an
522 * owner), then we need to stick around until it has completed:
523 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
524 * to clear the first 4 bytes.
525 * b. Otherwise, we spin waiting for the hardware cons pointer to
526 * advance past our command.
528 * The devil is in the details, particularly the use of locking for handling
529 * SYNC completion and freeing up space in the queue before we think that it is
532 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
533 u32 sprod, u32 eprod, bool set)
535 u32 swidx, sbidx, ewidx, ebidx;
536 struct arm_smmu_ll_queue llq = {
537 .max_n_shift = cmdq->q.llq.max_n_shift,
541 ewidx = BIT_WORD(Q_IDX(&llq, eprod));
542 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
544 while (llq.prod != eprod) {
547 u32 limit = BITS_PER_LONG;
549 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
550 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
552 ptr = &cmdq->valid_map[swidx];
554 if ((swidx == ewidx) && (sbidx < ebidx))
557 mask = GENMASK(limit - 1, sbidx);
560 * The valid bit is the inverse of the wrap bit. This means
561 * that a zero-initialised queue is invalid and, after marking
562 * all entries as valid, they become invalid again when we
566 atomic_long_xor(mask, ptr);
570 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
571 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
574 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
578 /* Mark all entries in the range [sprod, eprod) as valid */
579 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
580 u32 sprod, u32 eprod)
582 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
585 /* Wait for all entries in the range [sprod, eprod) to become valid */
586 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
587 u32 sprod, u32 eprod)
589 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
592 /* Wait for the command queue to become non-full */
593 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
594 struct arm_smmu_ll_queue *llq)
597 struct arm_smmu_queue_poll qp;
598 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
602 * Try to update our copy of cons by grabbing exclusive cmdq access. If
603 * that fails, spin until somebody else updates it for us.
605 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
606 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
607 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
608 llq->val = READ_ONCE(cmdq->q.llq.val);
612 queue_poll_init(smmu, &qp);
614 llq->val = READ_ONCE(cmdq->q.llq.val);
615 if (!queue_full(llq))
618 ret = queue_poll(&qp);
625 * Wait until the SMMU signals a CMD_SYNC completion MSI.
626 * Must be called with the cmdq lock held in some capacity.
628 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
629 struct arm_smmu_ll_queue *llq)
632 struct arm_smmu_queue_poll qp;
633 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
634 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
636 queue_poll_init(smmu, &qp);
639 * The MSI won't generate an event, since it's being written back
640 * into the command queue.
643 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
644 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
649 * Wait until the SMMU cons index passes llq->prod.
650 * Must be called with the cmdq lock held in some capacity.
652 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
653 struct arm_smmu_ll_queue *llq)
655 struct arm_smmu_queue_poll qp;
656 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
657 u32 prod = llq->prod;
660 queue_poll_init(smmu, &qp);
661 llq->val = READ_ONCE(cmdq->q.llq.val);
663 if (queue_consumed(llq, prod))
666 ret = queue_poll(&qp);
669 * This needs to be a readl() so that our subsequent call
670 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
672 * Specifically, we need to ensure that we observe all
673 * shared_lock()s by other CMD_SYNCs that share our owner,
674 * so that a failing call to tryunlock() means that we're
675 * the last one out and therefore we can safely advance
676 * cmdq->q.llq.cons. Roughly speaking:
678 * CPU 0 CPU1 CPU2 (us)
688 * <control dependency>
694 * Requires us to see CPU 0's shared_lock() acquisition.
696 llq->cons = readl(cmdq->q.cons_reg);
702 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
703 struct arm_smmu_ll_queue *llq)
705 if (smmu->options & ARM_SMMU_OPT_MSIPOLL)
706 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
708 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
711 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
715 struct arm_smmu_ll_queue llq = {
716 .max_n_shift = cmdq->q.llq.max_n_shift,
720 for (i = 0; i < n; ++i) {
721 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
723 prod = queue_inc_prod_n(&llq, i);
724 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
729 * This is the actual insertion function, and provides the following
730 * ordering guarantees to callers:
732 * - There is a dma_wmb() before publishing any commands to the queue.
733 * This can be relied upon to order prior writes to data structures
734 * in memory (such as a CD or an STE) before the command.
736 * - On completion of a CMD_SYNC, there is a control dependency.
737 * This can be relied upon to order subsequent writes to memory (e.g.
738 * freeing an IOVA) after completion of the CMD_SYNC.
740 * - Command insertion is totally ordered, so if two CPUs each race to
741 * insert their own list of commands then all of the commands from one
742 * CPU will appear before any of the commands from the other CPU.
744 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
745 u64 *cmds, int n, bool sync)
747 u64 cmd_sync[CMDQ_ENT_DWORDS];
751 struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
752 struct arm_smmu_ll_queue llq, head;
755 llq.max_n_shift = cmdq->q.llq.max_n_shift;
757 /* 1. Allocate some space in the queue */
758 local_irq_save(flags);
759 llq.val = READ_ONCE(cmdq->q.llq.val);
763 while (!queue_has_space(&llq, n + sync)) {
764 local_irq_restore(flags);
765 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
766 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
767 local_irq_save(flags);
770 head.cons = llq.cons;
771 head.prod = queue_inc_prod_n(&llq, n + sync) |
772 CMDQ_PROD_OWNED_FLAG;
774 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
780 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
781 head.prod &= ~CMDQ_PROD_OWNED_FLAG;
782 llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
785 * 2. Write our commands into the queue
786 * Dependency ordering from the cmpxchg() loop above.
788 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
790 prod = queue_inc_prod_n(&llq, n);
791 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);
792 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
795 * In order to determine completion of our CMD_SYNC, we must
796 * ensure that the queue can't wrap twice without us noticing.
797 * We achieve that by taking the cmdq lock as shared before
798 * marking our slot as valid.
800 arm_smmu_cmdq_shared_lock(cmdq);
803 /* 3. Mark our slots as valid, ensuring commands are visible first */
805 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
807 /* 4. If we are the owner, take control of the SMMU hardware */
809 /* a. Wait for previous owner to finish */
810 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
812 /* b. Stop gathering work by clearing the owned flag */
813 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
814 &cmdq->q.llq.atomic.prod);
815 prod &= ~CMDQ_PROD_OWNED_FLAG;
818 * c. Wait for any gathered work to be written to the queue.
819 * Note that we read our own entries so that we have the control
820 * dependency required by (d).
822 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
825 * d. Advance the hardware prod pointer
826 * Control dependency ordering from the entries becoming valid.
828 writel_relaxed(prod, cmdq->q.prod_reg);
831 * e. Tell the next owner we're done
832 * Make sure we've updated the hardware first, so that we don't
833 * race to update prod and potentially move it backwards.
835 atomic_set_release(&cmdq->owner_prod, prod);
838 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
840 llq.prod = queue_inc_prod_n(&llq, n);
841 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
843 dev_err_ratelimited(smmu->dev,
844 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
846 readl_relaxed(cmdq->q.prod_reg),
847 readl_relaxed(cmdq->q.cons_reg));
851 * Try to unlock the cmdq lock. This will fail if we're the last
852 * reader, in which case we can safely update cmdq->q.llq.cons
854 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
855 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
856 arm_smmu_cmdq_shared_unlock(cmdq);
860 local_irq_restore(flags);
864 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
865 struct arm_smmu_cmdq_ent *ent,
868 u64 cmd[CMDQ_ENT_DWORDS];
870 if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
871 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
876 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync);
879 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
880 struct arm_smmu_cmdq_ent *ent)
882 return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
885 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
886 struct arm_smmu_cmdq_ent *ent)
888 return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
891 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
892 struct arm_smmu_cmdq_batch *cmds,
893 struct arm_smmu_cmdq_ent *cmd)
897 if (cmds->num == CMDQ_BATCH_ENTRIES - 1 &&
898 (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC)) {
899 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
903 if (cmds->num == CMDQ_BATCH_ENTRIES) {
904 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);
908 index = cmds->num * CMDQ_ENT_DWORDS;
909 if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
910 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
918 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
919 struct arm_smmu_cmdq_batch *cmds)
921 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
924 static int arm_smmu_page_response(struct device *dev,
925 struct iommu_fault_event *unused,
926 struct iommu_page_response *resp)
928 struct arm_smmu_cmdq_ent cmd = {0};
929 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
930 int sid = master->streams[0].id;
932 if (master->stall_enabled) {
933 cmd.opcode = CMDQ_OP_RESUME;
934 cmd.resume.sid = sid;
935 cmd.resume.stag = resp->grpid;
936 switch (resp->code) {
937 case IOMMU_PAGE_RESP_INVALID:
938 case IOMMU_PAGE_RESP_FAILURE:
939 cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
941 case IOMMU_PAGE_RESP_SUCCESS:
942 cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
951 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
953 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
954 * RESUME consumption guarantees that the stalled transaction will be
955 * terminated... at some point in the future. PRI_RESP is fire and
962 /* Context descriptor manipulation functions */
963 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
965 struct arm_smmu_cmdq_ent cmd = {
966 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ?
967 CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
971 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
974 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
979 struct arm_smmu_master *master;
980 struct arm_smmu_cmdq_batch cmds;
981 struct arm_smmu_device *smmu = smmu_domain->smmu;
982 struct arm_smmu_cmdq_ent cmd = {
983 .opcode = CMDQ_OP_CFGI_CD,
992 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
993 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
994 for (i = 0; i < master->num_streams; i++) {
995 cmd.cfgi.sid = master->streams[i].id;
996 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
999 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1001 arm_smmu_cmdq_batch_submit(smmu, &cmds);
1004 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
1005 struct arm_smmu_l1_ctx_desc *l1_desc)
1007 size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1009 l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
1010 &l1_desc->l2ptr_dma, GFP_KERNEL);
1011 if (!l1_desc->l2ptr) {
1013 "failed to allocate context descriptor table\n");
1019 static void arm_smmu_write_cd_l1_desc(__le64 *dst,
1020 struct arm_smmu_l1_ctx_desc *l1_desc)
1022 u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |
1025 /* See comment in arm_smmu_write_ctx_desc() */
1026 WRITE_ONCE(*dst, cpu_to_le64(val));
1029 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,
1034 struct arm_smmu_l1_ctx_desc *l1_desc;
1035 struct arm_smmu_device *smmu = smmu_domain->smmu;
1036 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1038 if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1039 return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
1041 idx = ssid >> CTXDESC_SPLIT;
1042 l1_desc = &cdcfg->l1_desc[idx];
1043 if (!l1_desc->l2ptr) {
1044 if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
1047 l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
1048 arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
1049 /* An invalid L1CD can be cached */
1050 arm_smmu_sync_cd(smmu_domain, ssid, false);
1052 idx = ssid & (CTXDESC_L2_ENTRIES - 1);
1053 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
1056 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
1057 struct arm_smmu_ctx_desc *cd)
1060 * This function handles the following cases:
1062 * (1) Install primary CD, for normal DMA traffic (SSID = IOMMU_NO_PASID = 0).
1063 * (2) Install a secondary CD, for SID+SSID traffic.
1064 * (3) Update ASID of a CD. Atomically write the first 64 bits of the
1065 * CD, then invalidate the old entry and mappings.
1066 * (4) Quiesce the context without clearing the valid bit. Disable
1067 * translation, and ignore any translation fault.
1068 * (5) Remove a secondary CD.
1074 if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
1077 cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
1081 val = le64_to_cpu(cdptr[0]);
1082 cd_live = !!(val & CTXDESC_CD_0_V);
1084 if (!cd) { /* (5) */
1086 } else if (cd == &quiet_cd) { /* (4) */
1087 val |= CTXDESC_CD_0_TCR_EPD0;
1088 } else if (cd_live) { /* (3) */
1089 val &= ~CTXDESC_CD_0_ASID;
1090 val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
1092 * Until CD+TLB invalidation, both ASIDs may be used for tagging
1093 * this substream's traffic
1095 } else { /* (1) and (2) */
1096 cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
1098 cdptr[3] = cpu_to_le64(cd->mair);
1101 * STE is live, and the SMMU might read dwords of this CD in any
1102 * order. Ensure that it observes valid values before reading
1105 arm_smmu_sync_cd(smmu_domain, ssid, true);
1111 CTXDESC_CD_0_R | CTXDESC_CD_0_A |
1112 (cd->mm ? 0 : CTXDESC_CD_0_ASET) |
1114 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
1117 if (smmu_domain->stall_enabled)
1118 val |= CTXDESC_CD_0_S;
1122 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
1123 * "Configuration structures and configuration invalidation completion"
1125 * The size of single-copy atomic reads made by the SMMU is
1126 * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
1127 * field within an aligned 64-bit span of a structure can be altered
1128 * without first making the structure invalid.
1130 WRITE_ONCE(cdptr[0], cpu_to_le64(val));
1131 arm_smmu_sync_cd(smmu_domain, ssid, true);
1135 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
1139 size_t max_contexts;
1140 struct arm_smmu_device *smmu = smmu_domain->smmu;
1141 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1142 struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
1144 max_contexts = 1 << cfg->s1cdmax;
1146 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1147 max_contexts <= CTXDESC_L2_ENTRIES) {
1148 cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1149 cdcfg->num_l1_ents = max_contexts;
1151 l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
1153 cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1154 cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
1155 CTXDESC_L2_ENTRIES);
1157 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
1158 sizeof(*cdcfg->l1_desc),
1160 if (!cdcfg->l1_desc)
1163 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1166 cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
1168 if (!cdcfg->cdtab) {
1169 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1177 if (cdcfg->l1_desc) {
1178 devm_kfree(smmu->dev, cdcfg->l1_desc);
1179 cdcfg->l1_desc = NULL;
1184 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
1187 size_t size, l1size;
1188 struct arm_smmu_device *smmu = smmu_domain->smmu;
1189 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1191 if (cdcfg->l1_desc) {
1192 size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1194 for (i = 0; i < cdcfg->num_l1_ents; i++) {
1195 if (!cdcfg->l1_desc[i].l2ptr)
1198 dmam_free_coherent(smmu->dev, size,
1199 cdcfg->l1_desc[i].l2ptr,
1200 cdcfg->l1_desc[i].l2ptr_dma);
1202 devm_kfree(smmu->dev, cdcfg->l1_desc);
1203 cdcfg->l1_desc = NULL;
1205 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1207 l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
1210 dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
1211 cdcfg->cdtab_dma = 0;
1212 cdcfg->cdtab = NULL;
1215 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd)
1218 struct arm_smmu_ctx_desc *old_cd;
1223 free = refcount_dec_and_test(&cd->refs);
1225 old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid);
1226 WARN_ON(old_cd != cd);
1231 /* Stream table manipulation functions */
1233 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1237 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1238 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1240 /* See comment in arm_smmu_write_ctx_desc() */
1241 WRITE_ONCE(*dst, cpu_to_le64(val));
1244 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1246 struct arm_smmu_cmdq_ent cmd = {
1247 .opcode = CMDQ_OP_CFGI_STE,
1254 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1257 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1261 * This is hideously complicated, but we only really care about
1262 * three cases at the moment:
1264 * 1. Invalid (all zero) -> bypass/fault (init)
1265 * 2. Bypass/fault -> translation/bypass (attach)
1266 * 3. Translation/bypass -> bypass/fault (detach)
1268 * Given that we can't update the STE atomically and the SMMU
1269 * doesn't read the thing in a defined order, that leaves us
1270 * with the following maintenance requirements:
1272 * 1. Update Config, return (init time STEs aren't live)
1273 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1274 * 3. Update Config, sync
1276 u64 val = le64_to_cpu(dst[0]);
1277 bool ste_live = false;
1278 struct arm_smmu_device *smmu = NULL;
1279 struct arm_smmu_s1_cfg *s1_cfg = NULL;
1280 struct arm_smmu_s2_cfg *s2_cfg = NULL;
1281 struct arm_smmu_domain *smmu_domain = NULL;
1282 struct arm_smmu_cmdq_ent prefetch_cmd = {
1283 .opcode = CMDQ_OP_PREFETCH_CFG,
1290 smmu_domain = master->domain;
1291 smmu = master->smmu;
1295 switch (smmu_domain->stage) {
1296 case ARM_SMMU_DOMAIN_S1:
1297 s1_cfg = &smmu_domain->s1_cfg;
1299 case ARM_SMMU_DOMAIN_S2:
1300 case ARM_SMMU_DOMAIN_NESTED:
1301 s2_cfg = &smmu_domain->s2_cfg;
1308 if (val & STRTAB_STE_0_V) {
1309 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1310 case STRTAB_STE_0_CFG_BYPASS:
1312 case STRTAB_STE_0_CFG_S1_TRANS:
1313 case STRTAB_STE_0_CFG_S2_TRANS:
1316 case STRTAB_STE_0_CFG_ABORT:
1317 BUG_ON(!disable_bypass);
1320 BUG(); /* STE corruption */
1324 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1325 val = STRTAB_STE_0_V;
1328 if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1329 if (!smmu_domain && disable_bypass)
1330 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1332 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1334 dst[0] = cpu_to_le64(val);
1335 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1336 STRTAB_STE_1_SHCFG_INCOMING));
1337 dst[2] = 0; /* Nuke the VMID */
1339 * The SMMU can perform negative caching, so we must sync
1340 * the STE regardless of whether the old value was live.
1343 arm_smmu_sync_ste_for_sid(smmu, sid);
1348 u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?
1349 STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
1352 dst[1] = cpu_to_le64(
1353 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
1354 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1355 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1356 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1357 FIELD_PREP(STRTAB_STE_1_STRW, strw));
1359 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1360 !master->stall_enabled)
1361 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1363 val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1364 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1365 FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |
1366 FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1371 dst[2] = cpu_to_le64(
1372 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1373 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1375 STRTAB_STE_2_S2ENDI |
1377 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1380 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1382 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1385 if (master->ats_enabled)
1386 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1387 STRTAB_STE_1_EATS_TRANS));
1389 arm_smmu_sync_ste_for_sid(smmu, sid);
1390 /* See comment in arm_smmu_write_ctx_desc() */
1391 WRITE_ONCE(dst[0], cpu_to_le64(val));
1392 arm_smmu_sync_ste_for_sid(smmu, sid);
1394 /* It's likely that we'll want to use the new STE soon */
1395 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1396 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1399 static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent, bool force)
1402 u64 val = STRTAB_STE_0_V;
1404 if (disable_bypass && !force)
1405 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1407 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1409 for (i = 0; i < nent; ++i) {
1410 strtab[0] = cpu_to_le64(val);
1411 strtab[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1412 STRTAB_STE_1_SHCFG_INCOMING));
1414 strtab += STRTAB_STE_DWORDS;
1418 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1422 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1423 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1428 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1429 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1431 desc->span = STRTAB_SPLIT + 1;
1432 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1436 "failed to allocate l2 stream table for SID %u\n",
1441 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false);
1442 arm_smmu_write_strtab_l1_desc(strtab, desc);
1446 static struct arm_smmu_master *
1447 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
1449 struct rb_node *node;
1450 struct arm_smmu_stream *stream;
1452 lockdep_assert_held(&smmu->streams_mutex);
1454 node = smmu->streams.rb_node;
1456 stream = rb_entry(node, struct arm_smmu_stream, node);
1457 if (stream->id < sid)
1458 node = node->rb_right;
1459 else if (stream->id > sid)
1460 node = node->rb_left;
1462 return stream->master;
1468 /* IRQ and event handlers */
1469 static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
1474 struct arm_smmu_master *master;
1475 bool ssid_valid = evt[0] & EVTQ_0_SSV;
1476 u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]);
1477 struct iommu_fault_event fault_evt = { };
1478 struct iommu_fault *flt = &fault_evt.fault;
1480 switch (FIELD_GET(EVTQ_0_ID, evt[0])) {
1481 case EVT_ID_TRANSLATION_FAULT:
1482 reason = IOMMU_FAULT_REASON_PTE_FETCH;
1484 case EVT_ID_ADDR_SIZE_FAULT:
1485 reason = IOMMU_FAULT_REASON_OOR_ADDRESS;
1487 case EVT_ID_ACCESS_FAULT:
1488 reason = IOMMU_FAULT_REASON_ACCESS;
1490 case EVT_ID_PERMISSION_FAULT:
1491 reason = IOMMU_FAULT_REASON_PERMISSION;
1497 /* Stage-2 is always pinned at the moment */
1498 if (evt[1] & EVTQ_1_S2)
1501 if (evt[1] & EVTQ_1_RnW)
1502 perm |= IOMMU_FAULT_PERM_READ;
1504 perm |= IOMMU_FAULT_PERM_WRITE;
1506 if (evt[1] & EVTQ_1_InD)
1507 perm |= IOMMU_FAULT_PERM_EXEC;
1509 if (evt[1] & EVTQ_1_PnU)
1510 perm |= IOMMU_FAULT_PERM_PRIV;
1512 if (evt[1] & EVTQ_1_STALL) {
1513 flt->type = IOMMU_FAULT_PAGE_REQ;
1514 flt->prm = (struct iommu_fault_page_request) {
1515 .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1516 .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]),
1518 .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]),
1522 flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1523 flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1526 flt->type = IOMMU_FAULT_DMA_UNRECOV;
1527 flt->event = (struct iommu_fault_unrecoverable) {
1529 .flags = IOMMU_FAULT_UNRECOV_ADDR_VALID,
1531 .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]),
1535 flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID;
1536 flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1540 mutex_lock(&smmu->streams_mutex);
1541 master = arm_smmu_find_master(smmu, sid);
1547 ret = iommu_report_device_fault(master->dev, &fault_evt);
1548 if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) {
1549 /* Nobody cared, abort the access */
1550 struct iommu_page_response resp = {
1551 .pasid = flt->prm.pasid,
1552 .grpid = flt->prm.grpid,
1553 .code = IOMMU_PAGE_RESP_FAILURE,
1555 arm_smmu_page_response(master->dev, &fault_evt, &resp);
1559 mutex_unlock(&smmu->streams_mutex);
1563 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1566 struct arm_smmu_device *smmu = dev;
1567 struct arm_smmu_queue *q = &smmu->evtq.q;
1568 struct arm_smmu_ll_queue *llq = &q->llq;
1569 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1570 DEFAULT_RATELIMIT_BURST);
1571 u64 evt[EVTQ_ENT_DWORDS];
1574 while (!queue_remove_raw(q, evt)) {
1575 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1577 ret = arm_smmu_handle_evt(smmu, evt);
1578 if (!ret || !__ratelimit(&rs))
1581 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1582 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1583 dev_info(smmu->dev, "\t0x%016llx\n",
1584 (unsigned long long)evt[i]);
1590 * Not much we can do on overflow, so scream and pretend we're
1593 if (queue_sync_prod_in(q) == -EOVERFLOW)
1594 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1595 } while (!queue_empty(llq));
1597 /* Sync our overflow flag, as we believe we're up to speed */
1598 queue_sync_cons_ovf(q);
1602 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1608 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1609 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1610 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
1611 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1612 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1614 dev_info(smmu->dev, "unexpected PRI request received:\n");
1616 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1617 sid, ssid, grpid, last ? "L" : "",
1618 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1619 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1620 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1621 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1622 evt[1] & PRIQ_1_ADDR_MASK);
1625 struct arm_smmu_cmdq_ent cmd = {
1626 .opcode = CMDQ_OP_PRI_RESP,
1627 .substream_valid = ssv,
1632 .resp = PRI_RESP_DENY,
1636 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1640 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1642 struct arm_smmu_device *smmu = dev;
1643 struct arm_smmu_queue *q = &smmu->priq.q;
1644 struct arm_smmu_ll_queue *llq = &q->llq;
1645 u64 evt[PRIQ_ENT_DWORDS];
1648 while (!queue_remove_raw(q, evt))
1649 arm_smmu_handle_ppr(smmu, evt);
1651 if (queue_sync_prod_in(q) == -EOVERFLOW)
1652 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1653 } while (!queue_empty(llq));
1655 /* Sync our overflow flag, as we believe we're up to speed */
1656 queue_sync_cons_ovf(q);
1660 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1662 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1664 u32 gerror, gerrorn, active;
1665 struct arm_smmu_device *smmu = dev;
1667 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1668 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1670 active = gerror ^ gerrorn;
1671 if (!(active & GERROR_ERR_MASK))
1672 return IRQ_NONE; /* No errors pending */
1675 "unexpected global error reported (0x%08x), this could be serious\n",
1678 if (active & GERROR_SFM_ERR) {
1679 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1680 arm_smmu_device_disable(smmu);
1683 if (active & GERROR_MSI_GERROR_ABT_ERR)
1684 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1686 if (active & GERROR_MSI_PRIQ_ABT_ERR)
1687 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1689 if (active & GERROR_MSI_EVTQ_ABT_ERR)
1690 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1692 if (active & GERROR_MSI_CMDQ_ABT_ERR)
1693 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1695 if (active & GERROR_PRIQ_ABT_ERR)
1696 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1698 if (active & GERROR_EVTQ_ABT_ERR)
1699 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1701 if (active & GERROR_CMDQ_ERR)
1702 arm_smmu_cmdq_skip_err(smmu);
1704 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1708 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1710 struct arm_smmu_device *smmu = dev;
1712 arm_smmu_evtq_thread(irq, dev);
1713 if (smmu->features & ARM_SMMU_FEAT_PRI)
1714 arm_smmu_priq_thread(irq, dev);
1719 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1721 arm_smmu_gerror_handler(irq, dev);
1722 return IRQ_WAKE_THREAD;
1726 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1727 struct arm_smmu_cmdq_ent *cmd)
1731 /* ATC invalidates are always on 4096-bytes pages */
1732 size_t inval_grain_shift = 12;
1733 unsigned long page_start, page_end;
1738 * If substream_valid is clear, the PCIe TLP is sent without a PASID
1739 * prefix. In that case all ATC entries within the address range are
1740 * invalidated, including those that were requested with a PASID! There
1741 * is no way to invalidate only entries without PASID.
1743 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
1744 * traffic), translation requests without PASID create ATC entries
1745 * without PASID, which must be invalidated with substream_valid clear.
1746 * This has the unpleasant side-effect of invalidating all PASID-tagged
1747 * ATC entries within the address range.
1749 *cmd = (struct arm_smmu_cmdq_ent) {
1750 .opcode = CMDQ_OP_ATC_INV,
1751 .substream_valid = (ssid != IOMMU_NO_PASID),
1756 cmd->atc.size = ATC_INV_SIZE_ALL;
1760 page_start = iova >> inval_grain_shift;
1761 page_end = (iova + size - 1) >> inval_grain_shift;
1764 * In an ATS Invalidate Request, the address must be aligned on the
1765 * range size, which must be a power of two number of page sizes. We
1766 * thus have to choose between grossly over-invalidating the region, or
1767 * splitting the invalidation into multiple commands. For simplicity
1768 * we'll go with the first solution, but should refine it in the future
1769 * if multiple commands are shown to be more efficient.
1771 * Find the smallest power of two that covers the range. The most
1772 * significant differing bit between the start and end addresses,
1773 * fls(start ^ end), indicates the required span. For example:
1775 * We want to invalidate pages [8; 11]. This is already the ideal range:
1776 * x = 0b1000 ^ 0b1011 = 0b11
1777 * span = 1 << fls(x) = 4
1779 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1780 * x = 0b0111 ^ 0b1010 = 0b1101
1781 * span = 1 << fls(x) = 16
1783 log2_span = fls_long(page_start ^ page_end);
1784 span_mask = (1ULL << log2_span) - 1;
1786 page_start &= ~span_mask;
1788 cmd->atc.addr = page_start << inval_grain_shift;
1789 cmd->atc.size = log2_span;
1792 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
1795 struct arm_smmu_cmdq_ent cmd;
1796 struct arm_smmu_cmdq_batch cmds;
1798 arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
1801 for (i = 0; i < master->num_streams; i++) {
1802 cmd.atc.sid = master->streams[i].id;
1803 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
1806 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
1809 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
1810 unsigned long iova, size_t size)
1813 unsigned long flags;
1814 struct arm_smmu_cmdq_ent cmd;
1815 struct arm_smmu_master *master;
1816 struct arm_smmu_cmdq_batch cmds;
1818 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1822 * Ensure that we've completed prior invalidation of the main TLBs
1823 * before we read 'nr_ats_masters' in case of a concurrent call to
1824 * arm_smmu_enable_ats():
1826 * // unmap() // arm_smmu_enable_ats()
1827 * TLBI+SYNC atomic_inc(&nr_ats_masters);
1829 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel()
1831 * Ensures that we always see the incremented 'nr_ats_masters' count if
1832 * ATS was enabled at the PCI device before completion of the TLBI.
1835 if (!atomic_read(&smmu_domain->nr_ats_masters))
1838 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1842 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1843 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
1844 if (!master->ats_enabled)
1847 for (i = 0; i < master->num_streams; i++) {
1848 cmd.atc.sid = master->streams[i].id;
1849 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
1852 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1854 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
1857 /* IO_PGTABLE API */
1858 static void arm_smmu_tlb_inv_context(void *cookie)
1860 struct arm_smmu_domain *smmu_domain = cookie;
1861 struct arm_smmu_device *smmu = smmu_domain->smmu;
1862 struct arm_smmu_cmdq_ent cmd;
1865 * NOTE: when io-pgtable is in non-strict mode, we may get here with
1866 * PTEs previously cleared by unmaps on the current CPU not yet visible
1867 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
1868 * insertion to guarantee those are observed before the TLBI. Do be
1871 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1872 arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);
1874 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1875 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1876 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1878 arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, 0, 0);
1881 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
1882 unsigned long iova, size_t size,
1884 struct arm_smmu_domain *smmu_domain)
1886 struct arm_smmu_device *smmu = smmu_domain->smmu;
1887 unsigned long end = iova + size, num_pages = 0, tg = 0;
1888 size_t inv_range = granule;
1889 struct arm_smmu_cmdq_batch cmds;
1894 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1895 /* Get the leaf page size */
1896 tg = __ffs(smmu_domain->domain.pgsize_bitmap);
1898 num_pages = size >> tg;
1900 /* Convert page size of 12,14,16 (log2) to 1,2,3 */
1901 cmd->tlbi.tg = (tg - 10) / 2;
1904 * Determine what level the granule is at. For non-leaf, both
1905 * io-pgtable and SVA pass a nominal last-level granule because
1906 * they don't know what level(s) actually apply, so ignore that
1907 * and leave TTL=0. However for various errata reasons we still
1908 * want to use a range command, so avoid the SVA corner case
1909 * where both scale and num could be 0 as well.
1912 cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
1913 else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
1919 while (iova < end) {
1920 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1922 * On each iteration of the loop, the range is 5 bits
1923 * worth of the aligned size remaining.
1924 * The range in pages is:
1926 * range = (num_pages & (0x1f << __ffs(num_pages)))
1928 unsigned long scale, num;
1930 /* Determine the power of 2 multiple number of pages */
1931 scale = __ffs(num_pages);
1932 cmd->tlbi.scale = scale;
1934 /* Determine how many chunks of 2^scale size we have */
1935 num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
1936 cmd->tlbi.num = num - 1;
1938 /* range is num * 2^scale * pgsize */
1939 inv_range = num << (scale + tg);
1941 /* Clear out the lower order bits for the next iteration */
1942 num_pages -= num << scale;
1945 cmd->tlbi.addr = iova;
1946 arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
1949 arm_smmu_cmdq_batch_submit(smmu, &cmds);
1952 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
1953 size_t granule, bool leaf,
1954 struct arm_smmu_domain *smmu_domain)
1956 struct arm_smmu_cmdq_ent cmd = {
1962 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1963 cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
1964 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
1965 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1967 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1968 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1970 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
1973 * Unfortunately, this can't be leaf-only since we may have
1974 * zapped an entire table.
1976 arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, iova, size);
1979 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
1980 size_t granule, bool leaf,
1981 struct arm_smmu_domain *smmu_domain)
1983 struct arm_smmu_cmdq_ent cmd = {
1984 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
1985 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
1992 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
1995 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
1996 unsigned long iova, size_t granule,
1999 struct arm_smmu_domain *smmu_domain = cookie;
2000 struct iommu_domain *domain = &smmu_domain->domain;
2002 iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2005 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2006 size_t granule, void *cookie)
2008 arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
2011 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2012 .tlb_flush_all = arm_smmu_tlb_inv_context,
2013 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2014 .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2018 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2020 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2023 case IOMMU_CAP_CACHE_COHERENCY:
2024 /* Assume that a coherent TCU implies coherent TBUs */
2025 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2026 case IOMMU_CAP_NOEXEC:
2027 case IOMMU_CAP_DEFERRED_FLUSH:
2034 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
2036 struct arm_smmu_domain *smmu_domain;
2038 if (type == IOMMU_DOMAIN_SVA)
2039 return arm_smmu_sva_domain_alloc();
2041 if (type != IOMMU_DOMAIN_UNMANAGED &&
2042 type != IOMMU_DOMAIN_DMA &&
2043 type != IOMMU_DOMAIN_IDENTITY)
2047 * Allocate the domain and initialise some of its data structures.
2048 * We can't really do anything meaningful until we've added a
2051 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2055 mutex_init(&smmu_domain->init_mutex);
2056 INIT_LIST_HEAD(&smmu_domain->devices);
2057 spin_lock_init(&smmu_domain->devices_lock);
2058 INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
2060 return &smmu_domain->domain;
2063 static void arm_smmu_domain_free(struct iommu_domain *domain)
2065 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2066 struct arm_smmu_device *smmu = smmu_domain->smmu;
2068 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2070 /* Free the CD and ASID, if we allocated them */
2071 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2072 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2074 /* Prevent SVA from touching the CD while we're freeing it */
2075 mutex_lock(&arm_smmu_asid_lock);
2076 if (cfg->cdcfg.cdtab)
2077 arm_smmu_free_cd_tables(smmu_domain);
2078 arm_smmu_free_asid(&cfg->cd);
2079 mutex_unlock(&arm_smmu_asid_lock);
2081 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2083 ida_free(&smmu->vmid_map, cfg->vmid);
2089 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
2090 struct arm_smmu_master *master,
2091 struct io_pgtable_cfg *pgtbl_cfg)
2095 struct arm_smmu_device *smmu = smmu_domain->smmu;
2096 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2097 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2099 refcount_set(&cfg->cd.refs, 1);
2101 /* Prevent SVA from modifying the ASID until it is written to the CD */
2102 mutex_lock(&arm_smmu_asid_lock);
2103 ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,
2104 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2108 cfg->s1cdmax = master->ssid_bits;
2110 smmu_domain->stall_enabled = master->stall_enabled;
2112 ret = arm_smmu_alloc_cd_tables(smmu_domain);
2116 cfg->cd.asid = (u16)asid;
2117 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
2118 cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
2119 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
2120 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
2121 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
2122 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
2123 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
2124 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
2125 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
2128 * Note that this will end up calling arm_smmu_sync_cd() before
2129 * the master has been added to the devices list for this domain.
2130 * This isn't an issue because the STE hasn't been installed yet.
2132 ret = arm_smmu_write_ctx_desc(smmu_domain, IOMMU_NO_PASID, &cfg->cd);
2134 goto out_free_cd_tables;
2136 mutex_unlock(&arm_smmu_asid_lock);
2140 arm_smmu_free_cd_tables(smmu_domain);
2142 arm_smmu_free_asid(&cfg->cd);
2144 mutex_unlock(&arm_smmu_asid_lock);
2148 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
2149 struct arm_smmu_master *master,
2150 struct io_pgtable_cfg *pgtbl_cfg)
2153 struct arm_smmu_device *smmu = smmu_domain->smmu;
2154 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2155 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2157 /* Reserve VMID 0 for stage-2 bypass STEs */
2158 vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2163 vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2164 cfg->vmid = (u16)vmid;
2165 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2166 cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
2167 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
2168 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
2169 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
2170 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
2171 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
2172 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2176 static int arm_smmu_domain_finalise(struct iommu_domain *domain,
2177 struct arm_smmu_master *master)
2180 unsigned long ias, oas;
2181 enum io_pgtable_fmt fmt;
2182 struct io_pgtable_cfg pgtbl_cfg;
2183 struct io_pgtable_ops *pgtbl_ops;
2184 int (*finalise_stage_fn)(struct arm_smmu_domain *,
2185 struct arm_smmu_master *,
2186 struct io_pgtable_cfg *);
2187 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2188 struct arm_smmu_device *smmu = smmu_domain->smmu;
2190 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2191 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2195 /* Restrict the stage to what we can actually support */
2196 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2197 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2198 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2199 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2201 switch (smmu_domain->stage) {
2202 case ARM_SMMU_DOMAIN_S1:
2203 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2204 ias = min_t(unsigned long, ias, VA_BITS);
2206 fmt = ARM_64_LPAE_S1;
2207 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2209 case ARM_SMMU_DOMAIN_NESTED:
2210 case ARM_SMMU_DOMAIN_S2:
2213 fmt = ARM_64_LPAE_S2;
2214 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2220 pgtbl_cfg = (struct io_pgtable_cfg) {
2221 .pgsize_bitmap = smmu->pgsize_bitmap,
2224 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2225 .tlb = &arm_smmu_flush_ops,
2226 .iommu_dev = smmu->dev,
2229 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2233 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2234 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2235 domain->geometry.force_aperture = true;
2237 ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
2239 free_io_pgtable_ops(pgtbl_ops);
2243 smmu_domain->pgtbl_ops = pgtbl_ops;
2247 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2250 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2252 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2253 struct arm_smmu_strtab_l1_desc *l1_desc;
2256 /* Two-level walk */
2257 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
2258 l1_desc = &cfg->l1_desc[idx];
2259 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2260 step = &l1_desc->l2ptr[idx];
2262 /* Simple linear lookup */
2263 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2269 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
2272 struct arm_smmu_device *smmu = master->smmu;
2274 for (i = 0; i < master->num_streams; ++i) {
2275 u32 sid = master->streams[i].id;
2276 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
2278 /* Bridged PCI devices may end up with duplicated IDs */
2279 for (j = 0; j < i; j++)
2280 if (master->streams[j].id == sid)
2285 arm_smmu_write_strtab_ent(master, sid, step);
2289 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2291 struct device *dev = master->dev;
2292 struct arm_smmu_device *smmu = master->smmu;
2293 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2295 if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2298 if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2301 return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2304 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2307 struct pci_dev *pdev;
2308 struct arm_smmu_device *smmu = master->smmu;
2309 struct arm_smmu_domain *smmu_domain = master->domain;
2311 /* Don't enable ATS at the endpoint if it's not enabled in the STE */
2312 if (!master->ats_enabled)
2315 /* Smallest Translation Unit: log2 of the smallest supported granule */
2316 stu = __ffs(smmu->pgsize_bitmap);
2317 pdev = to_pci_dev(master->dev);
2319 atomic_inc(&smmu_domain->nr_ats_masters);
2320 arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, 0, 0);
2321 if (pci_enable_ats(pdev, stu))
2322 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2325 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
2327 struct arm_smmu_domain *smmu_domain = master->domain;
2329 if (!master->ats_enabled)
2332 pci_disable_ats(to_pci_dev(master->dev));
2334 * Ensure ATS is disabled at the endpoint before we issue the
2335 * ATC invalidation via the SMMU.
2338 arm_smmu_atc_inv_master(master);
2339 atomic_dec(&smmu_domain->nr_ats_masters);
2342 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2347 struct pci_dev *pdev;
2349 if (!dev_is_pci(master->dev))
2352 pdev = to_pci_dev(master->dev);
2354 features = pci_pasid_features(pdev);
2358 num_pasids = pci_max_pasids(pdev);
2359 if (num_pasids <= 0)
2362 ret = pci_enable_pasid(pdev, features);
2364 dev_err(&pdev->dev, "Failed to enable PASID\n");
2368 master->ssid_bits = min_t(u8, ilog2(num_pasids),
2369 master->smmu->ssid_bits);
2373 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2375 struct pci_dev *pdev;
2377 if (!dev_is_pci(master->dev))
2380 pdev = to_pci_dev(master->dev);
2382 if (!pdev->pasid_enabled)
2385 master->ssid_bits = 0;
2386 pci_disable_pasid(pdev);
2389 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
2391 unsigned long flags;
2392 struct arm_smmu_domain *smmu_domain = master->domain;
2397 arm_smmu_disable_ats(master);
2399 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2400 list_del(&master->domain_head);
2401 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2403 master->domain = NULL;
2404 master->ats_enabled = false;
2405 arm_smmu_install_ste_for_dev(master);
2408 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2411 unsigned long flags;
2412 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2413 struct arm_smmu_device *smmu;
2414 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2415 struct arm_smmu_master *master;
2420 master = dev_iommu_priv_get(dev);
2421 smmu = master->smmu;
2424 * Checking that SVA is disabled ensures that this device isn't bound to
2425 * any mm, and can be safely detached from its old domain. Bonds cannot
2426 * be removed concurrently since we're holding the group mutex.
2428 if (arm_smmu_master_sva_enabled(master)) {
2429 dev_err(dev, "cannot attach - SVA enabled\n");
2433 arm_smmu_detach_dev(master);
2435 mutex_lock(&smmu_domain->init_mutex);
2437 if (!smmu_domain->smmu) {
2438 smmu_domain->smmu = smmu;
2439 ret = arm_smmu_domain_finalise(domain, master);
2441 smmu_domain->smmu = NULL;
2444 } else if (smmu_domain->smmu != smmu) {
2447 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2448 master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
2451 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2452 smmu_domain->stall_enabled != master->stall_enabled) {
2457 master->domain = smmu_domain;
2460 * The SMMU does not support enabling ATS with bypass. When the STE is
2461 * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and
2462 * Translated transactions are denied as though ATS is disabled for the
2463 * stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and
2464 * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry).
2466 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2467 master->ats_enabled = arm_smmu_ats_supported(master);
2469 arm_smmu_install_ste_for_dev(master);
2471 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2472 list_add(&master->domain_head, &smmu_domain->devices);
2473 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2475 arm_smmu_enable_ats(master);
2478 mutex_unlock(&smmu_domain->init_mutex);
2482 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
2483 phys_addr_t paddr, size_t pgsize, size_t pgcount,
2484 int prot, gfp_t gfp, size_t *mapped)
2486 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2491 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
2494 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
2495 size_t pgsize, size_t pgcount,
2496 struct iommu_iotlb_gather *gather)
2498 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2499 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2504 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
2507 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
2509 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2511 if (smmu_domain->smmu)
2512 arm_smmu_tlb_inv_context(smmu_domain);
2515 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
2516 struct iommu_iotlb_gather *gather)
2518 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2520 if (!gather->pgsize)
2523 arm_smmu_tlb_inv_range_domain(gather->start,
2524 gather->end - gather->start + 1,
2525 gather->pgsize, true, smmu_domain);
2529 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2531 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2536 return ops->iova_to_phys(ops, iova);
2539 static struct platform_driver arm_smmu_driver;
2542 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2544 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
2547 return dev ? dev_get_drvdata(dev) : NULL;
2550 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2552 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2554 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2555 limit *= 1UL << STRTAB_SPLIT;
2560 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
2562 /* Check the SIDs are in range of the SMMU and our stream table */
2563 if (!arm_smmu_sid_in_range(smmu, sid))
2566 /* Ensure l2 strtab is initialised */
2567 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2568 return arm_smmu_init_l2_strtab(smmu, sid);
2573 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
2574 struct arm_smmu_master *master)
2578 struct arm_smmu_stream *new_stream, *cur_stream;
2579 struct rb_node **new_node, *parent_node = NULL;
2580 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2582 master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
2584 if (!master->streams)
2586 master->num_streams = fwspec->num_ids;
2588 mutex_lock(&smmu->streams_mutex);
2589 for (i = 0; i < fwspec->num_ids; i++) {
2590 u32 sid = fwspec->ids[i];
2592 new_stream = &master->streams[i];
2593 new_stream->id = sid;
2594 new_stream->master = master;
2596 ret = arm_smmu_init_sid_strtab(smmu, sid);
2600 /* Insert into SID tree */
2601 new_node = &(smmu->streams.rb_node);
2603 cur_stream = rb_entry(*new_node, struct arm_smmu_stream,
2605 parent_node = *new_node;
2606 if (cur_stream->id > new_stream->id) {
2607 new_node = &((*new_node)->rb_left);
2608 } else if (cur_stream->id < new_stream->id) {
2609 new_node = &((*new_node)->rb_right);
2611 dev_warn(master->dev,
2612 "stream %u already in tree\n",
2621 rb_link_node(&new_stream->node, parent_node, new_node);
2622 rb_insert_color(&new_stream->node, &smmu->streams);
2626 for (i--; i >= 0; i--)
2627 rb_erase(&master->streams[i].node, &smmu->streams);
2628 kfree(master->streams);
2630 mutex_unlock(&smmu->streams_mutex);
2635 static void arm_smmu_remove_master(struct arm_smmu_master *master)
2638 struct arm_smmu_device *smmu = master->smmu;
2639 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2641 if (!smmu || !master->streams)
2644 mutex_lock(&smmu->streams_mutex);
2645 for (i = 0; i < fwspec->num_ids; i++)
2646 rb_erase(&master->streams[i].node, &smmu->streams);
2647 mutex_unlock(&smmu->streams_mutex);
2649 kfree(master->streams);
2652 static struct iommu_ops arm_smmu_ops;
2654 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
2657 struct arm_smmu_device *smmu;
2658 struct arm_smmu_master *master;
2659 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2661 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2662 return ERR_PTR(-ENODEV);
2664 if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
2665 return ERR_PTR(-EBUSY);
2667 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2669 return ERR_PTR(-ENODEV);
2671 master = kzalloc(sizeof(*master), GFP_KERNEL);
2673 return ERR_PTR(-ENOMEM);
2676 master->smmu = smmu;
2677 INIT_LIST_HEAD(&master->bonds);
2678 dev_iommu_priv_set(dev, master);
2680 ret = arm_smmu_insert_master(smmu, master);
2682 goto err_free_master;
2684 device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
2685 master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
2688 * Note that PASID must be enabled before, and disabled after ATS:
2689 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
2691 * Behavior is undefined if this bit is Set and the value of the PASID
2692 * Enable, Execute Requested Enable, or Privileged Mode Requested bits
2695 arm_smmu_enable_pasid(master);
2697 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
2698 master->ssid_bits = min_t(u8, master->ssid_bits,
2699 CTXDESC_LINEAR_CDMAX);
2701 if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
2702 device_property_read_bool(dev, "dma-can-stall")) ||
2703 smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
2704 master->stall_enabled = true;
2706 return &smmu->iommu;
2710 dev_iommu_priv_set(dev, NULL);
2711 return ERR_PTR(ret);
2714 static void arm_smmu_release_device(struct device *dev)
2716 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2718 if (WARN_ON(arm_smmu_master_sva_enabled(master)))
2719 iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
2720 arm_smmu_detach_dev(master);
2721 arm_smmu_disable_pasid(master);
2722 arm_smmu_remove_master(master);
2726 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2728 struct iommu_group *group;
2731 * We don't support devices sharing stream IDs other than PCI RID
2732 * aliases, since the necessary ID-to-device lookup becomes rather
2733 * impractical given a potential sparse 32-bit stream ID space.
2735 if (dev_is_pci(dev))
2736 group = pci_device_group(dev);
2738 group = generic_device_group(dev);
2743 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
2745 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2748 mutex_lock(&smmu_domain->init_mutex);
2749 if (smmu_domain->smmu)
2752 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2753 mutex_unlock(&smmu_domain->init_mutex);
2758 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2760 return iommu_fwspec_add_ids(dev, args->args, 1);
2763 static void arm_smmu_get_resv_regions(struct device *dev,
2764 struct list_head *head)
2766 struct iommu_resv_region *region;
2767 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2769 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2770 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
2774 list_add_tail(®ion->list, head);
2776 iommu_dma_get_resv_regions(dev, head);
2779 static int arm_smmu_dev_enable_feature(struct device *dev,
2780 enum iommu_dev_features feat)
2782 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2788 case IOMMU_DEV_FEAT_IOPF:
2789 if (!arm_smmu_master_iopf_supported(master))
2791 if (master->iopf_enabled)
2793 master->iopf_enabled = true;
2795 case IOMMU_DEV_FEAT_SVA:
2796 if (!arm_smmu_master_sva_supported(master))
2798 if (arm_smmu_master_sva_enabled(master))
2800 return arm_smmu_master_enable_sva(master);
2806 static int arm_smmu_dev_disable_feature(struct device *dev,
2807 enum iommu_dev_features feat)
2809 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2815 case IOMMU_DEV_FEAT_IOPF:
2816 if (!master->iopf_enabled)
2818 if (master->sva_enabled)
2820 master->iopf_enabled = false;
2822 case IOMMU_DEV_FEAT_SVA:
2823 if (!arm_smmu_master_sva_enabled(master))
2825 return arm_smmu_master_disable_sva(master);
2832 * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
2833 * PCIe link and save the data to memory by DMA. The hardware is restricted to
2834 * use identity mapping only.
2836 #define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
2837 (pdev)->device == 0xa12e)
2839 static int arm_smmu_def_domain_type(struct device *dev)
2841 if (dev_is_pci(dev)) {
2842 struct pci_dev *pdev = to_pci_dev(dev);
2844 if (IS_HISI_PTT_DEVICE(pdev))
2845 return IOMMU_DOMAIN_IDENTITY;
2851 static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
2853 struct iommu_domain *domain;
2855 domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
2856 if (WARN_ON(IS_ERR(domain)) || !domain)
2859 arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
2862 static struct iommu_ops arm_smmu_ops = {
2863 .capable = arm_smmu_capable,
2864 .domain_alloc = arm_smmu_domain_alloc,
2865 .probe_device = arm_smmu_probe_device,
2866 .release_device = arm_smmu_release_device,
2867 .device_group = arm_smmu_device_group,
2868 .of_xlate = arm_smmu_of_xlate,
2869 .get_resv_regions = arm_smmu_get_resv_regions,
2870 .remove_dev_pasid = arm_smmu_remove_dev_pasid,
2871 .dev_enable_feat = arm_smmu_dev_enable_feature,
2872 .dev_disable_feat = arm_smmu_dev_disable_feature,
2873 .page_response = arm_smmu_page_response,
2874 .def_domain_type = arm_smmu_def_domain_type,
2875 .pgsize_bitmap = -1UL, /* Restricted during device attach */
2876 .owner = THIS_MODULE,
2877 .default_domain_ops = &(const struct iommu_domain_ops) {
2878 .attach_dev = arm_smmu_attach_dev,
2879 .map_pages = arm_smmu_map_pages,
2880 .unmap_pages = arm_smmu_unmap_pages,
2881 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
2882 .iotlb_sync = arm_smmu_iotlb_sync,
2883 .iova_to_phys = arm_smmu_iova_to_phys,
2884 .enable_nesting = arm_smmu_enable_nesting,
2885 .free = arm_smmu_domain_free,
2889 /* Probing and initialisation functions */
2890 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2891 struct arm_smmu_queue *q,
2893 unsigned long prod_off,
2894 unsigned long cons_off,
2895 size_t dwords, const char *name)
2900 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
2901 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2903 if (q->base || qsz < PAGE_SIZE)
2906 q->llq.max_n_shift--;
2911 "failed to allocate queue (0x%zx bytes) for %s\n",
2916 if (!WARN_ON(q->base_dma & (qsz - 1))) {
2917 dev_info(smmu->dev, "allocated %u entries for %s\n",
2918 1 << q->llq.max_n_shift, name);
2921 q->prod_reg = page + prod_off;
2922 q->cons_reg = page + cons_off;
2923 q->ent_dwords = dwords;
2925 q->q_base = Q_BASE_RWA;
2926 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2927 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
2929 q->llq.prod = q->llq.cons = 0;
2933 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
2935 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
2936 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
2938 atomic_set(&cmdq->owner_prod, 0);
2939 atomic_set(&cmdq->lock, 0);
2941 cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
2943 if (!cmdq->valid_map)
2949 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2954 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
2955 ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
2956 CMDQ_ENT_DWORDS, "cmdq");
2960 ret = arm_smmu_cmdq_init(smmu);
2965 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
2966 ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
2967 EVTQ_ENT_DWORDS, "evtq");
2971 if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
2972 (smmu->features & ARM_SMMU_FEAT_STALLS)) {
2973 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
2974 if (!smmu->evtq.iopf)
2979 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2982 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
2983 ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
2984 PRIQ_ENT_DWORDS, "priq");
2987 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2990 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2991 void *strtab = smmu->strtab_cfg.strtab;
2993 cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents,
2994 sizeof(*cfg->l1_desc), GFP_KERNEL);
2998 for (i = 0; i < cfg->num_l1_ents; ++i) {
2999 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
3000 strtab += STRTAB_L1_DESC_DWORDS << 3;
3006 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3011 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3013 /* Calculate the L1 size, capped to the SIDSIZE. */
3014 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
3015 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
3016 cfg->num_l1_ents = 1 << size;
3018 size += STRTAB_SPLIT;
3019 if (size < smmu->sid_bits)
3021 "2-level strtab only covers %u/%u bits of SID\n",
3022 size, smmu->sid_bits);
3024 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
3025 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
3029 "failed to allocate l1 stream table (%u bytes)\n",
3033 cfg->strtab = strtab;
3035 /* Configure strtab_base_cfg for 2 levels */
3036 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
3037 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
3038 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
3039 cfg->strtab_base_cfg = reg;
3041 return arm_smmu_init_l1_strtab(smmu);
3044 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3049 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3051 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
3052 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
3056 "failed to allocate linear stream table (%u bytes)\n",
3060 cfg->strtab = strtab;
3061 cfg->num_l1_ents = 1 << smmu->sid_bits;
3063 /* Configure strtab_base_cfg for a linear table covering all SIDs */
3064 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
3065 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
3066 cfg->strtab_base_cfg = reg;
3068 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false);
3072 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3077 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3078 ret = arm_smmu_init_strtab_2lvl(smmu);
3080 ret = arm_smmu_init_strtab_linear(smmu);
3085 /* Set the strtab base address */
3086 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
3087 reg |= STRTAB_BASE_RA;
3088 smmu->strtab_cfg.strtab_base = reg;
3090 ida_init(&smmu->vmid_map);
3095 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3099 mutex_init(&smmu->streams_mutex);
3100 smmu->streams = RB_ROOT;
3102 ret = arm_smmu_init_queues(smmu);
3106 return arm_smmu_init_strtab(smmu);
3109 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3110 unsigned int reg_off, unsigned int ack_off)
3114 writel_relaxed(val, smmu->base + reg_off);
3115 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3116 1, ARM_SMMU_POLL_TIMEOUT_US);
3119 /* GBPA is "special" */
3120 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3123 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3125 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3126 1, ARM_SMMU_POLL_TIMEOUT_US);
3132 writel_relaxed(reg | GBPA_UPDATE, gbpa);
3133 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3134 1, ARM_SMMU_POLL_TIMEOUT_US);
3137 dev_err(smmu->dev, "GBPA not responding to update\n");
3141 static void arm_smmu_free_msis(void *data)
3143 struct device *dev = data;
3144 platform_msi_domain_free_irqs(dev);
3147 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3149 phys_addr_t doorbell;
3150 struct device *dev = msi_desc_to_dev(desc);
3151 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3152 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
3154 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3155 doorbell &= MSI_CFG0_ADDR_MASK;
3157 writeq_relaxed(doorbell, smmu->base + cfg[0]);
3158 writel_relaxed(msg->data, smmu->base + cfg[1]);
3159 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3162 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3164 int ret, nvec = ARM_SMMU_MAX_MSIS;
3165 struct device *dev = smmu->dev;
3167 /* Clear the MSI address regs */
3168 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3169 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3171 if (smmu->features & ARM_SMMU_FEAT_PRI)
3172 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3176 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3179 if (!dev->msi.domain) {
3180 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3184 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3185 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3187 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3191 smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
3192 smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
3193 smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
3195 /* Add callback to free MSIs on teardown */
3196 devm_add_action(dev, arm_smmu_free_msis, dev);
3199 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3203 arm_smmu_setup_msis(smmu);
3205 /* Request interrupt lines */
3206 irq = smmu->evtq.q.irq;
3208 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3209 arm_smmu_evtq_thread,
3211 "arm-smmu-v3-evtq", smmu);
3213 dev_warn(smmu->dev, "failed to enable evtq irq\n");
3215 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3218 irq = smmu->gerr_irq;
3220 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3221 0, "arm-smmu-v3-gerror", smmu);
3223 dev_warn(smmu->dev, "failed to enable gerror irq\n");
3225 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3228 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3229 irq = smmu->priq.q.irq;
3231 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3232 arm_smmu_priq_thread,
3238 "failed to enable priq irq\n");
3240 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3245 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3248 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3250 /* Disable IRQs first */
3251 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3252 ARM_SMMU_IRQ_CTRLACK);
3254 dev_err(smmu->dev, "failed to disable irqs\n");
3258 irq = smmu->combined_irq;
3261 * Cavium ThunderX2 implementation doesn't support unique irq
3262 * lines. Use a single irq line for all the SMMUv3 interrupts.
3264 ret = devm_request_threaded_irq(smmu->dev, irq,
3265 arm_smmu_combined_irq_handler,
3266 arm_smmu_combined_irq_thread,
3268 "arm-smmu-v3-combined-irq", smmu);
3270 dev_warn(smmu->dev, "failed to enable combined irq\n");
3272 arm_smmu_setup_unique_irqs(smmu);
3274 if (smmu->features & ARM_SMMU_FEAT_PRI)
3275 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3277 /* Enable interrupt generation on the SMMU */
3278 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3279 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3281 dev_warn(smmu->dev, "failed to enable irqs\n");
3286 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3290 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3292 dev_err(smmu->dev, "failed to clear cr0\n");
3297 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
3301 struct arm_smmu_cmdq_ent cmd;
3303 /* Clear CR0 and sync (disables SMMU and queue processing) */
3304 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3305 if (reg & CR0_SMMUEN) {
3306 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3307 WARN_ON(is_kdump_kernel() && !disable_bypass);
3308 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3311 ret = arm_smmu_device_disable(smmu);
3315 /* CR1 (table and queue memory attributes) */
3316 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3317 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3318 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3319 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3320 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3321 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3322 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3324 /* CR2 (random crap) */
3325 reg = CR2_PTM | CR2_RECINVSID;
3327 if (smmu->features & ARM_SMMU_FEAT_E2H)
3330 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3333 writeq_relaxed(smmu->strtab_cfg.strtab_base,
3334 smmu->base + ARM_SMMU_STRTAB_BASE);
3335 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3336 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3339 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3340 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3341 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3343 enables = CR0_CMDQEN;
3344 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3347 dev_err(smmu->dev, "failed to enable command queue\n");
3351 /* Invalidate any cached configuration */
3352 cmd.opcode = CMDQ_OP_CFGI_ALL;
3353 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3355 /* Invalidate any stale TLB entries */
3356 if (smmu->features & ARM_SMMU_FEAT_HYP) {
3357 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3358 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3361 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
3362 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3365 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3366 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
3367 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
3369 enables |= CR0_EVTQEN;
3370 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3373 dev_err(smmu->dev, "failed to enable event queue\n");
3378 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3379 writeq_relaxed(smmu->priq.q.q_base,
3380 smmu->base + ARM_SMMU_PRIQ_BASE);
3381 writel_relaxed(smmu->priq.q.llq.prod,
3382 smmu->page1 + ARM_SMMU_PRIQ_PROD);
3383 writel_relaxed(smmu->priq.q.llq.cons,
3384 smmu->page1 + ARM_SMMU_PRIQ_CONS);
3386 enables |= CR0_PRIQEN;
3387 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3390 dev_err(smmu->dev, "failed to enable PRI queue\n");
3395 if (smmu->features & ARM_SMMU_FEAT_ATS) {
3396 enables |= CR0_ATSCHK;
3397 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3400 dev_err(smmu->dev, "failed to enable ATS check\n");
3405 ret = arm_smmu_setup_irqs(smmu);
3407 dev_err(smmu->dev, "failed to setup irqs\n");
3411 if (is_kdump_kernel())
3412 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
3414 /* Enable the SMMU interface, or ensure bypass */
3415 if (!bypass || disable_bypass) {
3416 enables |= CR0_SMMUEN;
3418 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
3422 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3425 dev_err(smmu->dev, "failed to enable SMMU interface\n");
3432 #define IIDR_IMPLEMENTER_ARM 0x43b
3433 #define IIDR_PRODUCTID_ARM_MMU_600 0x483
3434 #define IIDR_PRODUCTID_ARM_MMU_700 0x487
3436 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
3439 unsigned int implementer, productid, variant, revision;
3441 reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
3442 implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
3443 productid = FIELD_GET(IIDR_PRODUCTID, reg);
3444 variant = FIELD_GET(IIDR_VARIANT, reg);
3445 revision = FIELD_GET(IIDR_REVISION, reg);
3447 switch (implementer) {
3448 case IIDR_IMPLEMENTER_ARM:
3449 switch (productid) {
3450 case IIDR_PRODUCTID_ARM_MMU_600:
3451 /* Arm erratum 1076982 */
3452 if (variant == 0 && revision <= 2)
3453 smmu->features &= ~ARM_SMMU_FEAT_SEV;
3454 /* Arm erratum 1209401 */
3456 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
3458 case IIDR_PRODUCTID_ARM_MMU_700:
3459 /* Arm erratum 2812531 */
3460 smmu->features &= ~ARM_SMMU_FEAT_BTM;
3461 smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
3462 /* Arm errata 2268618, 2812531 */
3463 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
3470 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
3473 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3476 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3478 /* 2-level structures */
3479 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
3480 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3482 if (reg & IDR0_CD2L)
3483 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3486 * Translation table endianness.
3487 * We currently require the same endianness as the CPU, but this
3488 * could be changed later by adding a new IO_PGTABLE_QUIRK.
3490 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
3491 case IDR0_TTENDIAN_MIXED:
3492 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3495 case IDR0_TTENDIAN_BE:
3496 smmu->features |= ARM_SMMU_FEAT_TT_BE;
3499 case IDR0_TTENDIAN_LE:
3500 smmu->features |= ARM_SMMU_FEAT_TT_LE;
3504 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3508 /* Boolean feature flags */
3509 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
3510 smmu->features |= ARM_SMMU_FEAT_PRI;
3512 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
3513 smmu->features |= ARM_SMMU_FEAT_ATS;
3516 smmu->features |= ARM_SMMU_FEAT_SEV;
3518 if (reg & IDR0_MSI) {
3519 smmu->features |= ARM_SMMU_FEAT_MSI;
3520 if (coherent && !disable_msipolling)
3521 smmu->options |= ARM_SMMU_OPT_MSIPOLL;
3524 if (reg & IDR0_HYP) {
3525 smmu->features |= ARM_SMMU_FEAT_HYP;
3526 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
3527 smmu->features |= ARM_SMMU_FEAT_E2H;
3531 * The coherency feature as set by FW is used in preference to the ID
3532 * register, but warn on mismatch.
3534 if (!!(reg & IDR0_COHACC) != coherent)
3535 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3536 coherent ? "true" : "false");
3538 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
3539 case IDR0_STALL_MODEL_FORCE:
3540 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3542 case IDR0_STALL_MODEL_STALL:
3543 smmu->features |= ARM_SMMU_FEAT_STALLS;
3547 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3550 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3552 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
3553 dev_err(smmu->dev, "no translation support!\n");
3557 /* We only support the AArch64 table format at present */
3558 switch (FIELD_GET(IDR0_TTF, reg)) {
3559 case IDR0_TTF_AARCH32_64:
3562 case IDR0_TTF_AARCH64:
3565 dev_err(smmu->dev, "AArch64 table format not supported!\n");
3569 /* ASID/VMID sizes */
3570 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3571 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3574 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3575 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
3576 dev_err(smmu->dev, "embedded implementation not supported\n");
3580 /* Queue sizes, capped to ensure natural alignment */
3581 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3582 FIELD_GET(IDR1_CMDQS, reg));
3583 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3585 * We don't support splitting up batches, so one batch of
3586 * commands plus an extra sync needs to fit inside the command
3587 * queue. There's also no way we can handle the weird alignment
3588 * restrictions on the base pointer for a unit-length queue.
3590 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3591 CMDQ_BATCH_ENTRIES);
3595 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3596 FIELD_GET(IDR1_EVTQS, reg));
3597 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3598 FIELD_GET(IDR1_PRIQS, reg));
3600 /* SID/SSID sizes */
3601 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3602 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3603 smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
3606 * If the SMMU supports fewer bits than would fill a single L2 stream
3607 * table, use a linear table instead.
3609 if (smmu->sid_bits <= STRTAB_SPLIT)
3610 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3613 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
3614 if (FIELD_GET(IDR3_RIL, reg))
3615 smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
3618 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3620 /* Maximum number of outstanding stalls */
3621 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3624 if (reg & IDR5_GRAN64K)
3625 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3626 if (reg & IDR5_GRAN16K)
3627 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3628 if (reg & IDR5_GRAN4K)
3629 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3631 /* Input address size */
3632 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
3633 smmu->features |= ARM_SMMU_FEAT_VAX;
3635 /* Output address size */
3636 switch (FIELD_GET(IDR5_OAS, reg)) {
3637 case IDR5_OAS_32_BIT:
3640 case IDR5_OAS_36_BIT:
3643 case IDR5_OAS_40_BIT:
3646 case IDR5_OAS_42_BIT:
3649 case IDR5_OAS_44_BIT:
3652 case IDR5_OAS_52_BIT:
3654 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
3658 "unknown output address size. Truncating to 48-bit\n");
3660 case IDR5_OAS_48_BIT:
3664 if (arm_smmu_ops.pgsize_bitmap == -1UL)
3665 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3667 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3669 /* Set the DMA mask for our table walker */
3670 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3672 "failed to set DMA mask for table walker\n");
3674 smmu->ias = max(smmu->ias, smmu->oas);
3676 if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
3677 (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
3678 smmu->features |= ARM_SMMU_FEAT_NESTING;
3680 arm_smmu_device_iidr_probe(smmu);
3682 if (arm_smmu_sva_supported(smmu))
3683 smmu->features |= ARM_SMMU_FEAT_SVA;
3685 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3686 smmu->ias, smmu->oas, smmu->features);
3691 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
3694 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
3695 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3697 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3698 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3702 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3705 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3706 struct arm_smmu_device *smmu)
3708 struct acpi_iort_smmu_v3 *iort_smmu;
3709 struct device *dev = smmu->dev;
3710 struct acpi_iort_node *node;
3712 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3714 /* Retrieve SMMUv3 specific data */
3715 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3717 acpi_smmu_get_options(iort_smmu->model, smmu);
3719 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3720 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3725 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3726 struct arm_smmu_device *smmu)
3732 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3733 struct arm_smmu_device *smmu)
3735 struct device *dev = &pdev->dev;
3739 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3740 dev_err(dev, "missing #iommu-cells property\n");
3741 else if (cells != 1)
3742 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3746 parse_driver_options(smmu);
3748 if (of_dma_is_coherent(dev->of_node))
3749 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3754 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3756 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3762 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
3763 resource_size_t size)
3765 struct resource res = DEFINE_RES_MEM(start, size);
3767 return devm_ioremap_resource(dev, &res);
3770 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
3772 struct list_head rmr_list;
3773 struct iommu_resv_region *e;
3775 INIT_LIST_HEAD(&rmr_list);
3776 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3778 list_for_each_entry(e, &rmr_list, list) {
3780 struct iommu_iort_rmr_data *rmr;
3783 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
3784 for (i = 0; i < rmr->num_sids; i++) {
3785 ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
3787 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
3792 step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]);
3793 arm_smmu_init_bypass_stes(step, 1, true);
3797 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3800 static int arm_smmu_device_probe(struct platform_device *pdev)
3803 struct resource *res;
3804 resource_size_t ioaddr;
3805 struct arm_smmu_device *smmu;
3806 struct device *dev = &pdev->dev;
3809 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3815 ret = arm_smmu_device_dt_probe(pdev, smmu);
3817 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3822 /* Set bypass mode according to firmware probing result */
3826 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3829 if (resource_size(res) < arm_smmu_resource_size(smmu)) {
3830 dev_err(dev, "MMIO region too small (%pr)\n", res);
3833 ioaddr = res->start;
3836 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
3837 * the PMCG registers which are reserved by the PMU driver.
3839 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
3840 if (IS_ERR(smmu->base))
3841 return PTR_ERR(smmu->base);
3843 if (arm_smmu_resource_size(smmu) > SZ_64K) {
3844 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
3846 if (IS_ERR(smmu->page1))
3847 return PTR_ERR(smmu->page1);
3849 smmu->page1 = smmu->base;
3852 /* Interrupt lines */
3854 irq = platform_get_irq_byname_optional(pdev, "combined");
3856 smmu->combined_irq = irq;
3858 irq = platform_get_irq_byname_optional(pdev, "eventq");
3860 smmu->evtq.q.irq = irq;
3862 irq = platform_get_irq_byname_optional(pdev, "priq");
3864 smmu->priq.q.irq = irq;
3866 irq = platform_get_irq_byname_optional(pdev, "gerror");
3868 smmu->gerr_irq = irq;
3871 ret = arm_smmu_device_hw_probe(smmu);
3875 /* Initialise in-memory data structures */
3876 ret = arm_smmu_init_structures(smmu);
3880 /* Record our private device structure */
3881 platform_set_drvdata(pdev, smmu);
3883 /* Check for RMRs and install bypass STEs if any */
3884 arm_smmu_rmr_install_bypass_ste(smmu);
3886 /* Reset the device */
3887 ret = arm_smmu_device_reset(smmu, bypass);
3891 /* And we're up. Go go go! */
3892 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3893 "smmu3.%pa", &ioaddr);
3897 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
3899 dev_err(dev, "Failed to register iommu\n");
3900 iommu_device_sysfs_remove(&smmu->iommu);
3907 static void arm_smmu_device_remove(struct platform_device *pdev)
3909 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3911 iommu_device_unregister(&smmu->iommu);
3912 iommu_device_sysfs_remove(&smmu->iommu);
3913 arm_smmu_device_disable(smmu);
3914 iopf_queue_free(smmu->evtq.iopf);
3915 ida_destroy(&smmu->vmid_map);
3918 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3920 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3922 arm_smmu_device_disable(smmu);
3925 static const struct of_device_id arm_smmu_of_match[] = {
3926 { .compatible = "arm,smmu-v3", },
3929 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
3931 static void arm_smmu_driver_unregister(struct platform_driver *drv)
3933 arm_smmu_sva_notifier_synchronize();
3934 platform_driver_unregister(drv);
3937 static struct platform_driver arm_smmu_driver = {
3939 .name = "arm-smmu-v3",
3940 .of_match_table = arm_smmu_of_match,
3941 .suppress_bind_attrs = true,
3943 .probe = arm_smmu_device_probe,
3944 .remove_new = arm_smmu_device_remove,
3945 .shutdown = arm_smmu_device_shutdown,
3947 module_driver(arm_smmu_driver, platform_driver_register,
3948 arm_smmu_driver_unregister);
3950 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
3951 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
3952 MODULE_ALIAS("platform:arm-smmu-v3");
3953 MODULE_LICENSE("GPL v2");