drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * IOMMU API for ARM architected SMMUv3 implementations.
   4  *
   5  * Copyright (C) 2015 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver is powered by bad coffee and bombay mix.
  10  */
  11
  12 #include <linux/acpi.h>
  13 #include <linux/acpi_iort.h>
  14 #include <linux/bitops.h>
  15 #include <linux/crash_dump.h>
  16 #include <linux/delay.h>
  17 #include <linux/err.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/io-pgtable.h>
  20 #include <linux/iopoll.h>
  21 #include <linux/module.h>
  22 #include <linux/msi.h>
  23 #include <linux/of.h>
  24 #include <linux/of_address.h>
  25 #include <linux/of_platform.h>
  26 #include <linux/pci.h>
  27 #include <linux/pci-ats.h>
  28 #include <linux/platform_device.h>
  29
  30 #include "arm-smmu-v3.h"
  31 #include "../../dma-iommu.h"
  32 #include "../../iommu-sva.h"
  33
  34 static bool disable_bypass = true;
  35 module_param(disable_bypass, bool, 0444);
  36 MODULE_PARM_DESC(disable_bypass,
  37         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  38
  39 static bool disable_msipolling;
  40 module_param(disable_msipolling, bool, 0444);
  41 MODULE_PARM_DESC(disable_msipolling,
  42         "Disable MSI-based polling for CMD_SYNC completion.");
  43
  44 enum arm_smmu_msi_index {
  45         EVTQ_MSI_INDEX,
  46         GERROR_MSI_INDEX,
  47         PRIQ_MSI_INDEX,
  48         ARM_SMMU_MAX_MSIS,
  49 };
  50
  51 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
  52         [EVTQ_MSI_INDEX] = {
  53                 ARM_SMMU_EVTQ_IRQ_CFG0,
  54                 ARM_SMMU_EVTQ_IRQ_CFG1,
  55                 ARM_SMMU_EVTQ_IRQ_CFG2,
  56         },
  57         [GERROR_MSI_INDEX] = {
  58                 ARM_SMMU_GERROR_IRQ_CFG0,
  59                 ARM_SMMU_GERROR_IRQ_CFG1,
  60                 ARM_SMMU_GERROR_IRQ_CFG2,
  61         },
  62         [PRIQ_MSI_INDEX] = {
  63                 ARM_SMMU_PRIQ_IRQ_CFG0,
  64                 ARM_SMMU_PRIQ_IRQ_CFG1,
  65                 ARM_SMMU_PRIQ_IRQ_CFG2,
  66         },
  67 };
  68
  69 struct arm_smmu_option_prop {
  70         u32 opt;
  71         const char *prop;
  72 };
  73
  74 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
  75 DEFINE_MUTEX(arm_smmu_asid_lock);
  76
  77 /*
  78  * Special value used by SVA when a process dies, to quiesce a CD without
  79  * disabling it.
  80  */
  81 struct arm_smmu_ctx_desc quiet_cd = { 0 };
  82
  83 static struct arm_smmu_option_prop arm_smmu_options[] = {
  84         { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
  85         { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
  86         { 0, NULL},
  87 };
  88
  89 static void parse_driver_options(struct arm_smmu_device *smmu)
  90 {
  91         int i = 0;
  92
  93         do {
  94                 if (of_property_read_bool(smmu->dev->of_node,
  95                                                 arm_smmu_options[i].prop)) {
  96                         smmu->options |= arm_smmu_options[i].opt;
  97                         dev_notice(smmu->dev, "option %s\n",
  98                                 arm_smmu_options[i].prop);
  99                 }
 100         } while (arm_smmu_options[++i].opt);
 101 }
 102
 103 /* Low-level queue manipulation functions */
 104 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
 105 {
 106         u32 space, prod, cons;
 107
 108         prod = Q_IDX(q, q->prod);
 109         cons = Q_IDX(q, q->cons);
 110
 111         if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
 112                 space = (1 << q->max_n_shift) - (prod - cons);
 113         else
 114                 space = cons - prod;
 115
 116         return space >= n;
 117 }
 118
 119 static bool queue_full(struct arm_smmu_ll_queue *q)
 120 {
 121         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
 122                Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
 123 }
 124
 125 static bool queue_empty(struct arm_smmu_ll_queue *q)
 126 {
 127         return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
 128                Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
 129 }
 130
 131 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
 132 {
 133         return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
 134                 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
 135                ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
 136                 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
 137 }
 138
 139 static void queue_sync_cons_out(struct arm_smmu_queue *q)
 140 {
 141         /*
 142          * Ensure that all CPU accesses (reads and writes) to the queue
 143          * are complete before we update the cons pointer.
 144          */
 145         __iomb();
 146         writel_relaxed(q->llq.cons, q->cons_reg);
 147 }
 148
 149 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
 150 {
 151         u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
 152         q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
 153 }
 154
 155 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
 156 {
 157         struct arm_smmu_ll_queue *llq = &q->llq;
 158
 159         if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
 160                 return;
 161
 162         llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
 163                       Q_IDX(llq, llq->cons);
 164         queue_sync_cons_out(q);
 165 }
 166
 167 static int queue_sync_prod_in(struct arm_smmu_queue *q)
 168 {
 169         u32 prod;
 170         int ret = 0;
 171
 172         /*
 173          * We can't use the _relaxed() variant here, as we must prevent
 174          * speculative reads of the queue before we have determined that
 175          * prod has indeed moved.
 176          */
 177         prod = readl(q->prod_reg);
 178
 179         if (Q_OVF(prod) != Q_OVF(q->llq.prod))
 180                 ret = -EOVERFLOW;
 181
 182         q->llq.prod = prod;
 183         return ret;
 184 }
 185
 186 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
 187 {
 188         u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
 189         return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
 190 }
 191
 192 static void queue_poll_init(struct arm_smmu_device *smmu,
 193                             struct arm_smmu_queue_poll *qp)
 194 {
 195         qp->delay = 1;
 196         qp->spin_cnt = 0;
 197         qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
 198         qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
 199 }
 200
 201 static int queue_poll(struct arm_smmu_queue_poll *qp)
 202 {
 203         if (ktime_compare(ktime_get(), qp->timeout) > 0)
 204                 return -ETIMEDOUT;
 205
 206         if (qp->wfe) {
 207                 wfe();
 208         } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
 209                 cpu_relax();
 210         } else {
 211                 udelay(qp->delay);
 212                 qp->delay *= 2;
 213                 qp->spin_cnt = 0;
 214         }
 215
 216         return 0;
 217 }
 218
 219 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
 220 {
 221         int i;
 222
 223         for (i = 0; i < n_dwords; ++i)
 224                 *dst++ = cpu_to_le64(*src++);
 225 }
 226
 227 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
 228 {
 229         int i;
 230
 231         for (i = 0; i < n_dwords; ++i)
 232                 *dst++ = le64_to_cpu(*src++);
 233 }
 234
 235 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
 236 {
 237         if (queue_empty(&q->llq))
 238                 return -EAGAIN;
 239
 240         queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
 241         queue_inc_cons(&q->llq);
 242         queue_sync_cons_out(q);
 243         return 0;
 244 }
 245
 246 /* High-level queue accessors */
 247 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
 248 {
 249         memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
 250         cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
 251
 252         switch (ent->opcode) {
 253         case CMDQ_OP_TLBI_EL2_ALL:
 254         case CMDQ_OP_TLBI_NSNH_ALL:
 255                 break;
 256         case CMDQ_OP_PREFETCH_CFG:
 257                 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
 258                 break;
 259         case CMDQ_OP_CFGI_CD:
 260                 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
 261                 fallthrough;
 262         case CMDQ_OP_CFGI_STE:
 263                 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
 264                 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
 265                 break;
 266         case CMDQ_OP_CFGI_CD_ALL:
 267                 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
 268                 break;
 269         case CMDQ_OP_CFGI_ALL:
 270                 /* Cover the entire SID range */
 271                 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
 272                 break;
 273         case CMDQ_OP_TLBI_NH_VA:
 274                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 275                 fallthrough;
 276         case CMDQ_OP_TLBI_EL2_VA:
 277                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
 278                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
 279                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
 280                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
 281                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
 282                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
 283                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
 284                 break;
 285         case CMDQ_OP_TLBI_S2_IPA:
 286                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
 287                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
 288                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 289                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
 290                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
 291                 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
 292                 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
 293                 break;
 294         case CMDQ_OP_TLBI_NH_ASID:
 295                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
 296                 fallthrough;
 297         case CMDQ_OP_TLBI_S12_VMALL:
 298                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 299                 break;
 300         case CMDQ_OP_TLBI_EL2_ASID:
 301                 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
 302                 break;
 303         case CMDQ_OP_ATC_INV:
 304                 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
 305                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
 306                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
 307                 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
 308                 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
 309                 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
 310                 break;
 311         case CMDQ_OP_PRI_RESP:
 312                 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
 313                 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
 314                 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
 315                 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
 316                 switch (ent->pri.resp) {
 317                 case PRI_RESP_DENY:
 318                 case PRI_RESP_FAIL:
 319                 case PRI_RESP_SUCC:
 320                         break;
 321                 default:
 322                         return -EINVAL;
 323                 }
 324                 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
 325                 break;
 326         case CMDQ_OP_RESUME:
 327                 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
 328                 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
 329                 cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
 330                 break;
 331         case CMDQ_OP_CMD_SYNC:
 332                 if (ent->sync.msiaddr) {
 333                         cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
 334                         cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
 335                 } else {
 336                         cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
 337                 }
 338                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
 339                 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
 340                 break;
 341         default:
 342                 return -ENOENT;
 343         }
 344
 345         return 0;
 346 }
 347
 348 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)
 349 {
 350         return &smmu->cmdq;
 351 }
 352
 353 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
 354                                          struct arm_smmu_queue *q, u32 prod)
 355 {
 356         struct arm_smmu_cmdq_ent ent = {
 357                 .opcode = CMDQ_OP_CMD_SYNC,
 358         };
 359
 360         /*
 361          * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
 362          * payload, so the write will zero the entire command on that platform.
 363          */
 364         if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
 365                 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
 366                                    q->ent_dwords * 8;
 367         }
 368
 369         arm_smmu_cmdq_build_cmd(cmd, &ent);
 370 }
 371
 372 static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
 373                                      struct arm_smmu_queue *q)
 374 {
 375         static const char * const cerror_str[] = {
 376                 [CMDQ_ERR_CERROR_NONE_IDX]      = "No error",
 377                 [CMDQ_ERR_CERROR_ILL_IDX]       = "Illegal command",
 378                 [CMDQ_ERR_CERROR_ABT_IDX]       = "Abort on command fetch",
 379                 [CMDQ_ERR_CERROR_ATC_INV_IDX]   = "ATC invalidate timeout",
 380         };
 381
 382         int i;
 383         u64 cmd[CMDQ_ENT_DWORDS];
 384         u32 cons = readl_relaxed(q->cons_reg);
 385         u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
 386         struct arm_smmu_cmdq_ent cmd_sync = {
 387                 .opcode = CMDQ_OP_CMD_SYNC,
 388         };
 389
 390         dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
 391                 idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
 392
 393         switch (idx) {
 394         case CMDQ_ERR_CERROR_ABT_IDX:
 395                 dev_err(smmu->dev, "retrying command fetch\n");
 396                 return;
 397         case CMDQ_ERR_CERROR_NONE_IDX:
 398                 return;
 399         case CMDQ_ERR_CERROR_ATC_INV_IDX:
 400                 /*
 401                  * ATC Invalidation Completion timeout. CONS is still pointing
 402                  * at the CMD_SYNC. Attempt to complete other pending commands
 403                  * by repeating the CMD_SYNC, though we might well end up back
 404                  * here since the ATC invalidation may still be pending.
 405                  */
 406                 return;
 407         case CMDQ_ERR_CERROR_ILL_IDX:
 408         default:
 409                 break;
 410         }
 411
 412         /*
 413          * We may have concurrent producers, so we need to be careful
 414          * not to touch any of the shadow cmdq state.
 415          */
 416         queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
 417         dev_err(smmu->dev, "skipping command in error state:\n");
 418         for (i = 0; i < ARRAY_SIZE(cmd); ++i)
 419                 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
 420
 421         /* Convert the erroneous command into a CMD_SYNC */
 422         arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
 423
 424         queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
 425 }
 426
 427 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
 428 {
 429         __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q);
 430 }
 431
 432 /*
 433  * Command queue locking.
 434  * This is a form of bastardised rwlock with the following major changes:
 435  *
 436  * - The only LOCK routines are exclusive_trylock() and shared_lock().
 437  *   Neither have barrier semantics, and instead provide only a control
 438  *   dependency.
 439  *
 440  * - The UNLOCK routines are supplemented with shared_tryunlock(), which
 441  *   fails if the caller appears to be the last lock holder (yes, this is
 442  *   racy). All successful UNLOCK routines have RELEASE semantics.
 443  */
 444 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
 445 {
 446         int val;
 447
 448         /*
 449          * We can try to avoid the cmpxchg() loop by simply incrementing the
 450          * lock counter. When held in exclusive state, the lock counter is set
 451          * to INT_MIN so these increments won't hurt as the value will remain
 452          * negative.
 453          */
 454         if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
 455                 return;
 456
 457         do {
 458                 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
 459         } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
 460 }
 461
 462 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
 463 {
 464         (void)atomic_dec_return_release(&cmdq->lock);
 465 }
 466
 467 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
 468 {
 469         if (atomic_read(&cmdq->lock) == 1)
 470                 return false;
 471
 472         arm_smmu_cmdq_shared_unlock(cmdq);
 473         return true;
 474 }
 475
 476 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)            \
 477 ({                                                                      \
 478         bool __ret;                                                     \
 479         local_irq_save(flags);                                          \
 480         __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN);       \
 481         if (!__ret)                                                     \
 482                 local_irq_restore(flags);                               \
 483         __ret;                                                          \
 484 })
 485
 486 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)          \
 487 ({                                                                      \
 488         atomic_set_release(&cmdq->lock, 0);                             \
 489         local_irq_restore(flags);                                       \
 490 })
 491
 492
 493 /*
 494  * Command queue insertion.
 495  * This is made fiddly by our attempts to achieve some sort of scalability
 496  * since there is one queue shared amongst all of the CPUs in the system.  If
 497  * you like mixed-size concurrency, dependency ordering and relaxed atomics,
 498  * then you'll *love* this monstrosity.
 499  *
 500  * The basic idea is to split the queue up into ranges of commands that are
 501  * owned by a given CPU; the owner may not have written all of the commands
 502  * itself, but is responsible for advancing the hardware prod pointer when
 503  * the time comes. The algorithm is roughly:
 504  *
 505  *      1. Allocate some space in the queue. At this point we also discover
 506  *         whether the head of the queue is currently owned by another CPU,
 507  *         or whether we are the owner.
 508  *
 509  *      2. Write our commands into our allocated slots in the queue.
 510  *
 511  *      3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
 512  *
 513  *      4. If we are an owner:
 514  *              a. Wait for the previous owner to finish.
 515  *              b. Mark the queue head as unowned, which tells us the range
 516  *                 that we are responsible for publishing.
 517  *              c. Wait for all commands in our owned range to become valid.
 518  *              d. Advance the hardware prod pointer.
 519  *              e. Tell the next owner we've finished.
 520  *
 521  *      5. If we are inserting a CMD_SYNC (we may or may not have been an
 522  *         owner), then we need to stick around until it has completed:
 523  *              a. If we have MSIs, the SMMU can write back into the CMD_SYNC
 524  *                 to clear the first 4 bytes.
 525  *              b. Otherwise, we spin waiting for the hardware cons pointer to
 526  *                 advance past our command.
 527  *
 528  * The devil is in the details, particularly the use of locking for handling
 529  * SYNC completion and freeing up space in the queue before we think that it is
 530  * full.
 531  */
 532 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
 533                                                u32 sprod, u32 eprod, bool set)
 534 {
 535         u32 swidx, sbidx, ewidx, ebidx;
 536         struct arm_smmu_ll_queue llq = {
 537                 .max_n_shift    = cmdq->q.llq.max_n_shift,
 538                 .prod           = sprod,
 539         };
 540
 541         ewidx = BIT_WORD(Q_IDX(&llq, eprod));
 542         ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
 543
 544         while (llq.prod != eprod) {
 545                 unsigned long mask;
 546                 atomic_long_t *ptr;
 547                 u32 limit = BITS_PER_LONG;
 548
 549                 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
 550                 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
 551
 552                 ptr = &cmdq->valid_map[swidx];
 553
 554                 if ((swidx == ewidx) && (sbidx < ebidx))
 555                         limit = ebidx;
 556
 557                 mask = GENMASK(limit - 1, sbidx);
 558
 559                 /*
 560                  * The valid bit is the inverse of the wrap bit. This means
 561                  * that a zero-initialised queue is invalid and, after marking
 562                  * all entries as valid, they become invalid again when we
 563                  * wrap.
 564                  */
 565                 if (set) {
 566                         atomic_long_xor(mask, ptr);
 567                 } else { /* Poll */
 568                         unsigned long valid;
 569
 570                         valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
 571                         atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
 572                 }
 573
 574                 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
 575         }
 576 }
 577
 578 /* Mark all entries in the range [sprod, eprod) as valid */
 579 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
 580                                         u32 sprod, u32 eprod)
 581 {
 582         __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
 583 }
 584
 585 /* Wait for all entries in the range [sprod, eprod) to become valid */
 586 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
 587                                          u32 sprod, u32 eprod)
 588 {
 589         __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
 590 }
 591
 592 /* Wait for the command queue to become non-full */
 593 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
 594                                              struct arm_smmu_ll_queue *llq)
 595 {
 596         unsigned long flags;
 597         struct arm_smmu_queue_poll qp;
 598         struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
 599         int ret = 0;
 600
 601         /*
 602          * Try to update our copy of cons by grabbing exclusive cmdq access. If
 603          * that fails, spin until somebody else updates it for us.
 604          */
 605         if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
 606                 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
 607                 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
 608                 llq->val = READ_ONCE(cmdq->q.llq.val);
 609                 return 0;
 610         }
 611
 612         queue_poll_init(smmu, &qp);
 613         do {
 614                 llq->val = READ_ONCE(cmdq->q.llq.val);
 615                 if (!queue_full(llq))
 616                         break;
 617
 618                 ret = queue_poll(&qp);
 619         } while (!ret);
 620
 621         return ret;
 622 }
 623
 624 /*
 625  * Wait until the SMMU signals a CMD_SYNC completion MSI.
 626  * Must be called with the cmdq lock held in some capacity.
 627  */
 628 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
 629                                           struct arm_smmu_ll_queue *llq)
 630 {
 631         int ret = 0;
 632         struct arm_smmu_queue_poll qp;
 633         struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
 634         u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
 635
 636         queue_poll_init(smmu, &qp);
 637
 638         /*
 639          * The MSI won't generate an event, since it's being written back
 640          * into the command queue.
 641          */
 642         qp.wfe = false;
 643         smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
 644         llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
 645         return ret;
 646 }
 647
 648 /*
 649  * Wait until the SMMU cons index passes llq->prod.
 650  * Must be called with the cmdq lock held in some capacity.
 651  */
 652 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
 653                                                struct arm_smmu_ll_queue *llq)
 654 {
 655         struct arm_smmu_queue_poll qp;
 656         struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
 657         u32 prod = llq->prod;
 658         int ret = 0;
 659
 660         queue_poll_init(smmu, &qp);
 661         llq->val = READ_ONCE(cmdq->q.llq.val);
 662         do {
 663                 if (queue_consumed(llq, prod))
 664                         break;
 665
 666                 ret = queue_poll(&qp);
 667
 668                 /*
 669                  * This needs to be a readl() so that our subsequent call
 670                  * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
 671                  *
 672                  * Specifically, we need to ensure that we observe all
 673                  * shared_lock()s by other CMD_SYNCs that share our owner,
 674                  * so that a failing call to tryunlock() means that we're
 675                  * the last one out and therefore we can safely advance
 676                  * cmdq->q.llq.cons. Roughly speaking:
 677                  *
 678                  * CPU 0                CPU1                    CPU2 (us)
 679                  *
 680                  * if (sync)
 681                  *      shared_lock();
 682                  *
 683                  * dma_wmb();
 684                  * set_valid_map();
 685                  *
 686                  *                      if (owner) {
 687                  *                              poll_valid_map();
 688                  *                              <control dependency>
 689                  *                              writel(prod_reg);
 690                  *
 691                  *                                              readl(cons_reg);
 692                  *                                              tryunlock();
 693                  *
 694                  * Requires us to see CPU 0's shared_lock() acquisition.
 695                  */
 696                 llq->cons = readl(cmdq->q.cons_reg);
 697         } while (!ret);
 698
 699         return ret;
 700 }
 701
 702 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
 703                                          struct arm_smmu_ll_queue *llq)
 704 {
 705         if (smmu->options & ARM_SMMU_OPT_MSIPOLL)
 706                 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
 707
 708         return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
 709 }
 710
 711 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
 712                                         u32 prod, int n)
 713 {
 714         int i;
 715         struct arm_smmu_ll_queue llq = {
 716                 .max_n_shift    = cmdq->q.llq.max_n_shift,
 717                 .prod           = prod,
 718         };
 719
 720         for (i = 0; i < n; ++i) {
 721                 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
 722
 723                 prod = queue_inc_prod_n(&llq, i);
 724                 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
 725         }
 726 }
 727
 728 /*
 729  * This is the actual insertion function, and provides the following
 730  * ordering guarantees to callers:
 731  *
 732  * - There is a dma_wmb() before publishing any commands to the queue.
 733  *   This can be relied upon to order prior writes to data structures
 734  *   in memory (such as a CD or an STE) before the command.
 735  *
 736  * - On completion of a CMD_SYNC, there is a control dependency.
 737  *   This can be relied upon to order subsequent writes to memory (e.g.
 738  *   freeing an IOVA) after completion of the CMD_SYNC.
 739  *
 740  * - Command insertion is totally ordered, so if two CPUs each race to
 741  *   insert their own list of commands then all of the commands from one
 742  *   CPU will appear before any of the commands from the other CPU.
 743  */
 744 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
 745                                        u64 *cmds, int n, bool sync)
 746 {
 747         u64 cmd_sync[CMDQ_ENT_DWORDS];
 748         u32 prod;
 749         unsigned long flags;
 750         bool owner;
 751         struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
 752         struct arm_smmu_ll_queue llq, head;
 753         int ret = 0;
 754
 755         llq.max_n_shift = cmdq->q.llq.max_n_shift;
 756
 757         /* 1. Allocate some space in the queue */
 758         local_irq_save(flags);
 759         llq.val = READ_ONCE(cmdq->q.llq.val);
 760         do {
 761                 u64 old;
 762
 763                 while (!queue_has_space(&llq, n + sync)) {
 764                         local_irq_restore(flags);
 765                         if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
 766                                 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
 767                         local_irq_save(flags);
 768                 }
 769
 770                 head.cons = llq.cons;
 771                 head.prod = queue_inc_prod_n(&llq, n + sync) |
 772                                              CMDQ_PROD_OWNED_FLAG;
 773
 774                 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
 775                 if (old == llq.val)
 776                         break;
 777
 778                 llq.val = old;
 779         } while (1);
 780         owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
 781         head.prod &= ~CMDQ_PROD_OWNED_FLAG;
 782         llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
 783
 784         /*
 785          * 2. Write our commands into the queue
 786          * Dependency ordering from the cmpxchg() loop above.
 787          */
 788         arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
 789         if (sync) {
 790                 prod = queue_inc_prod_n(&llq, n);
 791                 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);
 792                 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
 793
 794                 /*
 795                  * In order to determine completion of our CMD_SYNC, we must
 796                  * ensure that the queue can't wrap twice without us noticing.
 797                  * We achieve that by taking the cmdq lock as shared before
 798                  * marking our slot as valid.
 799                  */
 800                 arm_smmu_cmdq_shared_lock(cmdq);
 801         }
 802
 803         /* 3. Mark our slots as valid, ensuring commands are visible first */
 804         dma_wmb();
 805         arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
 806
 807         /* 4. If we are the owner, take control of the SMMU hardware */
 808         if (owner) {
 809                 /* a. Wait for previous owner to finish */
 810                 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
 811
 812                 /* b. Stop gathering work by clearing the owned flag */
 813                 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
 814                                                    &cmdq->q.llq.atomic.prod);
 815                 prod &= ~CMDQ_PROD_OWNED_FLAG;
 816
 817                 /*
 818                  * c. Wait for any gathered work to be written to the queue.
 819                  * Note that we read our own entries so that we have the control
 820                  * dependency required by (d).
 821                  */
 822                 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
 823
 824                 /*
 825                  * d. Advance the hardware prod pointer
 826                  * Control dependency ordering from the entries becoming valid.
 827                  */
 828                 writel_relaxed(prod, cmdq->q.prod_reg);
 829
 830                 /*
 831                  * e. Tell the next owner we're done
 832                  * Make sure we've updated the hardware first, so that we don't
 833                  * race to update prod and potentially move it backwards.
 834                  */
 835                 atomic_set_release(&cmdq->owner_prod, prod);
 836         }
 837
 838         /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
 839         if (sync) {
 840                 llq.prod = queue_inc_prod_n(&llq, n);
 841                 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
 842                 if (ret) {
 843                         dev_err_ratelimited(smmu->dev,
 844                                             "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
 845                                             llq.prod,
 846                                             readl_relaxed(cmdq->q.prod_reg),
 847                                             readl_relaxed(cmdq->q.cons_reg));
 848                 }
 849
 850                 /*
 851                  * Try to unlock the cmdq lock. This will fail if we're the last
 852                  * reader, in which case we can safely update cmdq->q.llq.cons
 853                  */
 854                 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
 855                         WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
 856                         arm_smmu_cmdq_shared_unlock(cmdq);
 857                 }
 858         }
 859
 860         local_irq_restore(flags);
 861         return ret;
 862 }
 863
 864 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
 865                                      struct arm_smmu_cmdq_ent *ent,
 866                                      bool sync)
 867 {
 868         u64 cmd[CMDQ_ENT_DWORDS];
 869
 870         if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
 871                 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
 872                          ent->opcode);
 873                 return -EINVAL;
 874         }
 875
 876         return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync);
 877 }
 878
 879 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
 880                                    struct arm_smmu_cmdq_ent *ent)
 881 {
 882         return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
 883 }
 884
 885 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
 886                                              struct arm_smmu_cmdq_ent *ent)
 887 {
 888         return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
 889 }
 890
 891 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
 892                                     struct arm_smmu_cmdq_batch *cmds,
 893                                     struct arm_smmu_cmdq_ent *cmd)
 894 {
 895         int index;
 896
 897         if (cmds->num == CMDQ_BATCH_ENTRIES - 1 &&
 898             (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC)) {
 899                 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
 900                 cmds->num = 0;
 901         }
 902
 903         if (cmds->num == CMDQ_BATCH_ENTRIES) {
 904                 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);
 905                 cmds->num = 0;
 906         }
 907
 908         index = cmds->num * CMDQ_ENT_DWORDS;
 909         if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
 910                 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
 911                          cmd->opcode);
 912                 return;
 913         }
 914
 915         cmds->num++;
 916 }
 917
 918 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
 919                                       struct arm_smmu_cmdq_batch *cmds)
 920 {
 921         return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
 922 }
 923
 924 static int arm_smmu_page_response(struct device *dev,
 925                                   struct iommu_fault_event *unused,
 926                                   struct iommu_page_response *resp)
 927 {
 928         struct arm_smmu_cmdq_ent cmd = {0};
 929         struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 930         int sid = master->streams[0].id;
 931
 932         if (master->stall_enabled) {
 933                 cmd.opcode              = CMDQ_OP_RESUME;
 934                 cmd.resume.sid          = sid;
 935                 cmd.resume.stag         = resp->grpid;
 936                 switch (resp->code) {
 937                 case IOMMU_PAGE_RESP_INVALID:
 938                 case IOMMU_PAGE_RESP_FAILURE:
 939                         cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
 940                         break;
 941                 case IOMMU_PAGE_RESP_SUCCESS:
 942                         cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
 943                         break;
 944                 default:
 945                         return -EINVAL;
 946                 }
 947         } else {
 948                 return -ENODEV;
 949         }
 950
 951         arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
 952         /*
 953          * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
 954          * RESUME consumption guarantees that the stalled transaction will be
 955          * terminated... at some point in the future. PRI_RESP is fire and
 956          * forget.
 957          */
 958
 959         return 0;
 960 }
 961
 962 /* Context descriptor manipulation functions */
 963 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
 964 {
 965         struct arm_smmu_cmdq_ent cmd = {
 966                 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ?
 967                         CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
 968                 .tlbi.asid = asid,
 969         };
 970
 971         arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 972 }
 973
 974 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
 975                              int ssid, bool leaf)
 976 {
 977         size_t i;
 978         unsigned long flags;
 979         struct arm_smmu_master *master;
 980         struct arm_smmu_cmdq_batch cmds;
 981         struct arm_smmu_device *smmu = smmu_domain->smmu;
 982         struct arm_smmu_cmdq_ent cmd = {
 983                 .opcode = CMDQ_OP_CFGI_CD,
 984                 .cfgi   = {
 985                         .ssid   = ssid,
 986                         .leaf   = leaf,
 987                 },
 988         };
 989
 990         cmds.num = 0;
 991
 992         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
 993         list_for_each_entry(master, &smmu_domain->devices, domain_head) {
 994                 for (i = 0; i < master->num_streams; i++) {
 995                         cmd.cfgi.sid = master->streams[i].id;
 996                         arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
 997                 }
 998         }
 999         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1000
1001         arm_smmu_cmdq_batch_submit(smmu, &cmds);
1002 }
1003
1004 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
1005                                         struct arm_smmu_l1_ctx_desc *l1_desc)
1006 {
1007         size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1008
1009         l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
1010                                              &l1_desc->l2ptr_dma, GFP_KERNEL);
1011         if (!l1_desc->l2ptr) {
1012                 dev_warn(smmu->dev,
1013                          "failed to allocate context descriptor table\n");
1014                 return -ENOMEM;
1015         }
1016         return 0;
1017 }
1018
1019 static void arm_smmu_write_cd_l1_desc(__le64 *dst,
1020                                       struct arm_smmu_l1_ctx_desc *l1_desc)
1021 {
1022         u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |
1023                   CTXDESC_L1_DESC_V;
1024
1025         /* See comment in arm_smmu_write_ctx_desc() */
1026         WRITE_ONCE(*dst, cpu_to_le64(val));
1027 }
1028
1029 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,
1030                                    u32 ssid)
1031 {
1032         __le64 *l1ptr;
1033         unsigned int idx;
1034         struct arm_smmu_l1_ctx_desc *l1_desc;
1035         struct arm_smmu_device *smmu = smmu_domain->smmu;
1036         struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1037
1038         if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1039                 return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
1040
1041         idx = ssid >> CTXDESC_SPLIT;
1042         l1_desc = &cdcfg->l1_desc[idx];
1043         if (!l1_desc->l2ptr) {
1044                 if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
1045                         return NULL;
1046
1047                 l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
1048                 arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
1049                 /* An invalid L1CD can be cached */
1050                 arm_smmu_sync_cd(smmu_domain, ssid, false);
1051         }
1052         idx = ssid & (CTXDESC_L2_ENTRIES - 1);
1053         return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
1054 }
1055
1056 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
1057                             struct arm_smmu_ctx_desc *cd)
1058 {
1059         /*
1060          * This function handles the following cases:
1061          *
1062          * (1) Install primary CD, for normal DMA traffic (SSID = IOMMU_NO_PASID = 0).
1063          * (2) Install a secondary CD, for SID+SSID traffic.
1064          * (3) Update ASID of a CD. Atomically write the first 64 bits of the
1065          *     CD, then invalidate the old entry and mappings.
1066          * (4) Quiesce the context without clearing the valid bit. Disable
1067          *     translation, and ignore any translation fault.
1068          * (5) Remove a secondary CD.
1069          */
1070         u64 val;
1071         bool cd_live;
1072         __le64 *cdptr;
1073
1074         if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
1075                 return -E2BIG;
1076
1077         cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
1078         if (!cdptr)
1079                 return -ENOMEM;
1080
1081         val = le64_to_cpu(cdptr[0]);
1082         cd_live = !!(val & CTXDESC_CD_0_V);
1083
1084         if (!cd) { /* (5) */
1085                 val = 0;
1086         } else if (cd == &quiet_cd) { /* (4) */
1087                 val |= CTXDESC_CD_0_TCR_EPD0;
1088         } else if (cd_live) { /* (3) */
1089                 val &= ~CTXDESC_CD_0_ASID;
1090                 val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
1091                 /*
1092                  * Until CD+TLB invalidation, both ASIDs may be used for tagging
1093                  * this substream's traffic
1094                  */
1095         } else { /* (1) and (2) */
1096                 cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
1097                 cdptr[2] = 0;
1098                 cdptr[3] = cpu_to_le64(cd->mair);
1099
1100                 /*
1101                  * STE is live, and the SMMU might read dwords of this CD in any
1102                  * order. Ensure that it observes valid values before reading
1103                  * V=1.
1104                  */
1105                 arm_smmu_sync_cd(smmu_domain, ssid, true);
1106
1107                 val = cd->tcr |
1108 #ifdef __BIG_ENDIAN
1109                         CTXDESC_CD_0_ENDI |
1110 #endif
1111                         CTXDESC_CD_0_R | CTXDESC_CD_0_A |
1112                         (cd->mm ? 0 : CTXDESC_CD_0_ASET) |
1113                         CTXDESC_CD_0_AA64 |
1114                         FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
1115                         CTXDESC_CD_0_V;
1116
1117                 if (smmu_domain->stall_enabled)
1118                         val |= CTXDESC_CD_0_S;
1119         }
1120
1121         /*
1122          * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
1123          * "Configuration structures and configuration invalidation completion"
1124          *
1125          *   The size of single-copy atomic reads made by the SMMU is
1126          *   IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
1127          *   field within an aligned 64-bit span of a structure can be altered
1128          *   without first making the structure invalid.
1129          */
1130         WRITE_ONCE(cdptr[0], cpu_to_le64(val));
1131         arm_smmu_sync_cd(smmu_domain, ssid, true);
1132         return 0;
1133 }
1134
1135 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
1136 {
1137         int ret;
1138         size_t l1size;
1139         size_t max_contexts;
1140         struct arm_smmu_device *smmu = smmu_domain->smmu;
1141         struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1142         struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
1143
1144         max_contexts = 1 << cfg->s1cdmax;
1145
1146         if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1147             max_contexts <= CTXDESC_L2_ENTRIES) {
1148                 cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1149                 cdcfg->num_l1_ents = max_contexts;
1150
1151                 l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
1152         } else {
1153                 cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1154                 cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
1155                                                   CTXDESC_L2_ENTRIES);
1156
1157                 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
1158                                               sizeof(*cdcfg->l1_desc),
1159                                               GFP_KERNEL);
1160                 if (!cdcfg->l1_desc)
1161                         return -ENOMEM;
1162
1163                 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1164         }
1165
1166         cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
1167                                            GFP_KERNEL);
1168         if (!cdcfg->cdtab) {
1169                 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1170                 ret = -ENOMEM;
1171                 goto err_free_l1;
1172         }
1173
1174         return 0;
1175
1176 err_free_l1:
1177         if (cdcfg->l1_desc) {
1178                 devm_kfree(smmu->dev, cdcfg->l1_desc);
1179                 cdcfg->l1_desc = NULL;
1180         }
1181         return ret;
1182 }
1183
1184 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
1185 {
1186         int i;
1187         size_t size, l1size;
1188         struct arm_smmu_device *smmu = smmu_domain->smmu;
1189         struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1190
1191         if (cdcfg->l1_desc) {
1192                 size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1193
1194                 for (i = 0; i < cdcfg->num_l1_ents; i++) {
1195                         if (!cdcfg->l1_desc[i].l2ptr)
1196                                 continue;
1197
1198                         dmam_free_coherent(smmu->dev, size,
1199                                            cdcfg->l1_desc[i].l2ptr,
1200                                            cdcfg->l1_desc[i].l2ptr_dma);
1201                 }
1202                 devm_kfree(smmu->dev, cdcfg->l1_desc);
1203                 cdcfg->l1_desc = NULL;
1204
1205                 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1206         } else {
1207                 l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
1208         }
1209
1210         dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
1211         cdcfg->cdtab_dma = 0;
1212         cdcfg->cdtab = NULL;
1213 }
1214
1215 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd)
1216 {
1217         bool free;
1218         struct arm_smmu_ctx_desc *old_cd;
1219
1220         if (!cd->asid)
1221                 return false;
1222
1223         free = refcount_dec_and_test(&cd->refs);
1224         if (free) {
1225                 old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid);
1226                 WARN_ON(old_cd != cd);
1227         }
1228         return free;
1229 }
1230
1231 /* Stream table manipulation functions */
1232 static void
1233 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1234 {
1235         u64 val = 0;
1236
1237         val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1238         val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1239
1240         /* See comment in arm_smmu_write_ctx_desc() */
1241         WRITE_ONCE(*dst, cpu_to_le64(val));
1242 }
1243
1244 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1245 {
1246         struct arm_smmu_cmdq_ent cmd = {
1247                 .opcode = CMDQ_OP_CFGI_STE,
1248                 .cfgi   = {
1249                         .sid    = sid,
1250                         .leaf   = true,
1251                 },
1252         };
1253
1254         arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1255 }
1256
1257 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1258                                       __le64 *dst)
1259 {
1260         /*
1261          * This is hideously complicated, but we only really care about
1262          * three cases at the moment:
1263          *
1264          * 1. Invalid (all zero) -> bypass/fault (init)
1265          * 2. Bypass/fault -> translation/bypass (attach)
1266          * 3. Translation/bypass -> bypass/fault (detach)
1267          *
1268          * Given that we can't update the STE atomically and the SMMU
1269          * doesn't read the thing in a defined order, that leaves us
1270          * with the following maintenance requirements:
1271          *
1272          * 1. Update Config, return (init time STEs aren't live)
1273          * 2. Write everything apart from dword 0, sync, write dword 0, sync
1274          * 3. Update Config, sync
1275          */
1276         u64 val = le64_to_cpu(dst[0]);
1277         bool ste_live = false;
1278         struct arm_smmu_device *smmu = NULL;
1279         struct arm_smmu_s1_cfg *s1_cfg = NULL;
1280         struct arm_smmu_s2_cfg *s2_cfg = NULL;
1281         struct arm_smmu_domain *smmu_domain = NULL;
1282         struct arm_smmu_cmdq_ent prefetch_cmd = {
1283                 .opcode         = CMDQ_OP_PREFETCH_CFG,
1284                 .prefetch       = {
1285                         .sid    = sid,
1286                 },
1287         };
1288
1289         if (master) {
1290                 smmu_domain = master->domain;
1291                 smmu = master->smmu;
1292         }
1293
1294         if (smmu_domain) {
1295                 switch (smmu_domain->stage) {
1296                 case ARM_SMMU_DOMAIN_S1:
1297                         s1_cfg = &smmu_domain->s1_cfg;
1298                         break;
1299                 case ARM_SMMU_DOMAIN_S2:
1300                 case ARM_SMMU_DOMAIN_NESTED:
1301                         s2_cfg = &smmu_domain->s2_cfg;
1302                         break;
1303                 default:
1304                         break;
1305                 }
1306         }
1307
1308         if (val & STRTAB_STE_0_V) {
1309                 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1310                 case STRTAB_STE_0_CFG_BYPASS:
1311                         break;
1312                 case STRTAB_STE_0_CFG_S1_TRANS:
1313                 case STRTAB_STE_0_CFG_S2_TRANS:
1314                         ste_live = true;
1315                         break;
1316                 case STRTAB_STE_0_CFG_ABORT:
1317                         BUG_ON(!disable_bypass);
1318                         break;
1319                 default:
1320                         BUG(); /* STE corruption */
1321                 }
1322         }
1323
1324         /* Nuke the existing STE_0 value, as we're going to rewrite it */
1325         val = STRTAB_STE_0_V;
1326
1327         /* Bypass/fault */
1328         if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1329                 if (!smmu_domain && disable_bypass)
1330                         val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1331                 else
1332                         val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1333
1334                 dst[0] = cpu_to_le64(val);
1335                 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1336                                                 STRTAB_STE_1_SHCFG_INCOMING));
1337                 dst[2] = 0; /* Nuke the VMID */
1338                 /*
1339                  * The SMMU can perform negative caching, so we must sync
1340                  * the STE regardless of whether the old value was live.
1341                  */
1342                 if (smmu)
1343                         arm_smmu_sync_ste_for_sid(smmu, sid);
1344                 return;
1345         }
1346
1347         if (s1_cfg) {
1348                 u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?
1349                         STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
1350
1351                 BUG_ON(ste_live);
1352                 dst[1] = cpu_to_le64(
1353                          FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
1354                          FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1355                          FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1356                          FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1357                          FIELD_PREP(STRTAB_STE_1_STRW, strw));
1358
1359                 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1360                     !master->stall_enabled)
1361                         dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1362
1363                 val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1364                         FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1365                         FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |
1366                         FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1367         }
1368
1369         if (s2_cfg) {
1370                 BUG_ON(ste_live);
1371                 dst[2] = cpu_to_le64(
1372                          FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1373                          FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1374 #ifdef __BIG_ENDIAN
1375                          STRTAB_STE_2_S2ENDI |
1376 #endif
1377                          STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1378                          STRTAB_STE_2_S2R);
1379
1380                 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1381
1382                 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1383         }
1384
1385         if (master->ats_enabled)
1386                 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1387                                                  STRTAB_STE_1_EATS_TRANS));
1388
1389         arm_smmu_sync_ste_for_sid(smmu, sid);
1390         /* See comment in arm_smmu_write_ctx_desc() */
1391         WRITE_ONCE(dst[0], cpu_to_le64(val));
1392         arm_smmu_sync_ste_for_sid(smmu, sid);
1393
1394         /* It's likely that we'll want to use the new STE soon */
1395         if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1396                 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1397 }
1398
1399 static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent, bool force)
1400 {
1401         unsigned int i;
1402         u64 val = STRTAB_STE_0_V;
1403
1404         if (disable_bypass && !force)
1405                 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1406         else
1407                 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1408
1409         for (i = 0; i < nent; ++i) {
1410                 strtab[0] = cpu_to_le64(val);
1411                 strtab[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1412                                                    STRTAB_STE_1_SHCFG_INCOMING));
1413                 strtab[2] = 0;
1414                 strtab += STRTAB_STE_DWORDS;
1415         }
1416 }
1417
1418 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1419 {
1420         size_t size;
1421         void *strtab;
1422         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1423         struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1424
1425         if (desc->l2ptr)
1426                 return 0;
1427
1428         size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1429         strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1430
1431         desc->span = STRTAB_SPLIT + 1;
1432         desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1433                                           GFP_KERNEL);
1434         if (!desc->l2ptr) {
1435                 dev_err(smmu->dev,
1436                         "failed to allocate l2 stream table for SID %u\n",
1437                         sid);
1438                 return -ENOMEM;
1439         }
1440
1441         arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false);
1442         arm_smmu_write_strtab_l1_desc(strtab, desc);
1443         return 0;
1444 }
1445
1446 static struct arm_smmu_master *
1447 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
1448 {
1449         struct rb_node *node;
1450         struct arm_smmu_stream *stream;
1451
1452         lockdep_assert_held(&smmu->streams_mutex);
1453
1454         node = smmu->streams.rb_node;
1455         while (node) {
1456                 stream = rb_entry(node, struct arm_smmu_stream, node);
1457                 if (stream->id < sid)
1458                         node = node->rb_right;
1459                 else if (stream->id > sid)
1460                         node = node->rb_left;
1461                 else
1462                         return stream->master;
1463         }
1464
1465         return NULL;
1466 }
1467
1468 /* IRQ and event handlers */
1469 static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
1470 {
1471         int ret;
1472         u32 reason;
1473         u32 perm = 0;
1474         struct arm_smmu_master *master;
1475         bool ssid_valid = evt[0] & EVTQ_0_SSV;
1476         u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]);
1477         struct iommu_fault_event fault_evt = { };
1478         struct iommu_fault *flt = &fault_evt.fault;
1479
1480         switch (FIELD_GET(EVTQ_0_ID, evt[0])) {
1481         case EVT_ID_TRANSLATION_FAULT:
1482                 reason = IOMMU_FAULT_REASON_PTE_FETCH;
1483                 break;
1484         case EVT_ID_ADDR_SIZE_FAULT:
1485                 reason = IOMMU_FAULT_REASON_OOR_ADDRESS;
1486                 break;
1487         case EVT_ID_ACCESS_FAULT:
1488                 reason = IOMMU_FAULT_REASON_ACCESS;
1489                 break;
1490         case EVT_ID_PERMISSION_FAULT:
1491                 reason = IOMMU_FAULT_REASON_PERMISSION;
1492                 break;
1493         default:
1494                 return -EOPNOTSUPP;
1495         }
1496
1497         /* Stage-2 is always pinned at the moment */
1498         if (evt[1] & EVTQ_1_S2)
1499                 return -EFAULT;
1500
1501         if (evt[1] & EVTQ_1_RnW)
1502                 perm |= IOMMU_FAULT_PERM_READ;
1503         else
1504                 perm |= IOMMU_FAULT_PERM_WRITE;
1505
1506         if (evt[1] & EVTQ_1_InD)
1507                 perm |= IOMMU_FAULT_PERM_EXEC;
1508
1509         if (evt[1] & EVTQ_1_PnU)
1510                 perm |= IOMMU_FAULT_PERM_PRIV;
1511
1512         if (evt[1] & EVTQ_1_STALL) {
1513                 flt->type = IOMMU_FAULT_PAGE_REQ;
1514                 flt->prm = (struct iommu_fault_page_request) {
1515                         .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1516                         .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]),
1517                         .perm = perm,
1518                         .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]),
1519                 };
1520
1521                 if (ssid_valid) {
1522                         flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1523                         flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1524                 }
1525         } else {
1526                 flt->type = IOMMU_FAULT_DMA_UNRECOV;
1527                 flt->event = (struct iommu_fault_unrecoverable) {
1528                         .reason = reason,
1529                         .flags = IOMMU_FAULT_UNRECOV_ADDR_VALID,
1530                         .perm = perm,
1531                         .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]),
1532                 };
1533
1534                 if (ssid_valid) {
1535                         flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID;
1536                         flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1537                 }
1538         }
1539
1540         mutex_lock(&smmu->streams_mutex);
1541         master = arm_smmu_find_master(smmu, sid);
1542         if (!master) {
1543                 ret = -EINVAL;
1544                 goto out_unlock;
1545         }
1546
1547         ret = iommu_report_device_fault(master->dev, &fault_evt);
1548         if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) {
1549                 /* Nobody cared, abort the access */
1550                 struct iommu_page_response resp = {
1551                         .pasid          = flt->prm.pasid,
1552                         .grpid          = flt->prm.grpid,
1553                         .code           = IOMMU_PAGE_RESP_FAILURE,
1554                 };
1555                 arm_smmu_page_response(master->dev, &fault_evt, &resp);
1556         }
1557
1558 out_unlock:
1559         mutex_unlock(&smmu->streams_mutex);
1560         return ret;
1561 }
1562
1563 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1564 {
1565         int i, ret;
1566         struct arm_smmu_device *smmu = dev;
1567         struct arm_smmu_queue *q = &smmu->evtq.q;
1568         struct arm_smmu_ll_queue *llq = &q->llq;
1569         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1570                                       DEFAULT_RATELIMIT_BURST);
1571         u64 evt[EVTQ_ENT_DWORDS];
1572
1573         do {
1574                 while (!queue_remove_raw(q, evt)) {
1575                         u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1576
1577                         ret = arm_smmu_handle_evt(smmu, evt);
1578                         if (!ret || !__ratelimit(&rs))
1579                                 continue;
1580
1581                         dev_info(smmu->dev, "event 0x%02x received:\n", id);
1582                         for (i = 0; i < ARRAY_SIZE(evt); ++i)
1583                                 dev_info(smmu->dev, "\t0x%016llx\n",
1584                                          (unsigned long long)evt[i]);
1585
1586                         cond_resched();
1587                 }
1588
1589                 /*
1590                  * Not much we can do on overflow, so scream and pretend we're
1591                  * trying harder.
1592                  */
1593                 if (queue_sync_prod_in(q) == -EOVERFLOW)
1594                         dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1595         } while (!queue_empty(llq));
1596
1597         /* Sync our overflow flag, as we believe we're up to speed */
1598         queue_sync_cons_ovf(q);
1599         return IRQ_HANDLED;
1600 }
1601
1602 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1603 {
1604         u32 sid, ssid;
1605         u16 grpid;
1606         bool ssv, last;
1607
1608         sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1609         ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1610         ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
1611         last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1612         grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1613
1614         dev_info(smmu->dev, "unexpected PRI request received:\n");
1615         dev_info(smmu->dev,
1616                  "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1617                  sid, ssid, grpid, last ? "L" : "",
1618                  evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1619                  evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1620                  evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1621                  evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1622                  evt[1] & PRIQ_1_ADDR_MASK);
1623
1624         if (last) {
1625                 struct arm_smmu_cmdq_ent cmd = {
1626                         .opcode                 = CMDQ_OP_PRI_RESP,
1627                         .substream_valid        = ssv,
1628                         .pri                    = {
1629                                 .sid    = sid,
1630                                 .ssid   = ssid,
1631                                 .grpid  = grpid,
1632                                 .resp   = PRI_RESP_DENY,
1633                         },
1634                 };
1635
1636                 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1637         }
1638 }
1639
1640 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1641 {
1642         struct arm_smmu_device *smmu = dev;
1643         struct arm_smmu_queue *q = &smmu->priq.q;
1644         struct arm_smmu_ll_queue *llq = &q->llq;
1645         u64 evt[PRIQ_ENT_DWORDS];
1646
1647         do {
1648                 while (!queue_remove_raw(q, evt))
1649                         arm_smmu_handle_ppr(smmu, evt);
1650
1651                 if (queue_sync_prod_in(q) == -EOVERFLOW)
1652                         dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1653         } while (!queue_empty(llq));
1654
1655         /* Sync our overflow flag, as we believe we're up to speed */
1656         queue_sync_cons_ovf(q);
1657         return IRQ_HANDLED;
1658 }
1659
1660 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1661
1662 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1663 {
1664         u32 gerror, gerrorn, active;
1665         struct arm_smmu_device *smmu = dev;
1666
1667         gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1668         gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1669
1670         active = gerror ^ gerrorn;
1671         if (!(active & GERROR_ERR_MASK))
1672                 return IRQ_NONE; /* No errors pending */
1673
1674         dev_warn(smmu->dev,
1675                  "unexpected global error reported (0x%08x), this could be serious\n",
1676                  active);
1677
1678         if (active & GERROR_SFM_ERR) {
1679                 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1680                 arm_smmu_device_disable(smmu);
1681         }
1682
1683         if (active & GERROR_MSI_GERROR_ABT_ERR)
1684                 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1685
1686         if (active & GERROR_MSI_PRIQ_ABT_ERR)
1687                 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1688
1689         if (active & GERROR_MSI_EVTQ_ABT_ERR)
1690                 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1691
1692         if (active & GERROR_MSI_CMDQ_ABT_ERR)
1693                 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1694
1695         if (active & GERROR_PRIQ_ABT_ERR)
1696                 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1697
1698         if (active & GERROR_EVTQ_ABT_ERR)
1699                 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1700
1701         if (active & GERROR_CMDQ_ERR)
1702                 arm_smmu_cmdq_skip_err(smmu);
1703
1704         writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1705         return IRQ_HANDLED;
1706 }
1707
1708 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1709 {
1710         struct arm_smmu_device *smmu = dev;
1711
1712         arm_smmu_evtq_thread(irq, dev);
1713         if (smmu->features & ARM_SMMU_FEAT_PRI)
1714                 arm_smmu_priq_thread(irq, dev);
1715
1716         return IRQ_HANDLED;
1717 }
1718
1719 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1720 {
1721         arm_smmu_gerror_handler(irq, dev);
1722         return IRQ_WAKE_THREAD;
1723 }
1724
1725 static void
1726 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1727                         struct arm_smmu_cmdq_ent *cmd)
1728 {
1729         size_t log2_span;
1730         size_t span_mask;
1731         /* ATC invalidates are always on 4096-bytes pages */
1732         size_t inval_grain_shift = 12;
1733         unsigned long page_start, page_end;
1734
1735         /*
1736          * ATS and PASID:
1737          *
1738          * If substream_valid is clear, the PCIe TLP is sent without a PASID
1739          * prefix. In that case all ATC entries within the address range are
1740          * invalidated, including those that were requested with a PASID! There
1741          * is no way to invalidate only entries without PASID.
1742          *
1743          * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
1744          * traffic), translation requests without PASID create ATC entries
1745          * without PASID, which must be invalidated with substream_valid clear.
1746          * This has the unpleasant side-effect of invalidating all PASID-tagged
1747          * ATC entries within the address range.
1748          */
1749         *cmd = (struct arm_smmu_cmdq_ent) {
1750                 .opcode                 = CMDQ_OP_ATC_INV,
1751                 .substream_valid        = (ssid != IOMMU_NO_PASID),
1752                 .atc.ssid               = ssid,
1753         };
1754
1755         if (!size) {
1756                 cmd->atc.size = ATC_INV_SIZE_ALL;
1757                 return;
1758         }
1759
1760         page_start      = iova >> inval_grain_shift;
1761         page_end        = (iova + size - 1) >> inval_grain_shift;
1762
1763         /*
1764          * In an ATS Invalidate Request, the address must be aligned on the
1765          * range size, which must be a power of two number of page sizes. We
1766          * thus have to choose between grossly over-invalidating the region, or
1767          * splitting the invalidation into multiple commands. For simplicity
1768          * we'll go with the first solution, but should refine it in the future
1769          * if multiple commands are shown to be more efficient.
1770          *
1771          * Find the smallest power of two that covers the range. The most
1772          * significant differing bit between the start and end addresses,
1773          * fls(start ^ end), indicates the required span. For example:
1774          *
1775          * We want to invalidate pages [8; 11]. This is already the ideal range:
1776          *              x = 0b1000 ^ 0b1011 = 0b11
1777          *              span = 1 << fls(x) = 4
1778          *
1779          * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1780          *              x = 0b0111 ^ 0b1010 = 0b1101
1781          *              span = 1 << fls(x) = 16
1782          */
1783         log2_span       = fls_long(page_start ^ page_end);
1784         span_mask       = (1ULL << log2_span) - 1;
1785
1786         page_start      &= ~span_mask;
1787
1788         cmd->atc.addr   = page_start << inval_grain_shift;
1789         cmd->atc.size   = log2_span;
1790 }
1791
1792 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
1793 {
1794         int i;
1795         struct arm_smmu_cmdq_ent cmd;
1796         struct arm_smmu_cmdq_batch cmds;
1797
1798         arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
1799
1800         cmds.num = 0;
1801         for (i = 0; i < master->num_streams; i++) {
1802                 cmd.atc.sid = master->streams[i].id;
1803                 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
1804         }
1805
1806         return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
1807 }
1808
1809 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
1810                             unsigned long iova, size_t size)
1811 {
1812         int i;
1813         unsigned long flags;
1814         struct arm_smmu_cmdq_ent cmd;
1815         struct arm_smmu_master *master;
1816         struct arm_smmu_cmdq_batch cmds;
1817
1818         if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1819                 return 0;
1820
1821         /*
1822          * Ensure that we've completed prior invalidation of the main TLBs
1823          * before we read 'nr_ats_masters' in case of a concurrent call to
1824          * arm_smmu_enable_ats():
1825          *
1826          *      // unmap()                      // arm_smmu_enable_ats()
1827          *      TLBI+SYNC                       atomic_inc(&nr_ats_masters);
1828          *      smp_mb();                       [...]
1829          *      atomic_read(&nr_ats_masters);   pci_enable_ats() // writel()
1830          *
1831          * Ensures that we always see the incremented 'nr_ats_masters' count if
1832          * ATS was enabled at the PCI device before completion of the TLBI.
1833          */
1834         smp_mb();
1835         if (!atomic_read(&smmu_domain->nr_ats_masters))
1836                 return 0;
1837
1838         arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1839
1840         cmds.num = 0;
1841
1842         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1843         list_for_each_entry(master, &smmu_domain->devices, domain_head) {
1844                 if (!master->ats_enabled)
1845                         continue;
1846
1847                 for (i = 0; i < master->num_streams; i++) {
1848                         cmd.atc.sid = master->streams[i].id;
1849                         arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
1850                 }
1851         }
1852         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1853
1854         return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
1855 }
1856
1857 /* IO_PGTABLE API */
1858 static void arm_smmu_tlb_inv_context(void *cookie)
1859 {
1860         struct arm_smmu_domain *smmu_domain = cookie;
1861         struct arm_smmu_device *smmu = smmu_domain->smmu;
1862         struct arm_smmu_cmdq_ent cmd;
1863
1864         /*
1865          * NOTE: when io-pgtable is in non-strict mode, we may get here with
1866          * PTEs previously cleared by unmaps on the current CPU not yet visible
1867          * to the SMMU. We are relying on the dma_wmb() implicit during cmd
1868          * insertion to guarantee those are observed before the TLBI. Do be
1869          * careful, 007.
1870          */
1871         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1872                 arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);
1873         } else {
1874                 cmd.opcode      = CMDQ_OP_TLBI_S12_VMALL;
1875                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1876                 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1877         }
1878         arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, 0, 0);
1879 }
1880
1881 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
1882                                      unsigned long iova, size_t size,
1883                                      size_t granule,
1884                                      struct arm_smmu_domain *smmu_domain)
1885 {
1886         struct arm_smmu_device *smmu = smmu_domain->smmu;
1887         unsigned long end = iova + size, num_pages = 0, tg = 0;
1888         size_t inv_range = granule;
1889         struct arm_smmu_cmdq_batch cmds;
1890
1891         if (!size)
1892                 return;
1893
1894         if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1895                 /* Get the leaf page size */
1896                 tg = __ffs(smmu_domain->domain.pgsize_bitmap);
1897
1898                 num_pages = size >> tg;
1899
1900                 /* Convert page size of 12,14,16 (log2) to 1,2,3 */
1901                 cmd->tlbi.tg = (tg - 10) / 2;
1902
1903                 /*
1904                  * Determine what level the granule is at. For non-leaf, both
1905                  * io-pgtable and SVA pass a nominal last-level granule because
1906                  * they don't know what level(s) actually apply, so ignore that
1907                  * and leave TTL=0. However for various errata reasons we still
1908                  * want to use a range command, so avoid the SVA corner case
1909                  * where both scale and num could be 0 as well.
1910                  */
1911                 if (cmd->tlbi.leaf)
1912                         cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
1913                 else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
1914                         num_pages++;
1915         }
1916
1917         cmds.num = 0;
1918
1919         while (iova < end) {
1920                 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1921                         /*
1922                          * On each iteration of the loop, the range is 5 bits
1923                          * worth of the aligned size remaining.
1924                          * The range in pages is:
1925                          *
1926                          * range = (num_pages & (0x1f << __ffs(num_pages)))
1927                          */
1928                         unsigned long scale, num;
1929
1930                         /* Determine the power of 2 multiple number of pages */
1931                         scale = __ffs(num_pages);
1932                         cmd->tlbi.scale = scale;
1933
1934                         /* Determine how many chunks of 2^scale size we have */
1935                         num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
1936                         cmd->tlbi.num = num - 1;
1937
1938                         /* range is num * 2^scale * pgsize */
1939                         inv_range = num << (scale + tg);
1940
1941                         /* Clear out the lower order bits for the next iteration */
1942                         num_pages -= num << scale;
1943                 }
1944
1945                 cmd->tlbi.addr = iova;
1946                 arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
1947                 iova += inv_range;
1948         }
1949         arm_smmu_cmdq_batch_submit(smmu, &cmds);
1950 }
1951
1952 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
1953                                           size_t granule, bool leaf,
1954                                           struct arm_smmu_domain *smmu_domain)
1955 {
1956         struct arm_smmu_cmdq_ent cmd = {
1957                 .tlbi = {
1958                         .leaf   = leaf,
1959                 },
1960         };
1961
1962         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1963                 cmd.opcode      = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
1964                                   CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
1965                 cmd.tlbi.asid   = smmu_domain->s1_cfg.cd.asid;
1966         } else {
1967                 cmd.opcode      = CMDQ_OP_TLBI_S2_IPA;
1968                 cmd.tlbi.vmid   = smmu_domain->s2_cfg.vmid;
1969         }
1970         __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
1971
1972         /*
1973          * Unfortunately, this can't be leaf-only since we may have
1974          * zapped an entire table.
1975          */
1976         arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, iova, size);
1977 }
1978
1979 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
1980                                  size_t granule, bool leaf,
1981                                  struct arm_smmu_domain *smmu_domain)
1982 {
1983         struct arm_smmu_cmdq_ent cmd = {
1984                 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
1985                           CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
1986                 .tlbi = {
1987                         .asid   = asid,
1988                         .leaf   = leaf,
1989                 },
1990         };
1991
1992         __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
1993 }
1994
1995 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
1996                                          unsigned long iova, size_t granule,
1997                                          void *cookie)
1998 {
1999         struct arm_smmu_domain *smmu_domain = cookie;
2000         struct iommu_domain *domain = &smmu_domain->domain;
2001
2002         iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2003 }
2004
2005 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2006                                   size_t granule, void *cookie)
2007 {
2008         arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
2009 }
2010
2011 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2012         .tlb_flush_all  = arm_smmu_tlb_inv_context,
2013         .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2014         .tlb_add_page   = arm_smmu_tlb_inv_page_nosync,
2015 };
2016
2017 /* IOMMU API */
2018 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2019 {
2020         struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2021
2022         switch (cap) {
2023         case IOMMU_CAP_CACHE_COHERENCY:
2024                 /* Assume that a coherent TCU implies coherent TBUs */
2025                 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2026         case IOMMU_CAP_NOEXEC:
2027         case IOMMU_CAP_DEFERRED_FLUSH:
2028                 return true;
2029         default:
2030                 return false;
2031         }
2032 }
2033
2034 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
2035 {
2036         struct arm_smmu_domain *smmu_domain;
2037
2038         if (type == IOMMU_DOMAIN_SVA)
2039                 return arm_smmu_sva_domain_alloc();
2040
2041         if (type != IOMMU_DOMAIN_UNMANAGED &&
2042             type != IOMMU_DOMAIN_DMA &&
2043             type != IOMMU_DOMAIN_IDENTITY)
2044                 return NULL;
2045
2046         /*
2047          * Allocate the domain and initialise some of its data structures.
2048          * We can't really do anything meaningful until we've added a
2049          * master.
2050          */
2051         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2052         if (!smmu_domain)
2053                 return NULL;
2054
2055         mutex_init(&smmu_domain->init_mutex);
2056         INIT_LIST_HEAD(&smmu_domain->devices);
2057         spin_lock_init(&smmu_domain->devices_lock);
2058         INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
2059
2060         return &smmu_domain->domain;
2061 }
2062
2063 static void arm_smmu_domain_free(struct iommu_domain *domain)
2064 {
2065         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2066         struct arm_smmu_device *smmu = smmu_domain->smmu;
2067
2068         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2069
2070         /* Free the CD and ASID, if we allocated them */
2071         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2072                 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2073
2074                 /* Prevent SVA from touching the CD while we're freeing it */
2075                 mutex_lock(&arm_smmu_asid_lock);
2076                 if (cfg->cdcfg.cdtab)
2077                         arm_smmu_free_cd_tables(smmu_domain);
2078                 arm_smmu_free_asid(&cfg->cd);
2079                 mutex_unlock(&arm_smmu_asid_lock);
2080         } else {
2081                 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2082                 if (cfg->vmid)
2083                         ida_free(&smmu->vmid_map, cfg->vmid);
2084         }
2085
2086         kfree(smmu_domain);
2087 }
2088
2089 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
2090                                        struct arm_smmu_master *master,
2091                                        struct io_pgtable_cfg *pgtbl_cfg)
2092 {
2093         int ret;
2094         u32 asid;
2095         struct arm_smmu_device *smmu = smmu_domain->smmu;
2096         struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2097         typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2098
2099         refcount_set(&cfg->cd.refs, 1);
2100
2101         /* Prevent SVA from modifying the ASID until it is written to the CD */
2102         mutex_lock(&arm_smmu_asid_lock);
2103         ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,
2104                        XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2105         if (ret)
2106                 goto out_unlock;
2107
2108         cfg->s1cdmax = master->ssid_bits;
2109
2110         smmu_domain->stall_enabled = master->stall_enabled;
2111
2112         ret = arm_smmu_alloc_cd_tables(smmu_domain);
2113         if (ret)
2114                 goto out_free_asid;
2115
2116         cfg->cd.asid    = (u16)asid;
2117         cfg->cd.ttbr    = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
2118         cfg->cd.tcr     = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
2119                           FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
2120                           FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
2121                           FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
2122                           FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
2123                           FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
2124                           CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
2125         cfg->cd.mair    = pgtbl_cfg->arm_lpae_s1_cfg.mair;
2126
2127         /*
2128          * Note that this will end up calling arm_smmu_sync_cd() before
2129          * the master has been added to the devices list for this domain.
2130          * This isn't an issue because the STE hasn't been installed yet.
2131          */
2132         ret = arm_smmu_write_ctx_desc(smmu_domain, IOMMU_NO_PASID, &cfg->cd);
2133         if (ret)
2134                 goto out_free_cd_tables;
2135
2136         mutex_unlock(&arm_smmu_asid_lock);
2137         return 0;
2138
2139 out_free_cd_tables:
2140         arm_smmu_free_cd_tables(smmu_domain);
2141 out_free_asid:
2142         arm_smmu_free_asid(&cfg->cd);
2143 out_unlock:
2144         mutex_unlock(&arm_smmu_asid_lock);
2145         return ret;
2146 }
2147
2148 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
2149                                        struct arm_smmu_master *master,
2150                                        struct io_pgtable_cfg *pgtbl_cfg)
2151 {
2152         int vmid;
2153         struct arm_smmu_device *smmu = smmu_domain->smmu;
2154         struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2155         typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2156
2157         /* Reserve VMID 0 for stage-2 bypass STEs */
2158         vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2159                                GFP_KERNEL);
2160         if (vmid < 0)
2161                 return vmid;
2162
2163         vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2164         cfg->vmid       = (u16)vmid;
2165         cfg->vttbr      = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2166         cfg->vtcr       = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
2167                           FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
2168                           FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
2169                           FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
2170                           FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
2171                           FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
2172                           FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2173         return 0;
2174 }
2175
2176 static int arm_smmu_domain_finalise(struct iommu_domain *domain,
2177                                     struct arm_smmu_master *master)
2178 {
2179         int ret;
2180         unsigned long ias, oas;
2181         enum io_pgtable_fmt fmt;
2182         struct io_pgtable_cfg pgtbl_cfg;
2183         struct io_pgtable_ops *pgtbl_ops;
2184         int (*finalise_stage_fn)(struct arm_smmu_domain *,
2185                                  struct arm_smmu_master *,
2186                                  struct io_pgtable_cfg *);
2187         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2188         struct arm_smmu_device *smmu = smmu_domain->smmu;
2189
2190         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2191                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2192                 return 0;
2193         }
2194
2195         /* Restrict the stage to what we can actually support */
2196         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2197                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2198         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2199                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2200
2201         switch (smmu_domain->stage) {
2202         case ARM_SMMU_DOMAIN_S1:
2203                 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2204                 ias = min_t(unsigned long, ias, VA_BITS);
2205                 oas = smmu->ias;
2206                 fmt = ARM_64_LPAE_S1;
2207                 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2208                 break;
2209         case ARM_SMMU_DOMAIN_NESTED:
2210         case ARM_SMMU_DOMAIN_S2:
2211                 ias = smmu->ias;
2212                 oas = smmu->oas;
2213                 fmt = ARM_64_LPAE_S2;
2214                 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2215                 break;
2216         default:
2217                 return -EINVAL;
2218         }
2219
2220         pgtbl_cfg = (struct io_pgtable_cfg) {
2221                 .pgsize_bitmap  = smmu->pgsize_bitmap,
2222                 .ias            = ias,
2223                 .oas            = oas,
2224                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2225                 .tlb            = &arm_smmu_flush_ops,
2226                 .iommu_dev      = smmu->dev,
2227         };
2228
2229         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2230         if (!pgtbl_ops)
2231                 return -ENOMEM;
2232
2233         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2234         domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2235         domain->geometry.force_aperture = true;
2236
2237         ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
2238         if (ret < 0) {
2239                 free_io_pgtable_ops(pgtbl_ops);
2240                 return ret;
2241         }
2242
2243         smmu_domain->pgtbl_ops = pgtbl_ops;
2244         return 0;
2245 }
2246
2247 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2248 {
2249         __le64 *step;
2250         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2251
2252         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2253                 struct arm_smmu_strtab_l1_desc *l1_desc;
2254                 int idx;
2255
2256                 /* Two-level walk */
2257                 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
2258                 l1_desc = &cfg->l1_desc[idx];
2259                 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2260                 step = &l1_desc->l2ptr[idx];
2261         } else {
2262                 /* Simple linear lookup */
2263                 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2264         }
2265
2266         return step;
2267 }
2268
2269 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
2270 {
2271         int i, j;
2272         struct arm_smmu_device *smmu = master->smmu;
2273
2274         for (i = 0; i < master->num_streams; ++i) {
2275                 u32 sid = master->streams[i].id;
2276                 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
2277
2278                 /* Bridged PCI devices may end up with duplicated IDs */
2279                 for (j = 0; j < i; j++)
2280                         if (master->streams[j].id == sid)
2281                                 break;
2282                 if (j < i)
2283                         continue;
2284
2285                 arm_smmu_write_strtab_ent(master, sid, step);
2286         }
2287 }
2288
2289 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2290 {
2291         struct device *dev = master->dev;
2292         struct arm_smmu_device *smmu = master->smmu;
2293         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2294
2295         if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2296                 return false;
2297
2298         if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2299                 return false;
2300
2301         return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2302 }
2303
2304 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2305 {
2306         size_t stu;
2307         struct pci_dev *pdev;
2308         struct arm_smmu_device *smmu = master->smmu;
2309         struct arm_smmu_domain *smmu_domain = master->domain;
2310
2311         /* Don't enable ATS at the endpoint if it's not enabled in the STE */
2312         if (!master->ats_enabled)
2313                 return;
2314
2315         /* Smallest Translation Unit: log2 of the smallest supported granule */
2316         stu = __ffs(smmu->pgsize_bitmap);
2317         pdev = to_pci_dev(master->dev);
2318
2319         atomic_inc(&smmu_domain->nr_ats_masters);
2320         arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, 0, 0);
2321         if (pci_enable_ats(pdev, stu))
2322                 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2323 }
2324
2325 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
2326 {
2327         struct arm_smmu_domain *smmu_domain = master->domain;
2328
2329         if (!master->ats_enabled)
2330                 return;
2331
2332         pci_disable_ats(to_pci_dev(master->dev));
2333         /*
2334          * Ensure ATS is disabled at the endpoint before we issue the
2335          * ATC invalidation via the SMMU.
2336          */
2337         wmb();
2338         arm_smmu_atc_inv_master(master);
2339         atomic_dec(&smmu_domain->nr_ats_masters);
2340 }
2341
2342 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2343 {
2344         int ret;
2345         int features;
2346         int num_pasids;
2347         struct pci_dev *pdev;
2348
2349         if (!dev_is_pci(master->dev))
2350                 return -ENODEV;
2351
2352         pdev = to_pci_dev(master->dev);
2353
2354         features = pci_pasid_features(pdev);
2355         if (features < 0)
2356                 return features;
2357
2358         num_pasids = pci_max_pasids(pdev);
2359         if (num_pasids <= 0)
2360                 return num_pasids;
2361
2362         ret = pci_enable_pasid(pdev, features);
2363         if (ret) {
2364                 dev_err(&pdev->dev, "Failed to enable PASID\n");
2365                 return ret;
2366         }
2367
2368         master->ssid_bits = min_t(u8, ilog2(num_pasids),
2369                                   master->smmu->ssid_bits);
2370         return 0;
2371 }
2372
2373 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2374 {
2375         struct pci_dev *pdev;
2376
2377         if (!dev_is_pci(master->dev))
2378                 return;
2379
2380         pdev = to_pci_dev(master->dev);
2381
2382         if (!pdev->pasid_enabled)
2383                 return;
2384
2385         master->ssid_bits = 0;
2386         pci_disable_pasid(pdev);
2387 }
2388
2389 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
2390 {
2391         unsigned long flags;
2392         struct arm_smmu_domain *smmu_domain = master->domain;
2393
2394         if (!smmu_domain)
2395                 return;
2396
2397         arm_smmu_disable_ats(master);
2398
2399         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2400         list_del(&master->domain_head);
2401         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2402
2403         master->domain = NULL;
2404         master->ats_enabled = false;
2405         arm_smmu_install_ste_for_dev(master);
2406 }
2407
2408 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2409 {
2410         int ret = 0;
2411         unsigned long flags;
2412         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2413         struct arm_smmu_device *smmu;
2414         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2415         struct arm_smmu_master *master;
2416
2417         if (!fwspec)
2418                 return -ENOENT;
2419
2420         master = dev_iommu_priv_get(dev);
2421         smmu = master->smmu;
2422
2423         /*
2424          * Checking that SVA is disabled ensures that this device isn't bound to
2425          * any mm, and can be safely detached from its old domain. Bonds cannot
2426          * be removed concurrently since we're holding the group mutex.
2427          */
2428         if (arm_smmu_master_sva_enabled(master)) {
2429                 dev_err(dev, "cannot attach - SVA enabled\n");
2430                 return -EBUSY;
2431         }
2432
2433         arm_smmu_detach_dev(master);
2434
2435         mutex_lock(&smmu_domain->init_mutex);
2436
2437         if (!smmu_domain->smmu) {
2438                 smmu_domain->smmu = smmu;
2439                 ret = arm_smmu_domain_finalise(domain, master);
2440                 if (ret) {
2441                         smmu_domain->smmu = NULL;
2442                         goto out_unlock;
2443                 }
2444         } else if (smmu_domain->smmu != smmu) {
2445                 ret = -EINVAL;
2446                 goto out_unlock;
2447         } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2448                    master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
2449                 ret = -EINVAL;
2450                 goto out_unlock;
2451         } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2452                    smmu_domain->stall_enabled != master->stall_enabled) {
2453                 ret = -EINVAL;
2454                 goto out_unlock;
2455         }
2456
2457         master->domain = smmu_domain;
2458
2459         /*
2460          * The SMMU does not support enabling ATS with bypass. When the STE is
2461          * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and
2462          * Translated transactions are denied as though ATS is disabled for the
2463          * stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and
2464          * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry).
2465          */
2466         if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2467                 master->ats_enabled = arm_smmu_ats_supported(master);
2468
2469         arm_smmu_install_ste_for_dev(master);
2470
2471         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2472         list_add(&master->domain_head, &smmu_domain->devices);
2473         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2474
2475         arm_smmu_enable_ats(master);
2476
2477 out_unlock:
2478         mutex_unlock(&smmu_domain->init_mutex);
2479         return ret;
2480 }
2481
2482 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
2483                               phys_addr_t paddr, size_t pgsize, size_t pgcount,
2484                               int prot, gfp_t gfp, size_t *mapped)
2485 {
2486         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2487
2488         if (!ops)
2489                 return -ENODEV;
2490
2491         return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
2492 }
2493
2494 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
2495                                    size_t pgsize, size_t pgcount,
2496                                    struct iommu_iotlb_gather *gather)
2497 {
2498         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2499         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2500
2501         if (!ops)
2502                 return 0;
2503
2504         return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
2505 }
2506
2507 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
2508 {
2509         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2510
2511         if (smmu_domain->smmu)
2512                 arm_smmu_tlb_inv_context(smmu_domain);
2513 }
2514
2515 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
2516                                 struct iommu_iotlb_gather *gather)
2517 {
2518         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2519
2520         if (!gather->pgsize)
2521                 return;
2522
2523         arm_smmu_tlb_inv_range_domain(gather->start,
2524                                       gather->end - gather->start + 1,
2525                                       gather->pgsize, true, smmu_domain);
2526 }
2527
2528 static phys_addr_t
2529 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2530 {
2531         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2532
2533         if (!ops)
2534                 return 0;
2535
2536         return ops->iova_to_phys(ops, iova);
2537 }
2538
2539 static struct platform_driver arm_smmu_driver;
2540
2541 static
2542 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2543 {
2544         struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
2545                                                           fwnode);
2546         put_device(dev);
2547         return dev ? dev_get_drvdata(dev) : NULL;
2548 }
2549
2550 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2551 {
2552         unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2553
2554         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2555                 limit *= 1UL << STRTAB_SPLIT;
2556
2557         return sid < limit;
2558 }
2559
2560 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
2561 {
2562         /* Check the SIDs are in range of the SMMU and our stream table */
2563         if (!arm_smmu_sid_in_range(smmu, sid))
2564                 return -ERANGE;
2565
2566         /* Ensure l2 strtab is initialised */
2567         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2568                 return arm_smmu_init_l2_strtab(smmu, sid);
2569
2570         return 0;
2571 }
2572
2573 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
2574                                   struct arm_smmu_master *master)
2575 {
2576         int i;
2577         int ret = 0;
2578         struct arm_smmu_stream *new_stream, *cur_stream;
2579         struct rb_node **new_node, *parent_node = NULL;
2580         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2581
2582         master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
2583                                   GFP_KERNEL);
2584         if (!master->streams)
2585                 return -ENOMEM;
2586         master->num_streams = fwspec->num_ids;
2587
2588         mutex_lock(&smmu->streams_mutex);
2589         for (i = 0; i < fwspec->num_ids; i++) {
2590                 u32 sid = fwspec->ids[i];
2591
2592                 new_stream = &master->streams[i];
2593                 new_stream->id = sid;
2594                 new_stream->master = master;
2595
2596                 ret = arm_smmu_init_sid_strtab(smmu, sid);
2597                 if (ret)
2598                         break;
2599
2600                 /* Insert into SID tree */
2601                 new_node = &(smmu->streams.rb_node);
2602                 while (*new_node) {
2603                         cur_stream = rb_entry(*new_node, struct arm_smmu_stream,
2604                                               node);
2605                         parent_node = *new_node;
2606                         if (cur_stream->id > new_stream->id) {
2607                                 new_node = &((*new_node)->rb_left);
2608                         } else if (cur_stream->id < new_stream->id) {
2609                                 new_node = &((*new_node)->rb_right);
2610                         } else {
2611                                 dev_warn(master->dev,
2612                                          "stream %u already in tree\n",
2613                                          cur_stream->id);
2614                                 ret = -EINVAL;
2615                                 break;
2616                         }
2617                 }
2618                 if (ret)
2619                         break;
2620
2621                 rb_link_node(&new_stream->node, parent_node, new_node);
2622                 rb_insert_color(&new_stream->node, &smmu->streams);
2623         }
2624
2625         if (ret) {
2626                 for (i--; i >= 0; i--)
2627                         rb_erase(&master->streams[i].node, &smmu->streams);
2628                 kfree(master->streams);
2629         }
2630         mutex_unlock(&smmu->streams_mutex);
2631
2632         return ret;
2633 }
2634
2635 static void arm_smmu_remove_master(struct arm_smmu_master *master)
2636 {
2637         int i;
2638         struct arm_smmu_device *smmu = master->smmu;
2639         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2640
2641         if (!smmu || !master->streams)
2642                 return;
2643
2644         mutex_lock(&smmu->streams_mutex);
2645         for (i = 0; i < fwspec->num_ids; i++)
2646                 rb_erase(&master->streams[i].node, &smmu->streams);
2647         mutex_unlock(&smmu->streams_mutex);
2648
2649         kfree(master->streams);
2650 }
2651
2652 static struct iommu_ops arm_smmu_ops;
2653
2654 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
2655 {
2656         int ret;
2657         struct arm_smmu_device *smmu;
2658         struct arm_smmu_master *master;
2659         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2660
2661         if (!fwspec || fwspec->ops != &arm_smmu_ops)
2662                 return ERR_PTR(-ENODEV);
2663
2664         if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
2665                 return ERR_PTR(-EBUSY);
2666
2667         smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2668         if (!smmu)
2669                 return ERR_PTR(-ENODEV);
2670
2671         master = kzalloc(sizeof(*master), GFP_KERNEL);
2672         if (!master)
2673                 return ERR_PTR(-ENOMEM);
2674
2675         master->dev = dev;
2676         master->smmu = smmu;
2677         INIT_LIST_HEAD(&master->bonds);
2678         dev_iommu_priv_set(dev, master);
2679
2680         ret = arm_smmu_insert_master(smmu, master);
2681         if (ret)
2682                 goto err_free_master;
2683
2684         device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
2685         master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
2686
2687         /*
2688          * Note that PASID must be enabled before, and disabled after ATS:
2689          * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
2690          *
2691          *   Behavior is undefined if this bit is Set and the value of the PASID
2692          *   Enable, Execute Requested Enable, or Privileged Mode Requested bits
2693          *   are changed.
2694          */
2695         arm_smmu_enable_pasid(master);
2696
2697         if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
2698                 master->ssid_bits = min_t(u8, master->ssid_bits,
2699                                           CTXDESC_LINEAR_CDMAX);
2700
2701         if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
2702              device_property_read_bool(dev, "dma-can-stall")) ||
2703             smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
2704                 master->stall_enabled = true;
2705
2706         return &smmu->iommu;
2707
2708 err_free_master:
2709         kfree(master);
2710         dev_iommu_priv_set(dev, NULL);
2711         return ERR_PTR(ret);
2712 }
2713
2714 static void arm_smmu_release_device(struct device *dev)
2715 {
2716         struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2717
2718         if (WARN_ON(arm_smmu_master_sva_enabled(master)))
2719                 iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
2720         arm_smmu_detach_dev(master);
2721         arm_smmu_disable_pasid(master);
2722         arm_smmu_remove_master(master);
2723         kfree(master);
2724 }
2725
2726 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2727 {
2728         struct iommu_group *group;
2729
2730         /*
2731          * We don't support devices sharing stream IDs other than PCI RID
2732          * aliases, since the necessary ID-to-device lookup becomes rather
2733          * impractical given a potential sparse 32-bit stream ID space.
2734          */
2735         if (dev_is_pci(dev))
2736                 group = pci_device_group(dev);
2737         else
2738                 group = generic_device_group(dev);
2739
2740         return group;
2741 }
2742
2743 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
2744 {
2745         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2746         int ret = 0;
2747
2748         mutex_lock(&smmu_domain->init_mutex);
2749         if (smmu_domain->smmu)
2750                 ret = -EPERM;
2751         else
2752                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2753         mutex_unlock(&smmu_domain->init_mutex);
2754
2755         return ret;
2756 }
2757
2758 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2759 {
2760         return iommu_fwspec_add_ids(dev, args->args, 1);
2761 }
2762
2763 static void arm_smmu_get_resv_regions(struct device *dev,
2764                                       struct list_head *head)
2765 {
2766         struct iommu_resv_region *region;
2767         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2768
2769         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2770                                          prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
2771         if (!region)
2772                 return;
2773
2774         list_add_tail(&region->list, head);
2775
2776         iommu_dma_get_resv_regions(dev, head);
2777 }
2778
2779 static int arm_smmu_dev_enable_feature(struct device *dev,
2780                                        enum iommu_dev_features feat)
2781 {
2782         struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2783
2784         if (!master)
2785                 return -ENODEV;
2786
2787         switch (feat) {
2788         case IOMMU_DEV_FEAT_IOPF:
2789                 if (!arm_smmu_master_iopf_supported(master))
2790                         return -EINVAL;
2791                 if (master->iopf_enabled)
2792                         return -EBUSY;
2793                 master->iopf_enabled = true;
2794                 return 0;
2795         case IOMMU_DEV_FEAT_SVA:
2796                 if (!arm_smmu_master_sva_supported(master))
2797                         return -EINVAL;
2798                 if (arm_smmu_master_sva_enabled(master))
2799                         return -EBUSY;
2800                 return arm_smmu_master_enable_sva(master);
2801         default:
2802                 return -EINVAL;
2803         }
2804 }
2805
2806 static int arm_smmu_dev_disable_feature(struct device *dev,
2807                                         enum iommu_dev_features feat)
2808 {
2809         struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2810
2811         if (!master)
2812                 return -EINVAL;
2813
2814         switch (feat) {
2815         case IOMMU_DEV_FEAT_IOPF:
2816                 if (!master->iopf_enabled)
2817                         return -EINVAL;
2818                 if (master->sva_enabled)
2819                         return -EBUSY;
2820                 master->iopf_enabled = false;
2821                 return 0;
2822         case IOMMU_DEV_FEAT_SVA:
2823                 if (!arm_smmu_master_sva_enabled(master))
2824                         return -EINVAL;
2825                 return arm_smmu_master_disable_sva(master);
2826         default:
2827                 return -EINVAL;
2828         }
2829 }
2830
2831 /*
2832  * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
2833  * PCIe link and save the data to memory by DMA. The hardware is restricted to
2834  * use identity mapping only.
2835  */
2836 #define IS_HISI_PTT_DEVICE(pdev)        ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
2837                                          (pdev)->device == 0xa12e)
2838
2839 static int arm_smmu_def_domain_type(struct device *dev)
2840 {
2841         if (dev_is_pci(dev)) {
2842                 struct pci_dev *pdev = to_pci_dev(dev);
2843
2844                 if (IS_HISI_PTT_DEVICE(pdev))
2845                         return IOMMU_DOMAIN_IDENTITY;
2846         }
2847
2848         return 0;
2849 }
2850
2851 static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
2852 {
2853         struct iommu_domain *domain;
2854
2855         domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
2856         if (WARN_ON(IS_ERR(domain)) || !domain)
2857                 return;
2858
2859         arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
2860 }
2861
2862 static struct iommu_ops arm_smmu_ops = {
2863         .capable                = arm_smmu_capable,
2864         .domain_alloc           = arm_smmu_domain_alloc,
2865         .probe_device           = arm_smmu_probe_device,
2866         .release_device         = arm_smmu_release_device,
2867         .device_group           = arm_smmu_device_group,
2868         .of_xlate               = arm_smmu_of_xlate,
2869         .get_resv_regions       = arm_smmu_get_resv_regions,
2870         .remove_dev_pasid       = arm_smmu_remove_dev_pasid,
2871         .dev_enable_feat        = arm_smmu_dev_enable_feature,
2872         .dev_disable_feat       = arm_smmu_dev_disable_feature,
2873         .page_response          = arm_smmu_page_response,
2874         .def_domain_type        = arm_smmu_def_domain_type,
2875         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
2876         .owner                  = THIS_MODULE,
2877         .default_domain_ops = &(const struct iommu_domain_ops) {
2878                 .attach_dev             = arm_smmu_attach_dev,
2879                 .map_pages              = arm_smmu_map_pages,
2880                 .unmap_pages            = arm_smmu_unmap_pages,
2881                 .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
2882                 .iotlb_sync             = arm_smmu_iotlb_sync,
2883                 .iova_to_phys           = arm_smmu_iova_to_phys,
2884                 .enable_nesting         = arm_smmu_enable_nesting,
2885                 .free                   = arm_smmu_domain_free,
2886         }
2887 };
2888
2889 /* Probing and initialisation functions */
2890 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2891                                    struct arm_smmu_queue *q,
2892                                    void __iomem *page,
2893                                    unsigned long prod_off,
2894                                    unsigned long cons_off,
2895                                    size_t dwords, const char *name)
2896 {
2897         size_t qsz;
2898
2899         do {
2900                 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
2901                 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2902                                               GFP_KERNEL);
2903                 if (q->base || qsz < PAGE_SIZE)
2904                         break;
2905
2906                 q->llq.max_n_shift--;
2907         } while (1);
2908
2909         if (!q->base) {
2910                 dev_err(smmu->dev,
2911                         "failed to allocate queue (0x%zx bytes) for %s\n",
2912                         qsz, name);
2913                 return -ENOMEM;
2914         }
2915
2916         if (!WARN_ON(q->base_dma & (qsz - 1))) {
2917                 dev_info(smmu->dev, "allocated %u entries for %s\n",
2918                          1 << q->llq.max_n_shift, name);
2919         }
2920
2921         q->prod_reg     = page + prod_off;
2922         q->cons_reg     = page + cons_off;
2923         q->ent_dwords   = dwords;
2924
2925         q->q_base  = Q_BASE_RWA;
2926         q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2927         q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
2928
2929         q->llq.prod = q->llq.cons = 0;
2930         return 0;
2931 }
2932
2933 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
2934 {
2935         struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
2936         unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
2937
2938         atomic_set(&cmdq->owner_prod, 0);
2939         atomic_set(&cmdq->lock, 0);
2940
2941         cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
2942                                                               GFP_KERNEL);
2943         if (!cmdq->valid_map)
2944                 return -ENOMEM;
2945
2946         return 0;
2947 }
2948
2949 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2950 {
2951         int ret;
2952
2953         /* cmdq */
2954         ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
2955                                       ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
2956                                       CMDQ_ENT_DWORDS, "cmdq");
2957         if (ret)
2958                 return ret;
2959
2960         ret = arm_smmu_cmdq_init(smmu);
2961         if (ret)
2962                 return ret;
2963
2964         /* evtq */
2965         ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
2966                                       ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
2967                                       EVTQ_ENT_DWORDS, "evtq");
2968         if (ret)
2969                 return ret;
2970
2971         if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
2972             (smmu->features & ARM_SMMU_FEAT_STALLS)) {
2973                 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
2974                 if (!smmu->evtq.iopf)
2975                         return -ENOMEM;
2976         }
2977
2978         /* priq */
2979         if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2980                 return 0;
2981
2982         return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
2983                                        ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
2984                                        PRIQ_ENT_DWORDS, "priq");
2985 }
2986
2987 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2988 {
2989         unsigned int i;
2990         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2991         void *strtab = smmu->strtab_cfg.strtab;
2992
2993         cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents,
2994                                     sizeof(*cfg->l1_desc), GFP_KERNEL);
2995         if (!cfg->l1_desc)
2996                 return -ENOMEM;
2997
2998         for (i = 0; i < cfg->num_l1_ents; ++i) {
2999                 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
3000                 strtab += STRTAB_L1_DESC_DWORDS << 3;
3001         }
3002
3003         return 0;
3004 }
3005
3006 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3007 {
3008         void *strtab;
3009         u64 reg;
3010         u32 size, l1size;
3011         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3012
3013         /* Calculate the L1 size, capped to the SIDSIZE. */
3014         size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
3015         size = min(size, smmu->sid_bits - STRTAB_SPLIT);
3016         cfg->num_l1_ents = 1 << size;
3017
3018         size += STRTAB_SPLIT;
3019         if (size < smmu->sid_bits)
3020                 dev_warn(smmu->dev,
3021                          "2-level strtab only covers %u/%u bits of SID\n",
3022                          size, smmu->sid_bits);
3023
3024         l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
3025         strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
3026                                      GFP_KERNEL);
3027         if (!strtab) {
3028                 dev_err(smmu->dev,
3029                         "failed to allocate l1 stream table (%u bytes)\n",
3030                         l1size);
3031                 return -ENOMEM;
3032         }
3033         cfg->strtab = strtab;
3034
3035         /* Configure strtab_base_cfg for 2 levels */
3036         reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
3037         reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
3038         reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
3039         cfg->strtab_base_cfg = reg;
3040
3041         return arm_smmu_init_l1_strtab(smmu);
3042 }
3043
3044 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3045 {
3046         void *strtab;
3047         u64 reg;
3048         u32 size;
3049         struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3050
3051         size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
3052         strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
3053                                      GFP_KERNEL);
3054         if (!strtab) {
3055                 dev_err(smmu->dev,
3056                         "failed to allocate linear stream table (%u bytes)\n",
3057                         size);
3058                 return -ENOMEM;
3059         }
3060         cfg->strtab = strtab;
3061         cfg->num_l1_ents = 1 << smmu->sid_bits;
3062
3063         /* Configure strtab_base_cfg for a linear table covering all SIDs */
3064         reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
3065         reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
3066         cfg->strtab_base_cfg = reg;
3067
3068         arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false);
3069         return 0;
3070 }
3071
3072 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3073 {
3074         u64 reg;
3075         int ret;
3076
3077         if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3078                 ret = arm_smmu_init_strtab_2lvl(smmu);
3079         else
3080                 ret = arm_smmu_init_strtab_linear(smmu);
3081
3082         if (ret)
3083                 return ret;
3084
3085         /* Set the strtab base address */
3086         reg  = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
3087         reg |= STRTAB_BASE_RA;
3088         smmu->strtab_cfg.strtab_base = reg;
3089
3090         ida_init(&smmu->vmid_map);
3091
3092         return 0;
3093 }
3094
3095 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3096 {
3097         int ret;
3098
3099         mutex_init(&smmu->streams_mutex);
3100         smmu->streams = RB_ROOT;
3101
3102         ret = arm_smmu_init_queues(smmu);
3103         if (ret)
3104                 return ret;
3105
3106         return arm_smmu_init_strtab(smmu);
3107 }
3108
3109 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3110                                    unsigned int reg_off, unsigned int ack_off)
3111 {
3112         u32 reg;
3113
3114         writel_relaxed(val, smmu->base + reg_off);
3115         return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3116                                           1, ARM_SMMU_POLL_TIMEOUT_US);
3117 }
3118
3119 /* GBPA is "special" */
3120 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3121 {
3122         int ret;
3123         u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3124
3125         ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3126                                          1, ARM_SMMU_POLL_TIMEOUT_US);
3127         if (ret)
3128                 return ret;
3129
3130         reg &= ~clr;
3131         reg |= set;
3132         writel_relaxed(reg | GBPA_UPDATE, gbpa);
3133         ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3134                                          1, ARM_SMMU_POLL_TIMEOUT_US);
3135
3136         if (ret)
3137                 dev_err(smmu->dev, "GBPA not responding to update\n");
3138         return ret;
3139 }
3140
3141 static void arm_smmu_free_msis(void *data)
3142 {
3143         struct device *dev = data;
3144         platform_msi_domain_free_irqs(dev);
3145 }
3146
3147 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3148 {
3149         phys_addr_t doorbell;
3150         struct device *dev = msi_desc_to_dev(desc);
3151         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3152         phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
3153
3154         doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3155         doorbell &= MSI_CFG0_ADDR_MASK;
3156
3157         writeq_relaxed(doorbell, smmu->base + cfg[0]);
3158         writel_relaxed(msg->data, smmu->base + cfg[1]);
3159         writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3160 }
3161
3162 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3163 {
3164         int ret, nvec = ARM_SMMU_MAX_MSIS;
3165         struct device *dev = smmu->dev;
3166
3167         /* Clear the MSI address regs */
3168         writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3169         writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3170
3171         if (smmu->features & ARM_SMMU_FEAT_PRI)
3172                 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3173         else
3174                 nvec--;
3175
3176         if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3177                 return;
3178
3179         if (!dev->msi.domain) {
3180                 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3181                 return;
3182         }
3183
3184         /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3185         ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3186         if (ret) {
3187                 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3188                 return;
3189         }
3190
3191         smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
3192         smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
3193         smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
3194
3195         /* Add callback to free MSIs on teardown */
3196         devm_add_action(dev, arm_smmu_free_msis, dev);
3197 }
3198
3199 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3200 {
3201         int irq, ret;
3202
3203         arm_smmu_setup_msis(smmu);
3204
3205         /* Request interrupt lines */
3206         irq = smmu->evtq.q.irq;
3207         if (irq) {
3208                 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3209                                                 arm_smmu_evtq_thread,
3210                                                 IRQF_ONESHOT,
3211                                                 "arm-smmu-v3-evtq", smmu);
3212                 if (ret < 0)
3213                         dev_warn(smmu->dev, "failed to enable evtq irq\n");
3214         } else {
3215                 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3216         }
3217
3218         irq = smmu->gerr_irq;
3219         if (irq) {
3220                 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3221                                        0, "arm-smmu-v3-gerror", smmu);
3222                 if (ret < 0)
3223                         dev_warn(smmu->dev, "failed to enable gerror irq\n");
3224         } else {
3225                 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3226         }
3227
3228         if (smmu->features & ARM_SMMU_FEAT_PRI) {
3229                 irq = smmu->priq.q.irq;
3230                 if (irq) {
3231                         ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3232                                                         arm_smmu_priq_thread,
3233                                                         IRQF_ONESHOT,
3234                                                         "arm-smmu-v3-priq",
3235                                                         smmu);
3236                         if (ret < 0)
3237                                 dev_warn(smmu->dev,
3238                                          "failed to enable priq irq\n");
3239                 } else {
3240                         dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3241                 }
3242         }
3243 }
3244
3245 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3246 {
3247         int ret, irq;
3248         u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3249
3250         /* Disable IRQs first */
3251         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3252                                       ARM_SMMU_IRQ_CTRLACK);
3253         if (ret) {
3254                 dev_err(smmu->dev, "failed to disable irqs\n");
3255                 return ret;
3256         }
3257
3258         irq = smmu->combined_irq;
3259         if (irq) {
3260                 /*
3261                  * Cavium ThunderX2 implementation doesn't support unique irq
3262                  * lines. Use a single irq line for all the SMMUv3 interrupts.
3263                  */
3264                 ret = devm_request_threaded_irq(smmu->dev, irq,
3265                                         arm_smmu_combined_irq_handler,
3266                                         arm_smmu_combined_irq_thread,
3267                                         IRQF_ONESHOT,
3268                                         "arm-smmu-v3-combined-irq", smmu);
3269                 if (ret < 0)
3270                         dev_warn(smmu->dev, "failed to enable combined irq\n");
3271         } else
3272                 arm_smmu_setup_unique_irqs(smmu);
3273
3274         if (smmu->features & ARM_SMMU_FEAT_PRI)
3275                 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3276
3277         /* Enable interrupt generation on the SMMU */
3278         ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3279                                       ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3280         if (ret)
3281                 dev_warn(smmu->dev, "failed to enable irqs\n");
3282
3283         return 0;
3284 }
3285
3286 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3287 {
3288         int ret;
3289
3290         ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3291         if (ret)
3292                 dev_err(smmu->dev, "failed to clear cr0\n");
3293
3294         return ret;
3295 }
3296
3297 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
3298 {
3299         int ret;
3300         u32 reg, enables;
3301         struct arm_smmu_cmdq_ent cmd;
3302
3303         /* Clear CR0 and sync (disables SMMU and queue processing) */
3304         reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3305         if (reg & CR0_SMMUEN) {
3306                 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3307                 WARN_ON(is_kdump_kernel() && !disable_bypass);
3308                 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3309         }
3310
3311         ret = arm_smmu_device_disable(smmu);
3312         if (ret)
3313                 return ret;
3314
3315         /* CR1 (table and queue memory attributes) */
3316         reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3317               FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3318               FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3319               FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3320               FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3321               FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3322         writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3323
3324         /* CR2 (random crap) */
3325         reg = CR2_PTM | CR2_RECINVSID;
3326
3327         if (smmu->features & ARM_SMMU_FEAT_E2H)
3328                 reg |= CR2_E2H;
3329
3330         writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3331
3332         /* Stream table */
3333         writeq_relaxed(smmu->strtab_cfg.strtab_base,
3334                        smmu->base + ARM_SMMU_STRTAB_BASE);
3335         writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3336                        smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3337
3338         /* Command queue */
3339         writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3340         writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3341         writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3342
3343         enables = CR0_CMDQEN;
3344         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3345                                       ARM_SMMU_CR0ACK);
3346         if (ret) {
3347                 dev_err(smmu->dev, "failed to enable command queue\n");
3348                 return ret;
3349         }
3350
3351         /* Invalidate any cached configuration */
3352         cmd.opcode = CMDQ_OP_CFGI_ALL;
3353         arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3354
3355         /* Invalidate any stale TLB entries */
3356         if (smmu->features & ARM_SMMU_FEAT_HYP) {
3357                 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3358                 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3359         }
3360
3361         cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
3362         arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3363
3364         /* Event queue */
3365         writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3366         writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
3367         writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
3368
3369         enables |= CR0_EVTQEN;
3370         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3371                                       ARM_SMMU_CR0ACK);
3372         if (ret) {
3373                 dev_err(smmu->dev, "failed to enable event queue\n");
3374                 return ret;
3375         }
3376
3377         /* PRI queue */
3378         if (smmu->features & ARM_SMMU_FEAT_PRI) {
3379                 writeq_relaxed(smmu->priq.q.q_base,
3380                                smmu->base + ARM_SMMU_PRIQ_BASE);
3381                 writel_relaxed(smmu->priq.q.llq.prod,
3382                                smmu->page1 + ARM_SMMU_PRIQ_PROD);
3383                 writel_relaxed(smmu->priq.q.llq.cons,
3384                                smmu->page1 + ARM_SMMU_PRIQ_CONS);
3385
3386                 enables |= CR0_PRIQEN;
3387                 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3388                                               ARM_SMMU_CR0ACK);
3389                 if (ret) {
3390                         dev_err(smmu->dev, "failed to enable PRI queue\n");
3391                         return ret;
3392                 }
3393         }
3394
3395         if (smmu->features & ARM_SMMU_FEAT_ATS) {
3396                 enables |= CR0_ATSCHK;
3397                 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3398                                               ARM_SMMU_CR0ACK);
3399                 if (ret) {
3400                         dev_err(smmu->dev, "failed to enable ATS check\n");
3401                         return ret;
3402                 }
3403         }
3404
3405         ret = arm_smmu_setup_irqs(smmu);
3406         if (ret) {
3407                 dev_err(smmu->dev, "failed to setup irqs\n");
3408                 return ret;
3409         }
3410
3411         if (is_kdump_kernel())
3412                 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
3413
3414         /* Enable the SMMU interface, or ensure bypass */
3415         if (!bypass || disable_bypass) {
3416                 enables |= CR0_SMMUEN;
3417         } else {
3418                 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
3419                 if (ret)
3420                         return ret;
3421         }
3422         ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3423                                       ARM_SMMU_CR0ACK);
3424         if (ret) {
3425                 dev_err(smmu->dev, "failed to enable SMMU interface\n");
3426                 return ret;
3427         }
3428
3429         return 0;
3430 }
3431
3432 #define IIDR_IMPLEMENTER_ARM            0x43b
3433 #define IIDR_PRODUCTID_ARM_MMU_600      0x483
3434 #define IIDR_PRODUCTID_ARM_MMU_700      0x487
3435
3436 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
3437 {
3438         u32 reg;
3439         unsigned int implementer, productid, variant, revision;
3440
3441         reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
3442         implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
3443         productid = FIELD_GET(IIDR_PRODUCTID, reg);
3444         variant = FIELD_GET(IIDR_VARIANT, reg);
3445         revision = FIELD_GET(IIDR_REVISION, reg);
3446
3447         switch (implementer) {
3448         case IIDR_IMPLEMENTER_ARM:
3449                 switch (productid) {
3450                 case IIDR_PRODUCTID_ARM_MMU_600:
3451                         /* Arm erratum 1076982 */
3452                         if (variant == 0 && revision <= 2)
3453                                 smmu->features &= ~ARM_SMMU_FEAT_SEV;
3454                         /* Arm erratum 1209401 */
3455                         if (variant < 2)
3456                                 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
3457                         break;
3458                 case IIDR_PRODUCTID_ARM_MMU_700:
3459                         /* Arm erratum 2812531 */
3460                         smmu->features &= ~ARM_SMMU_FEAT_BTM;
3461                         smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
3462                         /* Arm errata 2268618, 2812531 */
3463                         smmu->features &= ~ARM_SMMU_FEAT_NESTING;
3464                         break;
3465                 }
3466                 break;
3467         }
3468 }
3469
3470 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
3471 {
3472         u32 reg;
3473         bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3474
3475         /* IDR0 */
3476         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3477
3478         /* 2-level structures */
3479         if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
3480                 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3481
3482         if (reg & IDR0_CD2L)
3483                 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3484
3485         /*
3486          * Translation table endianness.
3487          * We currently require the same endianness as the CPU, but this
3488          * could be changed later by adding a new IO_PGTABLE_QUIRK.
3489          */
3490         switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
3491         case IDR0_TTENDIAN_MIXED:
3492                 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3493                 break;
3494 #ifdef __BIG_ENDIAN
3495         case IDR0_TTENDIAN_BE:
3496                 smmu->features |= ARM_SMMU_FEAT_TT_BE;
3497                 break;
3498 #else
3499         case IDR0_TTENDIAN_LE:
3500                 smmu->features |= ARM_SMMU_FEAT_TT_LE;
3501                 break;
3502 #endif
3503         default:
3504                 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3505                 return -ENXIO;
3506         }
3507
3508         /* Boolean feature flags */
3509         if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
3510                 smmu->features |= ARM_SMMU_FEAT_PRI;
3511
3512         if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
3513                 smmu->features |= ARM_SMMU_FEAT_ATS;
3514
3515         if (reg & IDR0_SEV)
3516                 smmu->features |= ARM_SMMU_FEAT_SEV;
3517
3518         if (reg & IDR0_MSI) {
3519                 smmu->features |= ARM_SMMU_FEAT_MSI;
3520                 if (coherent && !disable_msipolling)
3521                         smmu->options |= ARM_SMMU_OPT_MSIPOLL;
3522         }
3523
3524         if (reg & IDR0_HYP) {
3525                 smmu->features |= ARM_SMMU_FEAT_HYP;
3526                 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
3527                         smmu->features |= ARM_SMMU_FEAT_E2H;
3528         }
3529
3530         /*
3531          * The coherency feature as set by FW is used in preference to the ID
3532          * register, but warn on mismatch.
3533          */
3534         if (!!(reg & IDR0_COHACC) != coherent)
3535                 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3536                          coherent ? "true" : "false");
3537
3538         switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
3539         case IDR0_STALL_MODEL_FORCE:
3540                 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3541                 fallthrough;
3542         case IDR0_STALL_MODEL_STALL:
3543                 smmu->features |= ARM_SMMU_FEAT_STALLS;
3544         }
3545
3546         if (reg & IDR0_S1P)
3547                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3548
3549         if (reg & IDR0_S2P)
3550                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3551
3552         if (!(reg & (IDR0_S1P | IDR0_S2P))) {
3553                 dev_err(smmu->dev, "no translation support!\n");
3554                 return -ENXIO;
3555         }
3556
3557         /* We only support the AArch64 table format at present */
3558         switch (FIELD_GET(IDR0_TTF, reg)) {
3559         case IDR0_TTF_AARCH32_64:
3560                 smmu->ias = 40;
3561                 fallthrough;
3562         case IDR0_TTF_AARCH64:
3563                 break;
3564         default:
3565                 dev_err(smmu->dev, "AArch64 table format not supported!\n");
3566                 return -ENXIO;
3567         }
3568
3569         /* ASID/VMID sizes */
3570         smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3571         smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3572
3573         /* IDR1 */
3574         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3575         if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
3576                 dev_err(smmu->dev, "embedded implementation not supported\n");
3577                 return -ENXIO;
3578         }
3579
3580         /* Queue sizes, capped to ensure natural alignment */
3581         smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3582                                              FIELD_GET(IDR1_CMDQS, reg));
3583         if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3584                 /*
3585                  * We don't support splitting up batches, so one batch of
3586                  * commands plus an extra sync needs to fit inside the command
3587                  * queue. There's also no way we can handle the weird alignment
3588                  * restrictions on the base pointer for a unit-length queue.
3589                  */
3590                 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3591                         CMDQ_BATCH_ENTRIES);
3592                 return -ENXIO;
3593         }
3594
3595         smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3596                                              FIELD_GET(IDR1_EVTQS, reg));
3597         smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3598                                              FIELD_GET(IDR1_PRIQS, reg));
3599
3600         /* SID/SSID sizes */
3601         smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3602         smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3603         smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
3604
3605         /*
3606          * If the SMMU supports fewer bits than would fill a single L2 stream
3607          * table, use a linear table instead.
3608          */
3609         if (smmu->sid_bits <= STRTAB_SPLIT)
3610                 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3611
3612         /* IDR3 */
3613         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
3614         if (FIELD_GET(IDR3_RIL, reg))
3615                 smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
3616
3617         /* IDR5 */
3618         reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3619
3620         /* Maximum number of outstanding stalls */
3621         smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3622
3623         /* Page sizes */
3624         if (reg & IDR5_GRAN64K)
3625                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3626         if (reg & IDR5_GRAN16K)
3627                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3628         if (reg & IDR5_GRAN4K)
3629                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3630
3631         /* Input address size */
3632         if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
3633                 smmu->features |= ARM_SMMU_FEAT_VAX;
3634
3635         /* Output address size */
3636         switch (FIELD_GET(IDR5_OAS, reg)) {
3637         case IDR5_OAS_32_BIT:
3638                 smmu->oas = 32;
3639                 break;
3640         case IDR5_OAS_36_BIT:
3641                 smmu->oas = 36;
3642                 break;
3643         case IDR5_OAS_40_BIT:
3644                 smmu->oas = 40;
3645                 break;
3646         case IDR5_OAS_42_BIT:
3647                 smmu->oas = 42;
3648                 break;
3649         case IDR5_OAS_44_BIT:
3650                 smmu->oas = 44;
3651                 break;
3652         case IDR5_OAS_52_BIT:
3653                 smmu->oas = 52;
3654                 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
3655                 break;
3656         default:
3657                 dev_info(smmu->dev,
3658                         "unknown output address size. Truncating to 48-bit\n");
3659                 fallthrough;
3660         case IDR5_OAS_48_BIT:
3661                 smmu->oas = 48;
3662         }
3663
3664         if (arm_smmu_ops.pgsize_bitmap == -1UL)
3665                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3666         else
3667                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3668
3669         /* Set the DMA mask for our table walker */
3670         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3671                 dev_warn(smmu->dev,
3672                          "failed to set DMA mask for table walker\n");
3673
3674         smmu->ias = max(smmu->ias, smmu->oas);
3675
3676         if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
3677             (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
3678                 smmu->features |= ARM_SMMU_FEAT_NESTING;
3679
3680         arm_smmu_device_iidr_probe(smmu);
3681
3682         if (arm_smmu_sva_supported(smmu))
3683                 smmu->features |= ARM_SMMU_FEAT_SVA;
3684
3685         dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3686                  smmu->ias, smmu->oas, smmu->features);
3687         return 0;
3688 }
3689
3690 #ifdef CONFIG_ACPI
3691 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
3692 {
3693         switch (model) {
3694         case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
3695                 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3696                 break;
3697         case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3698                 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3699                 break;
3700         }
3701
3702         dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3703 }
3704
3705 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3706                                       struct arm_smmu_device *smmu)
3707 {
3708         struct acpi_iort_smmu_v3 *iort_smmu;
3709         struct device *dev = smmu->dev;
3710         struct acpi_iort_node *node;
3711
3712         node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3713
3714         /* Retrieve SMMUv3 specific data */
3715         iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3716
3717         acpi_smmu_get_options(iort_smmu->model, smmu);
3718
3719         if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3720                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3721
3722         return 0;
3723 }
3724 #else
3725 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3726                                              struct arm_smmu_device *smmu)
3727 {
3728         return -ENODEV;
3729 }
3730 #endif
3731
3732 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3733                                     struct arm_smmu_device *smmu)
3734 {
3735         struct device *dev = &pdev->dev;
3736         u32 cells;
3737         int ret = -EINVAL;
3738
3739         if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3740                 dev_err(dev, "missing #iommu-cells property\n");
3741         else if (cells != 1)
3742                 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3743         else
3744                 ret = 0;
3745
3746         parse_driver_options(smmu);
3747
3748         if (of_dma_is_coherent(dev->of_node))
3749                 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3750
3751         return ret;
3752 }
3753
3754 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3755 {
3756         if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3757                 return SZ_64K;
3758         else
3759                 return SZ_128K;
3760 }
3761
3762 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
3763                                       resource_size_t size)
3764 {
3765         struct resource res = DEFINE_RES_MEM(start, size);
3766
3767         return devm_ioremap_resource(dev, &res);
3768 }
3769
3770 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
3771 {
3772         struct list_head rmr_list;
3773         struct iommu_resv_region *e;
3774
3775         INIT_LIST_HEAD(&rmr_list);
3776         iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3777
3778         list_for_each_entry(e, &rmr_list, list) {
3779                 __le64 *step;
3780                 struct iommu_iort_rmr_data *rmr;
3781                 int ret, i;
3782
3783                 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
3784                 for (i = 0; i < rmr->num_sids; i++) {
3785                         ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
3786                         if (ret) {
3787                                 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
3788                                         rmr->sids[i]);
3789                                 continue;
3790                         }
3791
3792                         step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]);
3793                         arm_smmu_init_bypass_stes(step, 1, true);
3794                 }
3795         }
3796
3797         iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3798 }
3799
3800 static int arm_smmu_device_probe(struct platform_device *pdev)
3801 {
3802         int irq, ret;
3803         struct resource *res;
3804         resource_size_t ioaddr;
3805         struct arm_smmu_device *smmu;
3806         struct device *dev = &pdev->dev;
3807         bool bypass;
3808
3809         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3810         if (!smmu)
3811                 return -ENOMEM;
3812         smmu->dev = dev;
3813
3814         if (dev->of_node) {
3815                 ret = arm_smmu_device_dt_probe(pdev, smmu);
3816         } else {
3817                 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3818                 if (ret == -ENODEV)
3819                         return ret;
3820         }
3821
3822         /* Set bypass mode according to firmware probing result */
3823         bypass = !!ret;
3824
3825         /* Base address */
3826         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3827         if (!res)
3828                 return -EINVAL;
3829         if (resource_size(res) < arm_smmu_resource_size(smmu)) {
3830                 dev_err(dev, "MMIO region too small (%pr)\n", res);
3831                 return -EINVAL;
3832         }
3833         ioaddr = res->start;
3834
3835         /*
3836          * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
3837          * the PMCG registers which are reserved by the PMU driver.
3838          */
3839         smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
3840         if (IS_ERR(smmu->base))
3841                 return PTR_ERR(smmu->base);
3842
3843         if (arm_smmu_resource_size(smmu) > SZ_64K) {
3844                 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
3845                                                ARM_SMMU_REG_SZ);
3846                 if (IS_ERR(smmu->page1))
3847                         return PTR_ERR(smmu->page1);
3848         } else {
3849                 smmu->page1 = smmu->base;
3850         }
3851
3852         /* Interrupt lines */
3853
3854         irq = platform_get_irq_byname_optional(pdev, "combined");
3855         if (irq > 0)
3856                 smmu->combined_irq = irq;
3857         else {
3858                 irq = platform_get_irq_byname_optional(pdev, "eventq");
3859                 if (irq > 0)
3860                         smmu->evtq.q.irq = irq;
3861
3862                 irq = platform_get_irq_byname_optional(pdev, "priq");
3863                 if (irq > 0)
3864                         smmu->priq.q.irq = irq;
3865
3866                 irq = platform_get_irq_byname_optional(pdev, "gerror");
3867                 if (irq > 0)
3868                         smmu->gerr_irq = irq;
3869         }
3870         /* Probe the h/w */
3871         ret = arm_smmu_device_hw_probe(smmu);
3872         if (ret)
3873                 return ret;
3874
3875         /* Initialise in-memory data structures */
3876         ret = arm_smmu_init_structures(smmu);
3877         if (ret)
3878                 return ret;
3879
3880         /* Record our private device structure */
3881         platform_set_drvdata(pdev, smmu);
3882
3883         /* Check for RMRs and install bypass STEs if any */
3884         arm_smmu_rmr_install_bypass_ste(smmu);
3885
3886         /* Reset the device */
3887         ret = arm_smmu_device_reset(smmu, bypass);
3888         if (ret)
3889                 return ret;
3890
3891         /* And we're up. Go go go! */
3892         ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3893                                      "smmu3.%pa", &ioaddr);
3894         if (ret)
3895                 return ret;
3896
3897         ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
3898         if (ret) {
3899                 dev_err(dev, "Failed to register iommu\n");
3900                 iommu_device_sysfs_remove(&smmu->iommu);
3901                 return ret;
3902         }
3903
3904         return 0;
3905 }
3906
3907 static void arm_smmu_device_remove(struct platform_device *pdev)
3908 {
3909         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3910
3911         iommu_device_unregister(&smmu->iommu);
3912         iommu_device_sysfs_remove(&smmu->iommu);
3913         arm_smmu_device_disable(smmu);
3914         iopf_queue_free(smmu->evtq.iopf);
3915         ida_destroy(&smmu->vmid_map);
3916 }
3917
3918 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3919 {
3920         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3921
3922         arm_smmu_device_disable(smmu);
3923 }
3924
3925 static const struct of_device_id arm_smmu_of_match[] = {
3926         { .compatible = "arm,smmu-v3", },
3927         { },
3928 };
3929 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
3930
3931 static void arm_smmu_driver_unregister(struct platform_driver *drv)
3932 {
3933         arm_smmu_sva_notifier_synchronize();
3934         platform_driver_unregister(drv);
3935 }
3936
3937 static struct platform_driver arm_smmu_driver = {
3938         .driver = {
3939                 .name                   = "arm-smmu-v3",
3940                 .of_match_table         = arm_smmu_of_match,
3941                 .suppress_bind_attrs    = true,
3942         },
3943         .probe  = arm_smmu_device_probe,
3944         .remove_new = arm_smmu_device_remove,
3945         .shutdown = arm_smmu_device_shutdown,
3946 };
3947 module_driver(arm_smmu_driver, platform_driver_register,
3948               arm_smmu_driver_unregister);
3949
3950 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
3951 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
3952 MODULE_ALIAS("platform:arm-smmu-v3");
3953 MODULE_LICENSE("GPL v2");