drivers/iommu/sun50i-iommu.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
   3 // Copyright (C) 2019-2020, Cerno
   4
   5 #include <linux/bitfield.h>
   6 #include <linux/bug.h>
   7 #include <linux/clk.h>
   8 #include <linux/device.h>
   9 #include <linux/dma-direction.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/err.h>
  12 #include <linux/errno.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/iommu.h>
  15 #include <linux/iopoll.h>
  16 #include <linux/ioport.h>
  17 #include <linux/log2.h>
  18 #include <linux/module.h>
  19 #include <linux/of_platform.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/pm.h>
  22 #include <linux/pm_runtime.h>
  23 #include <linux/reset.h>
  24 #include <linux/sizes.h>
  25 #include <linux/slab.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/types.h>
  28
  29 #define IOMMU_RESET_REG                 0x010
  30 #define IOMMU_ENABLE_REG                0x020
  31 #define IOMMU_ENABLE_ENABLE                     BIT(0)
  32
  33 #define IOMMU_BYPASS_REG                0x030
  34 #define IOMMU_AUTO_GATING_REG           0x040
  35 #define IOMMU_AUTO_GATING_ENABLE                BIT(0)
  36
  37 #define IOMMU_WBUF_CTRL_REG             0x044
  38 #define IOMMU_OOO_CTRL_REG              0x048
  39 #define IOMMU_4KB_BDY_PRT_CTRL_REG      0x04c
  40 #define IOMMU_TTB_REG                   0x050
  41 #define IOMMU_TLB_ENABLE_REG            0x060
  42 #define IOMMU_TLB_PREFETCH_REG          0x070
  43 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m)     BIT(m)
  44
  45 #define IOMMU_TLB_FLUSH_REG             0x080
  46 #define IOMMU_TLB_FLUSH_PTW_CACHE               BIT(17)
  47 #define IOMMU_TLB_FLUSH_MACRO_TLB               BIT(16)
  48 #define IOMMU_TLB_FLUSH_MICRO_TLB(i)            (BIT(i) & GENMASK(5, 0))
  49
  50 #define IOMMU_TLB_IVLD_ADDR_REG         0x090
  51 #define IOMMU_TLB_IVLD_ADDR_MASK_REG    0x094
  52 #define IOMMU_TLB_IVLD_ENABLE_REG       0x098
  53 #define IOMMU_TLB_IVLD_ENABLE_ENABLE            BIT(0)
  54
  55 #define IOMMU_PC_IVLD_ADDR_REG          0x0a0
  56 #define IOMMU_PC_IVLD_ENABLE_REG        0x0a8
  57 #define IOMMU_PC_IVLD_ENABLE_ENABLE             BIT(0)
  58
  59 #define IOMMU_DM_AUT_CTRL_REG(d)        (0x0b0 + ((d) / 2) * 4)
  60 #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m)      (1 << (((d & 1) * 16) + ((m) * 2)))
  61 #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m)      (1 << (((d & 1) * 16) + ((m) * 2) + 1))
  62
  63 #define IOMMU_DM_AUT_OVWT_REG           0x0d0
  64 #define IOMMU_INT_ENABLE_REG            0x100
  65 #define IOMMU_INT_CLR_REG               0x104
  66 #define IOMMU_INT_STA_REG               0x108
  67 #define IOMMU_INT_ERR_ADDR_REG(i)       (0x110 + (i) * 4)
  68 #define IOMMU_INT_ERR_ADDR_L1_REG       0x130
  69 #define IOMMU_INT_ERR_ADDR_L2_REG       0x134
  70 #define IOMMU_INT_ERR_DATA_REG(i)       (0x150 + (i) * 4)
  71 #define IOMMU_L1PG_INT_REG              0x0180
  72 #define IOMMU_L2PG_INT_REG              0x0184
  73
  74 #define IOMMU_INT_INVALID_L2PG                  BIT(17)
  75 #define IOMMU_INT_INVALID_L1PG                  BIT(16)
  76 #define IOMMU_INT_MASTER_PERMISSION(m)          BIT(m)
  77 #define IOMMU_INT_MASTER_MASK                   (IOMMU_INT_MASTER_PERMISSION(0) | \
  78                                                  IOMMU_INT_MASTER_PERMISSION(1) | \
  79                                                  IOMMU_INT_MASTER_PERMISSION(2) | \
  80                                                  IOMMU_INT_MASTER_PERMISSION(3) | \
  81                                                  IOMMU_INT_MASTER_PERMISSION(4) | \
  82                                                  IOMMU_INT_MASTER_PERMISSION(5))
  83 #define IOMMU_INT_MASK                          (IOMMU_INT_INVALID_L1PG | \
  84                                                  IOMMU_INT_INVALID_L2PG | \
  85                                                  IOMMU_INT_MASTER_MASK)
  86
  87 #define PT_ENTRY_SIZE                   sizeof(u32)
  88
  89 #define NUM_DT_ENTRIES                  4096
  90 #define DT_SIZE                         (NUM_DT_ENTRIES * PT_ENTRY_SIZE)
  91
  92 #define NUM_PT_ENTRIES                  256
  93 #define PT_SIZE                         (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
  94
  95 struct sun50i_iommu {
  96         struct iommu_device iommu;
  97
  98         /* Lock to modify the IOMMU registers */
  99         spinlock_t iommu_lock;
 100
 101         struct device *dev;
 102         void __iomem *base;
 103         struct reset_control *reset;
 104         struct clk *clk;
 105
 106         struct iommu_domain *domain;
 107         struct iommu_group *group;
 108         struct kmem_cache *pt_pool;
 109 };
 110
 111 struct sun50i_iommu_domain {
 112         struct iommu_domain domain;
 113
 114         /* Number of devices attached to the domain */
 115         refcount_t refcnt;
 116
 117         /* L1 Page Table */
 118         u32 *dt;
 119         dma_addr_t dt_dma;
 120
 121         struct sun50i_iommu *iommu;
 122 };
 123
 124 static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
 125 {
 126         return container_of(domain, struct sun50i_iommu_domain, domain);
 127 }
 128
 129 static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
 130 {
 131         return dev_iommu_priv_get(dev);
 132 }
 133
 134 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
 135 {
 136         return readl(iommu->base + offset);
 137 }
 138
 139 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
 140 {
 141         writel(value, iommu->base + offset);
 142 }
 143
 144 /*
 145  * The Allwinner H6 IOMMU uses a 2-level page table.
 146  *
 147  * The first level is the usual Directory Table (DT), that consists of
 148  * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
 149  * Table (PT).
 150  *
 151  * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
 152  * pointing to a 4kB page of physical memory.
 153  *
 154  * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
 155  * register that contains its physical address.
 156  */
 157
 158 #define SUN50I_IOVA_DTE_MASK    GENMASK(31, 20)
 159 #define SUN50I_IOVA_PTE_MASK    GENMASK(19, 12)
 160 #define SUN50I_IOVA_PAGE_MASK   GENMASK(11, 0)
 161
 162 static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
 163 {
 164         return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
 165 }
 166
 167 static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
 168 {
 169         return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
 170 }
 171
 172 static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
 173 {
 174         return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
 175 }
 176
 177 /*
 178  * Each Directory Table Entry has a Page Table address and a valid
 179  * bit:
 180
 181  * +---------------------+-----------+-+
 182  * | PT address          | Reserved  |V|
 183  * +---------------------+-----------+-+
 184  *  31:10 - Page Table address
 185  *   9:2  - Reserved
 186  *   1:0  - 1 if the entry is valid
 187  */
 188
 189 #define SUN50I_DTE_PT_ADDRESS_MASK      GENMASK(31, 10)
 190 #define SUN50I_DTE_PT_ATTRS             GENMASK(1, 0)
 191 #define SUN50I_DTE_PT_VALID             1
 192
 193 static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
 194 {
 195         return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
 196 }
 197
 198 static bool sun50i_dte_is_pt_valid(u32 dte)
 199 {
 200         return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
 201 }
 202
 203 static u32 sun50i_mk_dte(dma_addr_t pt_dma)
 204 {
 205         return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
 206 }
 207
 208 /*
 209  * Each PTE has a Page address, an authority index and a valid bit:
 210  *
 211  * +----------------+-----+-----+-----+---+-----+
 212  * | Page address   | Rsv | ACI | Rsv | V | Rsv |
 213  * +----------------+-----+-----+-----+---+-----+
 214  *  31:12 - Page address
 215  *  11:8  - Reserved
 216  *   7:4  - Authority Control Index
 217  *   3:2  - Reserved
 218  *     1  - 1 if the entry is valid
 219  *     0  - Reserved
 220  *
 221  * The way permissions work is that the IOMMU has 16 "domains" that
 222  * can be configured to give each masters either read or write
 223  * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain
 224  * 0 seems like the default domain, and its permissions in the
 225  * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
 226  * useful to enforce any particular permission.
 227  *
 228  * Each page entry will then have a reference to the domain they are
 229  * affected to, so that we can actually enforce them on a per-page
 230  * basis.
 231  *
 232  * In order to make it work with the IOMMU framework, we will be using
 233  * 4 different domains, starting at 1: RD_WR, RD, WR and NONE
 234  * depending on the permission we want to enforce. Each domain will
 235  * have each master setup in the same way, since the IOMMU framework
 236  * doesn't seem to restrict page access on a per-device basis. And
 237  * then we will use the relevant domain index when generating the page
 238  * table entry depending on the permissions we want to be enforced.
 239  */
 240
 241 enum sun50i_iommu_aci {
 242         SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
 243         SUN50I_IOMMU_ACI_NONE,
 244         SUN50I_IOMMU_ACI_RD,
 245         SUN50I_IOMMU_ACI_WR,
 246         SUN50I_IOMMU_ACI_RD_WR,
 247 };
 248
 249 #define SUN50I_PTE_PAGE_ADDRESS_MASK    GENMASK(31, 12)
 250 #define SUN50I_PTE_ACI_MASK             GENMASK(7, 4)
 251 #define SUN50I_PTE_PAGE_VALID           BIT(1)
 252
 253 static phys_addr_t sun50i_pte_get_page_address(u32 pte)
 254 {
 255         return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
 256 }
 257
 258 static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
 259 {
 260         return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
 261 }
 262
 263 static bool sun50i_pte_is_page_valid(u32 pte)
 264 {
 265         return pte & SUN50I_PTE_PAGE_VALID;
 266 }
 267
 268 static u32 sun50i_mk_pte(phys_addr_t page, int prot)
 269 {
 270         enum sun50i_iommu_aci aci;
 271         u32 flags = 0;
 272
 273         if (prot & (IOMMU_READ | IOMMU_WRITE))
 274                 aci = SUN50I_IOMMU_ACI_RD_WR;
 275         else if (prot & IOMMU_READ)
 276                 aci = SUN50I_IOMMU_ACI_RD;
 277         else if (prot & IOMMU_WRITE)
 278                 aci = SUN50I_IOMMU_ACI_WR;
 279         else
 280                 aci = SUN50I_IOMMU_ACI_NONE;
 281
 282         flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
 283         page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
 284         return page | flags | SUN50I_PTE_PAGE_VALID;
 285 }
 286
 287 static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
 288                                void *vaddr, unsigned int count)
 289 {
 290         struct sun50i_iommu *iommu = sun50i_domain->iommu;
 291         dma_addr_t dma = virt_to_phys(vaddr);
 292         size_t size = count * PT_ENTRY_SIZE;
 293
 294         dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
 295 }
 296
 297 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
 298 {
 299         u32 reg;
 300         int ret;
 301
 302         assert_spin_locked(&iommu->iommu_lock);
 303
 304         iommu_write(iommu,
 305                     IOMMU_TLB_FLUSH_REG,
 306                     IOMMU_TLB_FLUSH_PTW_CACHE |
 307                     IOMMU_TLB_FLUSH_MACRO_TLB |
 308                     IOMMU_TLB_FLUSH_MICRO_TLB(5) |
 309                     IOMMU_TLB_FLUSH_MICRO_TLB(4) |
 310                     IOMMU_TLB_FLUSH_MICRO_TLB(3) |
 311                     IOMMU_TLB_FLUSH_MICRO_TLB(2) |
 312                     IOMMU_TLB_FLUSH_MICRO_TLB(1) |
 313                     IOMMU_TLB_FLUSH_MICRO_TLB(0));
 314
 315         ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,
 316                                         reg, !reg,
 317                                         1, 2000);
 318         if (ret)
 319                 dev_warn(iommu->dev, "TLB Flush timed out!\n");
 320
 321         return ret;
 322 }
 323
 324 static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
 325 {
 326         struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
 327         struct sun50i_iommu *iommu = sun50i_domain->iommu;
 328         unsigned long flags;
 329
 330         /*
 331          * At boot, we'll have a first call into .flush_iotlb_all right after
 332          * .probe_device, and since we link our (single) domain to our iommu in
 333          * the .attach_device callback, we don't have that pointer set.
 334          *
 335          * It shouldn't really be any trouble to ignore it though since we flush
 336          * all caches as part of the device powerup.
 337          */
 338         if (!iommu)
 339                 return;
 340
 341         spin_lock_irqsave(&iommu->iommu_lock, flags);
 342         sun50i_iommu_flush_all_tlb(iommu);
 343         spin_unlock_irqrestore(&iommu->iommu_lock, flags);
 344 }
 345
 346 static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
 347                                     struct iommu_iotlb_gather *gather)
 348 {
 349         sun50i_iommu_flush_iotlb_all(domain);
 350 }
 351
 352 static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
 353 {
 354         struct sun50i_iommu_domain *sun50i_domain;
 355         unsigned long flags;
 356         int ret;
 357
 358         if (!iommu->domain)
 359                 return 0;
 360
 361         sun50i_domain = to_sun50i_domain(iommu->domain);
 362
 363         ret = reset_control_deassert(iommu->reset);
 364         if (ret)
 365                 return ret;
 366
 367         ret = clk_prepare_enable(iommu->clk);
 368         if (ret)
 369                 goto err_reset_assert;
 370
 371         spin_lock_irqsave(&iommu->iommu_lock, flags);
 372
 373         iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
 374         iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
 375                     IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
 376                     IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
 377                     IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
 378                     IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
 379                     IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
 380                     IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
 381         iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
 382         iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
 383                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
 384                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
 385                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
 386                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
 387                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
 388                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
 389                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
 390                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
 391                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
 392                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
 393                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
 394                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));
 395
 396         iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
 397                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
 398                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
 399                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
 400                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
 401                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
 402                     IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));
 403
 404         iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
 405                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
 406                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
 407                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
 408                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
 409                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
 410                     IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));
 411
 412         ret = sun50i_iommu_flush_all_tlb(iommu);
 413         if (ret) {
 414                 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
 415                 goto err_clk_disable;
 416         }
 417
 418         iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
 419         iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);
 420
 421         spin_unlock_irqrestore(&iommu->iommu_lock, flags);
 422
 423         return 0;
 424
 425 err_clk_disable:
 426         clk_disable_unprepare(iommu->clk);
 427
 428 err_reset_assert:
 429         reset_control_assert(iommu->reset);
 430
 431         return ret;
 432 }
 433
 434 static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
 435 {
 436         unsigned long flags;
 437
 438         spin_lock_irqsave(&iommu->iommu_lock, flags);
 439
 440         iommu_write(iommu, IOMMU_ENABLE_REG, 0);
 441         iommu_write(iommu, IOMMU_TTB_REG, 0);
 442
 443         spin_unlock_irqrestore(&iommu->iommu_lock, flags);
 444
 445         clk_disable_unprepare(iommu->clk);
 446         reset_control_assert(iommu->reset);
 447 }
 448
 449 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
 450                                            gfp_t gfp)
 451 {
 452         dma_addr_t pt_dma;
 453         u32 *page_table;
 454
 455         page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
 456         if (!page_table)
 457                 return ERR_PTR(-ENOMEM);
 458
 459         pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
 460         if (dma_mapping_error(iommu->dev, pt_dma)) {
 461                 dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
 462                 kmem_cache_free(iommu->pt_pool, page_table);
 463                 return ERR_PTR(-ENOMEM);
 464         }
 465
 466         /* We rely on the physical address and DMA address being the same */
 467         WARN_ON(pt_dma != virt_to_phys(page_table));
 468
 469         return page_table;
 470 }
 471
 472 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
 473                                          u32 *page_table)
 474 {
 475         phys_addr_t pt_phys = virt_to_phys(page_table);
 476
 477         dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
 478         kmem_cache_free(iommu->pt_pool, page_table);
 479 }
 480
 481 static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
 482                                       dma_addr_t iova, gfp_t gfp)
 483 {
 484         struct sun50i_iommu *iommu = sun50i_domain->iommu;
 485         u32 *page_table;
 486         u32 *dte_addr;
 487         u32 old_dte;
 488         u32 dte;
 489
 490         dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
 491         dte = *dte_addr;
 492         if (sun50i_dte_is_pt_valid(dte)) {
 493                 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
 494                 return (u32 *)phys_to_virt(pt_phys);
 495         }
 496
 497         page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
 498         if (IS_ERR(page_table))
 499                 return page_table;
 500
 501         dte = sun50i_mk_dte(virt_to_phys(page_table));
 502         old_dte = cmpxchg(dte_addr, 0, dte);
 503         if (old_dte) {
 504                 phys_addr_t installed_pt_phys =
 505                         sun50i_dte_get_pt_address(old_dte);
 506                 u32 *installed_pt = phys_to_virt(installed_pt_phys);
 507                 u32 *drop_pt = page_table;
 508
 509                 page_table = installed_pt;
 510                 dte = old_dte;
 511                 sun50i_iommu_free_page_table(iommu, drop_pt);
 512         }
 513
 514         sun50i_table_flush(sun50i_domain, page_table, PT_SIZE);
 515         sun50i_table_flush(sun50i_domain, dte_addr, 1);
 516
 517         return page_table;
 518 }
 519
 520 static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
 521                             phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 522 {
 523         struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
 524         struct sun50i_iommu *iommu = sun50i_domain->iommu;
 525         u32 pte_index;
 526         u32 *page_table, *pte_addr;
 527         int ret = 0;
 528
 529         page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
 530         if (IS_ERR(page_table)) {
 531                 ret = PTR_ERR(page_table);
 532                 goto out;
 533         }
 534
 535         pte_index = sun50i_iova_get_pte_index(iova);
 536         pte_addr = &page_table[pte_index];
 537         if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
 538                 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
 539                 dev_err(iommu->dev,
 540                         "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
 541                         &iova, &page_phys, &paddr, prot);
 542                 ret = -EBUSY;
 543                 goto out;
 544         }
 545
 546         *pte_addr = sun50i_mk_pte(paddr, prot);
 547         sun50i_table_flush(sun50i_domain, pte_addr, 1);
 548
 549 out:
 550         return ret;
 551 }
 552
 553 static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
 554                                  size_t size, struct iommu_iotlb_gather *gather)
 555 {
 556         struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
 557         phys_addr_t pt_phys;
 558         u32 *pte_addr;
 559         u32 dte;
 560
 561         dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
 562         if (!sun50i_dte_is_pt_valid(dte))
 563                 return 0;
 564
 565         pt_phys = sun50i_dte_get_pt_address(dte);
 566         pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);
 567
 568         if (!sun50i_pte_is_page_valid(*pte_addr))
 569                 return 0;
 570
 571         memset(pte_addr, 0, sizeof(*pte_addr));
 572         sun50i_table_flush(sun50i_domain, pte_addr, 1);
 573
 574         return SZ_4K;
 575 }
 576
 577 static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
 578                                              dma_addr_t iova)
 579 {
 580         struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
 581         phys_addr_t pt_phys;
 582         u32 *page_table;
 583         u32 dte, pte;
 584
 585         dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
 586         if (!sun50i_dte_is_pt_valid(dte))
 587                 return 0;
 588
 589         pt_phys = sun50i_dte_get_pt_address(dte);
 590         page_table = (u32 *)phys_to_virt(pt_phys);
 591         pte = page_table[sun50i_iova_get_pte_index(iova)];
 592         if (!sun50i_pte_is_page_valid(pte))
 593                 return 0;
 594
 595         return sun50i_pte_get_page_address(pte) +
 596                 sun50i_iova_get_page_offset(iova);
 597 }
 598
 599 static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
 600 {
 601         struct sun50i_iommu_domain *sun50i_domain;
 602
 603         if (type != IOMMU_DOMAIN_DMA &&
 604             type != IOMMU_DOMAIN_IDENTITY &&
 605             type != IOMMU_DOMAIN_UNMANAGED)
 606                 return NULL;
 607
 608         sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
 609         if (!sun50i_domain)
 610                 return NULL;
 611
 612         sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 613                                                     get_order(DT_SIZE));
 614         if (!sun50i_domain->dt)
 615                 goto err_free_domain;
 616
 617         refcount_set(&sun50i_domain->refcnt, 1);
 618
 619         sun50i_domain->domain.geometry.aperture_start = 0;
 620         sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
 621         sun50i_domain->domain.geometry.force_aperture = true;
 622
 623         return &sun50i_domain->domain;
 624
 625 err_free_domain:
 626         kfree(sun50i_domain);
 627
 628         return NULL;
 629 }
 630
 631 static void sun50i_iommu_domain_free(struct iommu_domain *domain)
 632 {
 633         struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
 634
 635         free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE));
 636         sun50i_domain->dt = NULL;
 637
 638         kfree(sun50i_domain);
 639 }
 640
 641 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
 642                                       struct sun50i_iommu_domain *sun50i_domain)
 643 {
 644         iommu->domain = &sun50i_domain->domain;
 645         sun50i_domain->iommu = iommu;
 646
 647         sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
 648                                                DT_SIZE, DMA_TO_DEVICE);
 649         if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
 650                 dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
 651                 return -ENOMEM;
 652         }
 653
 654         return sun50i_iommu_enable(iommu);
 655 }
 656
 657 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
 658                                        struct sun50i_iommu_domain *sun50i_domain)
 659 {
 660         unsigned int i;
 661
 662         for (i = 0; i < NUM_DT_ENTRIES; i++) {
 663                 phys_addr_t pt_phys;
 664                 u32 *page_table;
 665                 u32 *dte_addr;
 666                 u32 dte;
 667
 668                 dte_addr = &sun50i_domain->dt[i];
 669                 dte = *dte_addr;
 670                 if (!sun50i_dte_is_pt_valid(dte))
 671                         continue;
 672
 673                 memset(dte_addr, 0, sizeof(*dte_addr));
 674                 sun50i_table_flush(sun50i_domain, dte_addr, 1);
 675
 676                 pt_phys = sun50i_dte_get_pt_address(dte);
 677                 page_table = phys_to_virt(pt_phys);
 678                 sun50i_iommu_free_page_table(iommu, page_table);
 679         }
 680
 681
 682         sun50i_iommu_disable(iommu);
 683
 684         dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
 685                          DT_SIZE, DMA_TO_DEVICE);
 686
 687         iommu->domain = NULL;
 688 }
 689
 690 static void sun50i_iommu_detach_device(struct iommu_domain *domain,
 691                                        struct device *dev)
 692 {
 693         struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
 694         struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
 695
 696         dev_dbg(dev, "Detaching from IOMMU domain\n");
 697
 698         if (iommu->domain != domain)
 699                 return;
 700
 701         if (refcount_dec_and_test(&sun50i_domain->refcnt))
 702                 sun50i_iommu_detach_domain(iommu, sun50i_domain);
 703 }
 704
 705 static int sun50i_iommu_attach_device(struct iommu_domain *domain,
 706                                       struct device *dev)
 707 {
 708         struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
 709         struct sun50i_iommu *iommu;
 710
 711         iommu = sun50i_iommu_from_dev(dev);
 712         if (!iommu)
 713                 return -ENODEV;
 714
 715         dev_dbg(dev, "Attaching to IOMMU domain\n");
 716
 717         refcount_inc(&sun50i_domain->refcnt);
 718
 719         if (iommu->domain == domain)
 720                 return 0;
 721
 722         if (iommu->domain)
 723                 sun50i_iommu_detach_device(iommu->domain, dev);
 724
 725         sun50i_iommu_attach_domain(iommu, sun50i_domain);
 726
 727         return 0;
 728 }
 729
 730 static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
 731 {
 732         struct sun50i_iommu *iommu;
 733
 734         iommu = sun50i_iommu_from_dev(dev);
 735         if (!iommu)
 736                 return ERR_PTR(-ENODEV);
 737
 738         return &iommu->iommu;
 739 }
 740
 741 static struct iommu_group *sun50i_iommu_device_group(struct device *dev)
 742 {
 743         struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev);
 744
 745         return iommu_group_ref_get(iommu->group);
 746 }
 747
 748 static int sun50i_iommu_of_xlate(struct device *dev,
 749                                  struct of_phandle_args *args)
 750 {
 751         struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
 752         unsigned id = args->args[0];
 753
 754         dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));
 755
 756         return iommu_fwspec_add_ids(dev, &id, 1);
 757 }
 758
 759 static const struct iommu_ops sun50i_iommu_ops = {
 760         .pgsize_bitmap  = SZ_4K,
 761         .device_group   = sun50i_iommu_device_group,
 762         .domain_alloc   = sun50i_iommu_domain_alloc,
 763         .of_xlate       = sun50i_iommu_of_xlate,
 764         .probe_device   = sun50i_iommu_probe_device,
 765         .default_domain_ops = &(const struct iommu_domain_ops) {
 766                 .attach_dev     = sun50i_iommu_attach_device,
 767                 .detach_dev     = sun50i_iommu_detach_device,
 768                 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
 769                 .iotlb_sync     = sun50i_iommu_iotlb_sync,
 770                 .iova_to_phys   = sun50i_iommu_iova_to_phys,
 771                 .map            = sun50i_iommu_map,
 772                 .unmap          = sun50i_iommu_unmap,
 773                 .free           = sun50i_iommu_domain_free,
 774         }
 775 };
 776
 777 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
 778                                       unsigned master, phys_addr_t iova,
 779                                       unsigned prot)
 780 {
 781         dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
 782                 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");
 783
 784         if (iommu->domain)
 785                 report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
 786         else
 787                 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
 788 }
 789
 790 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
 791                                               unsigned addr_reg,
 792                                               unsigned blame_reg)
 793 {
 794         phys_addr_t iova;
 795         unsigned master;
 796         u32 blame;
 797
 798         assert_spin_locked(&iommu->iommu_lock);
 799
 800         iova = iommu_read(iommu, addr_reg);
 801         blame = iommu_read(iommu, blame_reg);
 802         master = ilog2(blame & IOMMU_INT_MASTER_MASK);
 803
 804         /*
 805          * If the address is not in the page table, we can't get what
 806          * operation triggered the fault. Assume it's a read
 807          * operation.
 808          */
 809         sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);
 810
 811         return iova;
 812 }
 813
 814 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
 815 {
 816         enum sun50i_iommu_aci aci;
 817         phys_addr_t iova;
 818         unsigned master;
 819         unsigned dir;
 820         u32 blame;
 821
 822         assert_spin_locked(&iommu->iommu_lock);
 823
 824         blame = iommu_read(iommu, IOMMU_INT_STA_REG);
 825         master = ilog2(blame & IOMMU_INT_MASTER_MASK);
 826         iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
 827         aci = sun50i_get_pte_aci(iommu_read(iommu,
 828                                             IOMMU_INT_ERR_DATA_REG(master)));
 829
 830         switch (aci) {
 831                 /*
 832                  * If we are in the read-only domain, then it means we
 833                  * tried to write.
 834                  */
 835         case SUN50I_IOMMU_ACI_RD:
 836                 dir = IOMMU_FAULT_WRITE;
 837                 break;
 838
 839                 /*
 840                  * If we are in the write-only domain, then it means
 841                  * we tried to read.
 842                  */
 843         case SUN50I_IOMMU_ACI_WR:
 844
 845                 /*
 846                  * If we are in the domain without any permission, we
 847                  * can't really tell. Let's default to a read
 848                  * operation.
 849                  */
 850         case SUN50I_IOMMU_ACI_NONE:
 851
 852                 /* WTF? */
 853         case SUN50I_IOMMU_ACI_RD_WR:
 854         default:
 855                 dir = IOMMU_FAULT_READ;
 856                 break;
 857         }
 858
 859         /*
 860          * If the address is not in the page table, we can't get what
 861          * operation triggered the fault. Assume it's a read
 862          * operation.
 863          */
 864         sun50i_iommu_report_fault(iommu, master, iova, dir);
 865
 866         return iova;
 867 }
 868
 869 static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
 870 {
 871         struct sun50i_iommu *iommu = dev_id;
 872         u32 status;
 873
 874         spin_lock(&iommu->iommu_lock);
 875
 876         status = iommu_read(iommu, IOMMU_INT_STA_REG);
 877         if (!(status & IOMMU_INT_MASK)) {
 878                 spin_unlock(&iommu->iommu_lock);
 879                 return IRQ_NONE;
 880         }
 881
 882         if (status & IOMMU_INT_INVALID_L2PG)
 883                 sun50i_iommu_handle_pt_irq(iommu,
 884                                             IOMMU_INT_ERR_ADDR_L2_REG,
 885                                             IOMMU_L2PG_INT_REG);
 886         else if (status & IOMMU_INT_INVALID_L1PG)
 887                 sun50i_iommu_handle_pt_irq(iommu,
 888                                            IOMMU_INT_ERR_ADDR_L1_REG,
 889                                            IOMMU_L1PG_INT_REG);
 890         else
 891                 sun50i_iommu_handle_perm_irq(iommu);
 892
 893         iommu_write(iommu, IOMMU_INT_CLR_REG, status);
 894
 895         iommu_write(iommu, IOMMU_RESET_REG, ~status);
 896         iommu_write(iommu, IOMMU_RESET_REG, status);
 897
 898         spin_unlock(&iommu->iommu_lock);
 899
 900         return IRQ_HANDLED;
 901 }
 902
 903 static int sun50i_iommu_probe(struct platform_device *pdev)
 904 {
 905         struct sun50i_iommu *iommu;
 906         int ret, irq;
 907
 908         iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
 909         if (!iommu)
 910                 return -ENOMEM;
 911         spin_lock_init(&iommu->iommu_lock);
 912         platform_set_drvdata(pdev, iommu);
 913         iommu->dev = &pdev->dev;
 914
 915         iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
 916                                            PT_SIZE, PT_SIZE,
 917                                            SLAB_HWCACHE_ALIGN,
 918                                            NULL);
 919         if (!iommu->pt_pool)
 920                 return -ENOMEM;
 921
 922         iommu->group = iommu_group_alloc();
 923         if (IS_ERR(iommu->group)) {
 924                 ret = PTR_ERR(iommu->group);
 925                 goto err_free_cache;
 926         }
 927
 928         iommu->base = devm_platform_ioremap_resource(pdev, 0);
 929         if (IS_ERR(iommu->base)) {
 930                 ret = PTR_ERR(iommu->base);
 931                 goto err_free_group;
 932         }
 933
 934         irq = platform_get_irq(pdev, 0);
 935         if (irq < 0) {
 936                 ret = irq;
 937                 goto err_free_group;
 938         }
 939
 940         iommu->clk = devm_clk_get(&pdev->dev, NULL);
 941         if (IS_ERR(iommu->clk)) {
 942                 dev_err(&pdev->dev, "Couldn't get our clock.\n");
 943                 ret = PTR_ERR(iommu->clk);
 944                 goto err_free_group;
 945         }
 946
 947         iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
 948         if (IS_ERR(iommu->reset)) {
 949                 dev_err(&pdev->dev, "Couldn't get our reset line.\n");
 950                 ret = PTR_ERR(iommu->reset);
 951                 goto err_free_group;
 952         }
 953
 954         ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
 955                                      NULL, dev_name(&pdev->dev));
 956         if (ret)
 957                 goto err_free_group;
 958
 959         ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);
 960         if (ret)
 961                 goto err_remove_sysfs;
 962
 963         ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
 964                                dev_name(&pdev->dev), iommu);
 965         if (ret < 0)
 966                 goto err_unregister;
 967
 968         return 0;
 969
 970 err_unregister:
 971         iommu_device_unregister(&iommu->iommu);
 972
 973 err_remove_sysfs:
 974         iommu_device_sysfs_remove(&iommu->iommu);
 975
 976 err_free_group:
 977         iommu_group_put(iommu->group);
 978
 979 err_free_cache:
 980         kmem_cache_destroy(iommu->pt_pool);
 981
 982         return ret;
 983 }
 984
 985 static const struct of_device_id sun50i_iommu_dt[] = {
 986         { .compatible = "allwinner,sun50i-h6-iommu", },
 987         { /* sentinel */ },
 988 };
 989 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);
 990
 991 static struct platform_driver sun50i_iommu_driver = {
 992         .driver         = {
 993                 .name                   = "sun50i-iommu",
 994                 .of_match_table         = sun50i_iommu_dt,
 995                 .suppress_bind_attrs    = true,
 996         }
 997 };
 998 builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
 999
1000 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
1001 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
1002 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
1003 MODULE_LICENSE("Dual BSD/GPL");