drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2016-2018 Etnaviv Project
   4  */
   5
   6 #include <linux/bitops.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/platform_device.h>
   9 #include <linux/sizes.h>
  10 #include <linux/slab.h>
  11 #include <linux/vmalloc.h>
  12
  13 #include "etnaviv_cmdbuf.h"
  14 #include "etnaviv_gpu.h"
  15 #include "etnaviv_mmu.h"
  16 #include "state.xml.h"
  17 #include "state_hi.xml.h"
  18
  19 #define MMUv2_PTE_PRESENT               BIT(0)
  20 #define MMUv2_PTE_EXCEPTION             BIT(1)
  21 #define MMUv2_PTE_WRITEABLE             BIT(2)
  22
  23 #define MMUv2_MTLB_MASK                 0xffc00000
  24 #define MMUv2_MTLB_SHIFT                22
  25 #define MMUv2_STLB_MASK                 0x003ff000
  26 #define MMUv2_STLB_SHIFT                12
  27
  28 #define MMUv2_MAX_STLB_ENTRIES          1024
  29
  30 struct etnaviv_iommuv2_context {
  31         struct etnaviv_iommu_context base;
  32         unsigned short id;
  33         /* M(aster) TLB aka first level pagetable */
  34         u32 *mtlb_cpu;
  35         dma_addr_t mtlb_dma;
  36         /* S(lave) TLB aka second level pagetable */
  37         u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES];
  38         dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES];
  39 };
  40
  41 static struct etnaviv_iommuv2_context *
  42 to_v2_context(struct etnaviv_iommu_context *context)
  43 {
  44         return container_of(context, struct etnaviv_iommuv2_context, base);
  45 }
  46
  47 static void etnaviv_iommuv2_free(struct etnaviv_iommu_context *context)
  48 {
  49         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
  50         int i;
  51
  52         drm_mm_takedown(&context->mm);
  53
  54         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
  55                 if (v2_context->stlb_cpu[i])
  56                         dma_free_wc(context->global->dev, SZ_4K,
  57                                     v2_context->stlb_cpu[i],
  58                                     v2_context->stlb_dma[i]);
  59         }
  60
  61         dma_free_wc(context->global->dev, SZ_4K, v2_context->mtlb_cpu,
  62                     v2_context->mtlb_dma);
  63
  64         clear_bit(v2_context->id, context->global->v2.pta_alloc);
  65
  66         vfree(v2_context);
  67 }
  68 static int
  69 etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_context *v2_context,
  70                             int stlb)
  71 {
  72         if (v2_context->stlb_cpu[stlb])
  73                 return 0;
  74
  75         v2_context->stlb_cpu[stlb] =
  76                         dma_alloc_wc(v2_context->base.global->dev, SZ_4K,
  77                                      &v2_context->stlb_dma[stlb],
  78                                      GFP_KERNEL);
  79
  80         if (!v2_context->stlb_cpu[stlb])
  81                 return -ENOMEM;
  82
  83         memset32(v2_context->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION,
  84                  SZ_4K / sizeof(u32));
  85
  86         v2_context->mtlb_cpu[stlb] =
  87                         v2_context->stlb_dma[stlb] | MMUv2_PTE_PRESENT;
  88
  89         return 0;
  90 }
  91
  92 static int etnaviv_iommuv2_map(struct etnaviv_iommu_context *context,
  93                                unsigned long iova, phys_addr_t paddr,
  94                                size_t size, int prot)
  95 {
  96         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
  97         int mtlb_entry, stlb_entry, ret;
  98         u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT;
  99
 100         if (size != SZ_4K)
 101                 return -EINVAL;
 102
 103         if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
 104                 entry |= (upper_32_bits(paddr) & 0xff) << 4;
 105
 106         if (prot & ETNAVIV_PROT_WRITE)
 107                 entry |= MMUv2_PTE_WRITEABLE;
 108
 109         mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
 110         stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
 111
 112         ret = etnaviv_iommuv2_ensure_stlb(v2_context, mtlb_entry);
 113         if (ret)
 114                 return ret;
 115
 116         v2_context->stlb_cpu[mtlb_entry][stlb_entry] = entry;
 117
 118         return 0;
 119 }
 120
 121 static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_context *context,
 122                                     unsigned long iova, size_t size)
 123 {
 124         struct etnaviv_iommuv2_context *etnaviv_domain = to_v2_context(context);
 125         int mtlb_entry, stlb_entry;
 126
 127         if (size != SZ_4K)
 128                 return -EINVAL;
 129
 130         mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
 131         stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
 132
 133         etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
 134
 135         return SZ_4K;
 136 }
 137
 138 static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_context *context)
 139 {
 140         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
 141         size_t dump_size = SZ_4K;
 142         int i;
 143
 144         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
 145                 if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
 146                         dump_size += SZ_4K;
 147
 148         return dump_size;
 149 }
 150
 151 static void etnaviv_iommuv2_dump(struct etnaviv_iommu_context *context, void *buf)
 152 {
 153         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
 154         int i;
 155
 156         memcpy(buf, v2_context->mtlb_cpu, SZ_4K);
 157         buf += SZ_4K;
 158         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
 159                 if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) {
 160                         memcpy(buf, v2_context->stlb_cpu[i], SZ_4K);
 161                         buf += SZ_4K;
 162                 }
 163 }
 164
 165 static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu,
 166         struct etnaviv_iommu_context *context)
 167 {
 168         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
 169         u16 prefetch;
 170
 171         /* If the MMU is already enabled the state is still there. */
 172         if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)
 173                 return;
 174
 175         if (gpu->mmu_context)
 176                 etnaviv_iommu_context_put(gpu->mmu_context);
 177         gpu->mmu_context = etnaviv_iommu_context_get(context);
 178
 179         prefetch = etnaviv_buffer_config_mmuv2(gpu,
 180                                 (u32)v2_context->mtlb_dma,
 181                                 (u32)context->global->bad_page_dma);
 182         etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
 183                              prefetch);
 184         etnaviv_gpu_wait_idle(gpu, 100);
 185
 186         gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
 187 }
 188
 189 static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu,
 190         struct etnaviv_iommu_context *context)
 191 {
 192         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
 193         u16 prefetch;
 194
 195         /* If the MMU is already enabled the state is still there. */
 196         if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE)
 197                 return;
 198
 199         if (gpu->mmu_context)
 200                 etnaviv_iommu_context_put(gpu->mmu_context);
 201         gpu->mmu_context = etnaviv_iommu_context_get(context);
 202
 203         gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW,
 204                   lower_32_bits(context->global->v2.pta_dma));
 205         gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH,
 206                   upper_32_bits(context->global->v2.pta_dma));
 207         gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE);
 208
 209         gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW,
 210                   lower_32_bits(context->global->bad_page_dma));
 211         gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW,
 212                   lower_32_bits(context->global->bad_page_dma));
 213         gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG,
 214                   VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH(
 215                   upper_32_bits(context->global->bad_page_dma)) |
 216                   VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH(
 217                   upper_32_bits(context->global->bad_page_dma)));
 218
 219         context->global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma |
 220                                          VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K;
 221
 222         /* trigger a PTA load through the FE */
 223         prefetch = etnaviv_buffer_config_pta(gpu, v2_context->id);
 224         etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
 225                              prefetch);
 226         etnaviv_gpu_wait_idle(gpu, 100);
 227
 228         gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE);
 229 }
 230
 231 u32 etnaviv_iommuv2_get_mtlb_addr(struct etnaviv_iommu_context *context)
 232 {
 233         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
 234
 235         return v2_context->mtlb_dma;
 236 }
 237
 238 unsigned short etnaviv_iommuv2_get_pta_id(struct etnaviv_iommu_context *context)
 239 {
 240         struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
 241
 242         return v2_context->id;
 243 }
 244 static void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu,
 245                                     struct etnaviv_iommu_context *context)
 246 {
 247         switch (gpu->sec_mode) {
 248         case ETNA_SEC_NONE:
 249                 etnaviv_iommuv2_restore_nonsec(gpu, context);
 250                 break;
 251         case ETNA_SEC_KERNEL:
 252                 etnaviv_iommuv2_restore_sec(gpu, context);
 253                 break;
 254         default:
 255                 WARN(1, "unhandled GPU security mode\n");
 256                 break;
 257         }
 258 }
 259
 260 const struct etnaviv_iommu_ops etnaviv_iommuv2_ops = {
 261         .free = etnaviv_iommuv2_free,
 262         .map = etnaviv_iommuv2_map,
 263         .unmap = etnaviv_iommuv2_unmap,
 264         .dump_size = etnaviv_iommuv2_dump_size,
 265         .dump = etnaviv_iommuv2_dump,
 266         .restore = etnaviv_iommuv2_restore,
 267 };
 268
 269 struct etnaviv_iommu_context *
 270 etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global *global)
 271 {
 272         struct etnaviv_iommuv2_context *v2_context;
 273         struct etnaviv_iommu_context *context;
 274
 275         v2_context = vzalloc(sizeof(*v2_context));
 276         if (!v2_context)
 277                 return NULL;
 278
 279         mutex_lock(&global->lock);
 280         v2_context->id = find_first_zero_bit(global->v2.pta_alloc,
 281                                              ETNAVIV_PTA_ENTRIES);
 282         if (v2_context->id < ETNAVIV_PTA_ENTRIES) {
 283                 set_bit(v2_context->id, global->v2.pta_alloc);
 284         } else {
 285                 mutex_unlock(&global->lock);
 286                 goto out_free;
 287         }
 288         mutex_unlock(&global->lock);
 289
 290         v2_context->mtlb_cpu = dma_alloc_wc(global->dev, SZ_4K,
 291                                             &v2_context->mtlb_dma, GFP_KERNEL);
 292         if (!v2_context->mtlb_cpu)
 293                 goto out_free_id;
 294
 295         memset32(v2_context->mtlb_cpu, MMUv2_PTE_EXCEPTION,
 296                  MMUv2_MAX_STLB_ENTRIES);
 297
 298         global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma;
 299
 300         context = &v2_context->base;
 301         context->global = global;
 302         kref_init(&context->refcount);
 303         mutex_init(&context->lock);
 304         INIT_LIST_HEAD(&context->mappings);
 305         drm_mm_init(&context->mm, SZ_4K, (u64)SZ_1G * 4 - SZ_4K);
 306
 307         return context;
 308
 309 out_free_id:
 310         clear_bit(v2_context->id, global->v2.pta_alloc);
 311 out_free:
 312         vfree(v2_context);
 313         return NULL;
 314 }