drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c

   1 /*
   2  * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_kernel_common.h"
  12 #include "mali_kernel_core.h"
  13 #include "mali_osk.h"
  14 #include "mali_uk_types.h"
  15 #include "mali_mmu_page_directory.h"
  16 #include "mali_memory.h"
  17 #include "mali_l2_cache.h"
  18 #include "mali_group.h"
  19
  20 static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data);
  21
  22 u32 mali_allocate_empty_page(void)
  23 {
  24         _mali_osk_errcode_t err;
  25         mali_io_address mapping;
  26         u32 address;
  27
  28         if(_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping))
  29         {
  30                 /* Allocation failed */
  31                 return 0;
  32         }
  33
  34         MALI_DEBUG_ASSERT_POINTER( mapping );
  35
  36         err = fill_page(mapping, 0);
  37         if (_MALI_OSK_ERR_OK != err)
  38         {
  39                 mali_mmu_release_table_page(address);
  40         }
  41         return address;
  42 }
  43
  44 void mali_free_empty_page(u32 address)
  45 {
  46         if (MALI_INVALID_PAGE != address)
  47         {
  48                 mali_mmu_release_table_page(address);
  49         }
  50 }
  51
  52 _mali_osk_errcode_t mali_create_fault_flush_pages(u32 *page_directory, u32 *page_table, u32 *data_page)
  53 {
  54         _mali_osk_errcode_t err;
  55         mali_io_address page_directory_mapping;
  56         mali_io_address page_table_mapping;
  57         mali_io_address data_page_mapping;
  58
  59         err = mali_mmu_get_table_page(data_page, &data_page_mapping);
  60         if (_MALI_OSK_ERR_OK == err)
  61         {
  62                 err = mali_mmu_get_table_page(page_table, &page_table_mapping);
  63                 if (_MALI_OSK_ERR_OK == err)
  64                 {
  65                         err = mali_mmu_get_table_page(page_directory, &page_directory_mapping);
  66                         if (_MALI_OSK_ERR_OK == err)
  67                         {
  68                                 fill_page(data_page_mapping, 0);
  69                                 fill_page(page_table_mapping, *data_page | MALI_MMU_FLAGS_WRITE_PERMISSION | MALI_MMU_FLAGS_READ_PERMISSION | MALI_MMU_FLAGS_PRESENT);
  70                                 fill_page(page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT);
  71                                 MALI_SUCCESS;
  72                         }
  73                         mali_mmu_release_table_page(*page_table);
  74                         *page_table = MALI_INVALID_PAGE;
  75                 }
  76                 mali_mmu_release_table_page(*data_page);
  77                 *data_page = MALI_INVALID_PAGE;
  78         }
  79         return err;
  80 }
  81
  82 void mali_destroy_fault_flush_pages(u32 *page_directory, u32 *page_table, u32 *data_page)
  83 {
  84         if (MALI_INVALID_PAGE != *page_directory)
  85         {
  86                 mali_mmu_release_table_page(*page_directory);
  87                 *page_directory = MALI_INVALID_PAGE;
  88         }
  89
  90         if (MALI_INVALID_PAGE != *page_table)
  91         {
  92                 mali_mmu_release_table_page(*page_table);
  93                 *page_table = MALI_INVALID_PAGE;
  94         }
  95
  96         if (MALI_INVALID_PAGE != *data_page)
  97         {
  98                 mali_mmu_release_table_page(*data_page);
  99                 *data_page = MALI_INVALID_PAGE;
 100         }
 101 }
 102
 103 static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data)
 104 {
 105         int i;
 106         MALI_DEBUG_ASSERT_POINTER( mapping );
 107
 108         for(i = 0; i < MALI_MMU_PAGE_SIZE/4; i++)
 109         {
 110 #ifdef CONFIG_SLP_MALI_DBG
 111                 _mali_osk_mem_iowrite32_relaxed_cpu( mapping, i * sizeof(u32), data);
 112 #else
 113                 _mali_osk_mem_iowrite32_relaxed( mapping, i * sizeof(u32), data);
 114 #endif
 115         }
 116         _mali_osk_mem_barrier();
 117         MALI_SUCCESS;
 118 }
 119
 120 _mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
 121 {
 122         const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
 123         const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
 124         _mali_osk_errcode_t err;
 125         mali_io_address pde_mapping;
 126         u32 pde_phys;
 127         int i;
 128
 129         for(i = first_pde; i <= last_pde; i++)
 130         {
 131 #ifdef CONFIG_SLP_MALI_DBG
 132                 if(0 == (_mali_osk_mem_ioread32_cpu(pagedir->page_directory_mapped, i*sizeof(u32))&MALI_MMU_FLAGS_PRESENT))
 133 #else
 134                 if(0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i*sizeof(u32)) & MALI_MMU_FLAGS_PRESENT))
 135 #endif
 136                 {
 137                         /* Page table not present */
 138                         MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
 139                         MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]);
 140
 141                         err = mali_mmu_get_table_page(&pde_phys, &pde_mapping);
 142                         if(_MALI_OSK_ERR_OK != err)
 143                         {
 144                                 MALI_PRINT_ERROR(("Failed to allocate page table page.\n"));
 145                                 return err;
 146                         }
 147                         pagedir->page_entries_mapped[i] = pde_mapping;
 148
 149                         /* Update PDE, mark as present */
 150 #ifdef CONFIG_SLP_MALI_DBG
 151                         _mali_osk_mem_iowrite32_relaxed_cpu(pagedir->page_directory_mapped, i*sizeof(u32),
 152                                         pde_phys | MALI_MMU_FLAGS_PRESENT);
 153 #else
 154                         _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i*sizeof(u32),
 155                                         pde_phys | MALI_MMU_FLAGS_PRESENT);
 156 #endif
 157
 158                         MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
 159                         pagedir->page_entries_usage_count[i] = 1;
 160                 }
 161                 else
 162                 {
 163                         pagedir->page_entries_usage_count[i]++;
 164                 }
 165         }
 166         _mali_osk_write_mem_barrier();
 167
 168         MALI_SUCCESS;
 169 }
 170
 171 MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size)
 172 {
 173         int i;
 174         const int first_pte = MALI_MMU_PTE_ENTRY(mali_address);
 175         const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1);
 176
 177         for (i = first_pte; i <= last_pte; i++)
 178         {
 179 #ifdef CONFIG_SLP_MALI_DBG
 180                 _mali_osk_mem_iowrite32_relaxed_cpu(page_table, i * sizeof(u32), 0);
 181 #else
 182                 _mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0);
 183 #endif
 184         }
 185 }
 186
 187 _mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
 188 {
 189         const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
 190         const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
 191         u32 left = size;
 192         int i;
 193 #ifndef MALI_UNMAP_FLUSH_ALL_MALI_L2
 194         mali_bool pd_changed = MALI_FALSE;
 195         u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */
 196         u32 num_pages_inv = 0;
 197 #endif
 198
 199         /* For all page directory entries in range. */
 200         for (i = first_pde; i <= last_pde; i++)
 201         {
 202                 u32 size_in_pde, offset;
 203
 204                 MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]);
 205                 MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]);
 206
 207                 /* Offset into page table, 0 if mali_address is 4MiB aligned */
 208                 offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1));
 209                 if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset)
 210                 {
 211                         size_in_pde = left;
 212                 }
 213                 else
 214                 {
 215                         size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset;
 216                 }
 217
 218                 pagedir->page_entries_usage_count[i]--;
 219
 220                 /* If entire page table is unused, free it */
 221                 if (0 == pagedir->page_entries_usage_count[i])
 222                 {
 223                         u32 page_address;
 224                         MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n"));
 225                         /* last reference removed, no need to zero out each PTE  */
 226
 227 #ifdef CONFIG_SLP_MALI_DBG
 228                         page_address = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32_cpu(pagedir->page_directory_mapped, i*sizeof(u32)));
 229 #else
 230                         page_address = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i*sizeof(u32)));
 231 #endif
 232                         pagedir->page_entries_mapped[i] = NULL;
 233 #ifdef CONFIG_SLP_MALI_DBG
 234                         _mali_osk_mem_iowrite32_relaxed_cpu(pagedir->page_directory_mapped, i*sizeof(u32), 0);
 235 #else
 236                         _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i*sizeof(u32), 0);
 237 #endif
 238
 239                         mali_mmu_release_table_page(page_address);
 240 #ifndef MALI_UNMAP_FLUSH_ALL_MALI_L2
 241                         pd_changed = MALI_TRUE;
 242 #endif
 243                 }
 244                 else
 245                 {
 246 #ifndef MALI_UNMAP_FLUSH_ALL_MALI_L2
 247                         pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i);
 248                         num_pages_inv++;
 249                         MALI_DEBUG_ASSERT(num_pages_inv<3);
 250 #endif
 251
 252                         /* If part of the page table is still in use, zero the relevant PTEs */
 253                         mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde);
 254                 }
 255
 256                 left -= size_in_pde;
 257                 mali_address += size_in_pde;
 258         }
 259         _mali_osk_write_mem_barrier();
 260
 261 #ifndef MALI_UNMAP_FLUSH_ALL_MALI_L2
 262         /* L2 pages invalidation */
 263         if (MALI_TRUE == pd_changed)
 264         {
 265                 pages_to_invalidate[num_pages_inv] = pagedir->page_directory;
 266                 num_pages_inv++;
 267                 MALI_DEBUG_ASSERT(num_pages_inv<3);
 268         }
 269
 270         if (_MALI_PRODUCT_ID_MALI200 != mali_kernel_core_get_product_id())
 271         {
 272                 mali_l2_cache_invalidate_pages_conditional(pages_to_invalidate, num_pages_inv);
 273         }
 274 #endif
 275
 276         MALI_SUCCESS;
 277 }
 278
 279 struct mali_page_directory *mali_mmu_pagedir_alloc(void)
 280 {
 281         struct mali_page_directory *pagedir;
 282
 283         pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory));
 284         if(NULL == pagedir)
 285         {
 286                 return NULL;
 287         }
 288
 289         if(_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&pagedir->page_directory, &pagedir->page_directory_mapped))
 290         {
 291                 _mali_osk_free(pagedir);
 292                 return NULL;
 293         }
 294
 295         /* Zero page directory */
 296         fill_page(pagedir->page_directory_mapped, 0);
 297
 298         return pagedir;
 299 }
 300
 301 void mali_mmu_pagedir_free(struct mali_page_directory *pagedir)
 302 {
 303         const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]);
 304         int i;
 305
 306         /* Free referenced page tables and zero PDEs. */
 307         for (i = 0; i < num_page_table_entries; i++)
 308         {
 309 #ifdef CONFIG_SLP_MALI_DBG
 310                 if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32_cpu(pagedir->page_directory_mapped, sizeof(u32)*i)&MALI_MMU_FLAGS_PRESENT))
 311                 {
 312                         mali_mmu_release_table_page( _mali_osk_mem_ioread32_cpu(pagedir->page_directory_mapped, i*sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
 313                         _mali_osk_mem_iowrite32_relaxed_cpu(pagedir->page_directory_mapped, i * sizeof(u32), 0);
 314                 }
 315 #else
 316                 if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT))
 317                 {
 318                         mali_mmu_release_table_page( _mali_osk_mem_ioread32(pagedir->page_directory_mapped, i*sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
 319                         _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
 320                 }
 321 #endif
 322         }
 323         _mali_osk_write_mem_barrier();
 324
 325         /* Free the page directory page. */
 326         mali_mmu_release_table_page(pagedir->page_directory);
 327
 328         _mali_osk_free(pagedir);
 329 }
 330
 331
 332 void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, u32 phys_address, u32 size, mali_memory_cache_settings cache_settings)
 333 {
 334         u32 end_address = mali_address + size;
 335         u32 permission_bits;
 336
 337         switch ( cache_settings )
 338         {
 339                 case MALI_CACHE_GP_READ_ALLOCATE:
 340                 MALI_DEBUG_PRINT(5, ("Map L2 GP_Read_allocate\n"));
 341                 permission_bits = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE;
 342                 break;
 343
 344                 case MALI_CACHE_STANDARD:
 345                 MALI_DEBUG_PRINT(5, ("Map L2 Standard\n"));
 346                 /*falltrough */
 347                 default:
 348                 if ( MALI_CACHE_STANDARD != cache_settings) MALI_PRINT_ERROR(("Wrong cache settings\n"));
 349                 permission_bits = MALI_MMU_FLAGS_WRITE_PERMISSION | MALI_MMU_FLAGS_READ_PERMISSION | MALI_MMU_FLAGS_PRESENT;
 350         }
 351
 352         /* Map physical pages into MMU page tables */
 353         for ( ; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, phys_address += MALI_MMU_PAGE_SIZE)
 354         {
 355                 MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]);
 356 #ifdef CONFIG_SLP_MALI_DBG
 357                 _mali_osk_mem_iowrite32_relaxed_cpu( pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY( mali_address)],
 358                                 MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
 359                                 phys_address | permission_bits);
 360 #else
 361                 _mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)],
 362                                 MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
 363                                 phys_address | permission_bits);
 364 #endif
 365         }
 366         _mali_osk_write_mem_barrier();
 367 }
 368
 369 u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index)
 370 {
 371 #ifdef CONFIG_SLP_MALI_DBG
 372         return (_mali_osk_mem_ioread32_cpu( pagedir->page_directory_mapped, index*sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
 373 #else
 374         return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, index*sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
 375 #endif
 376 }
 377
 378 /* For instrumented */
 379 struct dump_info
 380 {
 381         u32 buffer_left;
 382         u32 register_writes_size;
 383         u32 page_table_dump_size;
 384         u32 *buffer;
 385 };
 386
 387 static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info)
 388 {
 389         if (NULL != info)
 390         {
 391                 info->register_writes_size += sizeof(u32)*2; /* two 32-bit words */
 392
 393                 if (NULL != info->buffer)
 394                 {
 395                         /* check that we have enough space */
 396                         if (info->buffer_left < sizeof(u32)*2) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
 397
 398                         *info->buffer = where;
 399                         info->buffer++;
 400
 401                         *info->buffer = what;
 402                         info->buffer++;
 403
 404                         info->buffer_left -= sizeof(u32)*2;
 405                 }
 406         }
 407
 408         MALI_SUCCESS;
 409 }
 410
 411 static _mali_osk_errcode_t dump_page(mali_io_address page, u32 phys_addr, struct dump_info * info)
 412 {
 413         if (NULL != info)
 414         {
 415                 /* 4096 for the page and 4 bytes for the address */
 416                 const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4;
 417                 const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE;
 418                 const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4;
 419
 420                 info->page_table_dump_size += dump_size_in_bytes;
 421
 422                 if (NULL != info->buffer)
 423                 {
 424                         if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
 425
 426                         *info->buffer = phys_addr;
 427                         info->buffer++;
 428
 429                         _mali_osk_memcpy(info->buffer, page, page_size_in_bytes);
 430                         info->buffer += page_size_in_elements;
 431
 432                         info->buffer_left -= dump_size_in_bytes;
 433                 }
 434         }
 435
 436         MALI_SUCCESS;
 437 }
 438
 439 static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info * info)
 440 {
 441         MALI_DEBUG_ASSERT_POINTER(pagedir);
 442         MALI_DEBUG_ASSERT_POINTER(info);
 443
 444         if (NULL != pagedir->page_directory_mapped)
 445         {
 446                 int i;
 447
 448                 MALI_CHECK_NO_ERROR(
 449                         dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info)
 450                         );
 451
 452                 for (i = 0; i < 1024; i++)
 453                 {
 454                         if (NULL != pagedir->page_entries_mapped[i])
 455                         {
 456                                 MALI_CHECK_NO_ERROR(
 457                                     dump_page(pagedir->page_entries_mapped[i],
 458 #ifdef CONFIG_SLP_MALI_DBG
 459                                         _mali_osk_mem_ioread32_cpu(pagedir->page_directory_mapped,
 460                                         i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
 461 #else
 462                                         _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
 463                                         i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
 464 #endif
 465                                 );
 466                         }
 467                 }
 468         }
 469
 470         MALI_SUCCESS;
 471 }
 472
 473 static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info * info)
 474 {
 475         MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory,
 476                                      "set the page directory address", info));
 477         MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info));
 478         MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info));
 479         MALI_SUCCESS;
 480 }
 481
 482 _mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size( _mali_uk_query_mmu_page_table_dump_size_s *args )
 483 {
 484         struct dump_info info = { 0, 0, 0, NULL };
 485         struct mali_session_data * session_data;
 486
 487         MALI_DEBUG_ASSERT_POINTER(args);
 488         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
 489
 490         session_data = (struct mali_session_data *)(args->ctx);
 491
 492         MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
 493         MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
 494         args->size = info.register_writes_size + info.page_table_dump_size;
 495         MALI_SUCCESS;
 496 }
 497
 498 _mali_osk_errcode_t _mali_ukk_dump_mmu_page_table( _mali_uk_dump_mmu_page_table_s * args )
 499 {
 500         struct dump_info info = { 0, 0, 0, NULL };
 501         struct mali_session_data * session_data;
 502
 503         MALI_DEBUG_ASSERT_POINTER(args);
 504         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
 505         MALI_CHECK_NON_NULL(args->buffer, _MALI_OSK_ERR_INVALID_ARGS);
 506
 507         session_data = (struct mali_session_data *)(args->ctx);
 508
 509         info.buffer_left = args->size;
 510         info.buffer = args->buffer;
 511
 512         args->register_writes = info.buffer;
 513         MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
 514
 515         args->page_table_dump = info.buffer;
 516         MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
 517
 518         args->register_writes_size = info.register_writes_size;
 519         args->page_table_dump_size = info.page_table_dump_size;
 520
 521         MALI_SUCCESS;
 522 }