drivers/gpu/arm/mali400/mali/common/mali_mmu.c

   1 /*
   2  * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_kernel_common.h"
  12 #include "mali_osk.h"
  13 #include "mali_osk_bitops.h"
  14 #include "mali_osk_list.h"
  15 #include "mali_ukk.h"
  16
  17 #include "mali_mmu.h"
  18 #include "mali_hw_core.h"
  19 #include "mali_group.h"
  20 #include "mali_mmu_page_directory.h"
  21
  22 /**
  23  * Size of the MMU registers in bytes
  24  */
  25 #define MALI_MMU_REGISTERS_SIZE 0x24
  26
  27 /**
  28  * MMU commands
  29  * These are the commands that can be sent
  30  * to the MMU unit.
  31  */
  32 typedef enum mali_mmu_command
  33 {
  34         MALI_MMU_COMMAND_ENABLE_PAGING = 0x00, /**< Enable paging (memory translation) */
  35         MALI_MMU_COMMAND_DISABLE_PAGING = 0x01, /**< Disable paging (memory translation) */
  36         MALI_MMU_COMMAND_ENABLE_STALL = 0x02, /**<  Enable stall on page fault */
  37         MALI_MMU_COMMAND_DISABLE_STALL = 0x03, /**< Disable stall on page fault */
  38         MALI_MMU_COMMAND_ZAP_CACHE = 0x04, /**< Zap the entire page table cache */
  39         MALI_MMU_COMMAND_PAGE_FAULT_DONE = 0x05, /**< Page fault processed */
  40         MALI_MMU_COMMAND_HARD_RESET = 0x06 /**< Reset the MMU back to power-on settings */
  41 } mali_mmu_command;
  42
  43 static void mali_mmu_probe_trigger(void *data);
  44 static _mali_osk_errcode_t mali_mmu_probe_ack(void *data);
  45
  46 MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu);
  47
  48 /* page fault queue flush helper pages
  49  * note that the mapping pointers are currently unused outside of the initialization functions */
  50 static u32 mali_page_fault_flush_page_directory = MALI_INVALID_PAGE;
  51 static u32 mali_page_fault_flush_page_table = MALI_INVALID_PAGE;
  52 static u32 mali_page_fault_flush_data_page = MALI_INVALID_PAGE;
  53
  54 /* an empty page directory (no address valid) which is active on any MMU not currently marked as in use */
  55 static u32 mali_empty_page_directory = MALI_INVALID_PAGE;
  56
  57 _mali_osk_errcode_t mali_mmu_initialize(void)
  58 {
  59         /* allocate the helper pages */
  60         mali_empty_page_directory = mali_allocate_empty_page();
  61         if(0 == mali_empty_page_directory)
  62         {
  63                 mali_empty_page_directory = MALI_INVALID_PAGE;
  64                 return _MALI_OSK_ERR_NOMEM;
  65         }
  66
  67         if (_MALI_OSK_ERR_OK != mali_create_fault_flush_pages(&mali_page_fault_flush_page_directory,
  68                                         &mali_page_fault_flush_page_table, &mali_page_fault_flush_data_page))
  69         {
  70                 mali_free_empty_page(mali_empty_page_directory);
  71                 return _MALI_OSK_ERR_FAULT;
  72         }
  73
  74         return _MALI_OSK_ERR_OK;
  75 }
  76
  77 void mali_mmu_terminate(void)
  78 {
  79         MALI_DEBUG_PRINT(3, ("Mali MMU: terminating\n"));
  80
  81         /* Free global helper pages */
  82         mali_free_empty_page(mali_empty_page_directory);
  83
  84         /* Free the page fault flush pages */
  85         mali_destroy_fault_flush_pages(&mali_page_fault_flush_page_directory,
  86                                     &mali_page_fault_flush_page_table, &mali_page_fault_flush_data_page);
  87 }
  88
  89 struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual)
  90 {
  91         struct mali_mmu_core* mmu = NULL;
  92
  93         MALI_DEBUG_ASSERT_POINTER(resource);
  94
  95         MALI_DEBUG_PRINT(2, ("Mali MMU: Creating Mali MMU: %s\n", resource->description));
  96
  97         mmu = _mali_osk_calloc(1,sizeof(struct mali_mmu_core));
  98         if (NULL != mmu)
  99         {
 100                 if (_MALI_OSK_ERR_OK == mali_hw_core_create(&mmu->hw_core, resource, MALI_MMU_REGISTERS_SIZE))
 101                 {
 102                         if (_MALI_OSK_ERR_OK == mali_group_add_mmu_core(group, mmu))
 103                         {
 104                                 if (is_virtual)
 105                                 {
 106                                         /* Skip reset and IRQ setup for virtual MMU */
 107                                         return mmu;
 108                                 }
 109
 110                                 if (_MALI_OSK_ERR_OK == mali_mmu_reset(mmu))
 111                                 {
 112                                         /* Setup IRQ handlers (which will do IRQ probing if needed) */
 113                                         mmu->irq = _mali_osk_irq_init(resource->irq,
 114                                                                       mali_group_upper_half_mmu,
 115                                                                       group,
 116                                                                       mali_mmu_probe_trigger,
 117                                                                       mali_mmu_probe_ack,
 118                                                                       mmu,
 119                                                                       "mali_mmu_irq_handlers");
 120                                         if (NULL != mmu->irq)
 121                                         {
 122                                                 return mmu;
 123                                         }
 124                                         else
 125                                         {
 126                                                 MALI_PRINT_ERROR(("Mali MMU: Failed to setup interrupt handlers for MMU %s\n", mmu->hw_core.description));
 127                                         }
 128                                 }
 129                                 mali_group_remove_mmu_core(group);
 130                         }
 131                         else
 132                         {
 133                                 MALI_PRINT_ERROR(("Mali MMU: Failed to add core %s to group\n", mmu->hw_core.description));
 134                         }
 135                         mali_hw_core_delete(&mmu->hw_core);
 136                 }
 137
 138                 _mali_osk_free(mmu);
 139         }
 140         else
 141         {
 142                 MALI_PRINT_ERROR(("Failed to allocate memory for MMU\n"));
 143         }
 144
 145         return NULL;
 146 }
 147
 148 void mali_mmu_delete(struct mali_mmu_core *mmu)
 149 {
 150         if (NULL != mmu->irq)
 151         {
 152                 _mali_osk_irq_term(mmu->irq);
 153         }
 154
 155         mali_hw_core_delete(&mmu->hw_core);
 156         _mali_osk_free(mmu);
 157 }
 158
 159 static void mali_mmu_enable_paging(struct mali_mmu_core *mmu)
 160 {
 161         int i;
 162
 163         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING);
 164
 165         for (i = 0; i < MALI_REG_POLL_COUNT_SLOW; ++i)
 166         {
 167                 if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_PAGING_ENABLED)
 168                 {
 169                         break;
 170                 }
 171         }
 172         if (MALI_REG_POLL_COUNT_SLOW == i)
 173         {
 174                 MALI_PRINT_ERROR(("Enable paging request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS)));
 175         }
 176 }
 177
 178 mali_bool mali_mmu_enable_stall(struct mali_mmu_core *mmu)
 179 {
 180         int i;
 181         u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS);
 182
 183         if ( 0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED) )
 184         {
 185                 MALI_DEBUG_PRINT(4, ("MMU stall is implicit when Paging is not enebled.\n"));
 186                 return MALI_TRUE;
 187         }
 188
 189         if ( mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE )
 190         {
 191                 MALI_DEBUG_PRINT(3, ("Aborting MMU stall request since it is in pagefault state.\n"));
 192                 return MALI_FALSE;
 193         }
 194
 195         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL);
 196
 197         for (i = 0; i < MALI_REG_POLL_COUNT_SLOW; ++i)
 198         {
 199                 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS);
 200                 if (mmu_status & (MALI_MMU_STATUS_BIT_STALL_ACTIVE|MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) &&
 201                     (0 == (mmu_status & MALI_MMU_STATUS_BIT_STALL_NOT_ACTIVE)))
 202                 {
 203                         break;
 204                 }
 205                 if (0 == (mmu_status & ( MALI_MMU_STATUS_BIT_PAGING_ENABLED )))
 206                 {
 207                         break;
 208                 }
 209         }
 210         if (MALI_REG_POLL_COUNT_SLOW == i)
 211         {
 212                 MALI_PRINT_ERROR(("Enable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS)));
 213                 return MALI_FALSE;
 214         }
 215
 216         if ( mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE )
 217         {
 218                 MALI_DEBUG_PRINT(3, ("Aborting MMU stall request since it has a pagefault.\n"));
 219                 return MALI_FALSE;
 220         }
 221
 222         return MALI_TRUE;
 223 }
 224
 225 void mali_mmu_disable_stall(struct mali_mmu_core *mmu)
 226 {
 227         int i;
 228         u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS);
 229
 230         if ( 0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED ))
 231         {
 232                 MALI_DEBUG_PRINT(3, ("MMU disable skipped since it was not enabled.\n"));
 233                 return;
 234         }
 235         if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE)
 236         {
 237                 MALI_DEBUG_PRINT(2, ("Aborting MMU disable stall request since it is in pagefault state.\n"));
 238                 return;
 239         }
 240
 241         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL);
 242
 243         for (i = 0; i < MALI_REG_POLL_COUNT_SLOW; ++i)
 244         {
 245                 u32 status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS);
 246                 if ( 0 == (status & MALI_MMU_STATUS_BIT_STALL_ACTIVE) )
 247                 {
 248                         break;
 249                 }
 250                 if ( status &  MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE )
 251                 {
 252                         break;
 253                 }
 254                 if ( 0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED ))
 255                 {
 256                         break;
 257                 }
 258         }
 259         if (MALI_REG_POLL_COUNT_SLOW == i) MALI_DEBUG_PRINT(1,("Disable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS)));
 260 }
 261
 262 void mali_mmu_page_fault_done(struct mali_mmu_core *mmu)
 263 {
 264         MALI_DEBUG_PRINT(4, ("Mali MMU: %s: Leaving page fault mode\n", mmu->hw_core.description));
 265         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_PAGE_FAULT_DONE);
 266 }
 267
 268 MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu)
 269 {
 270         int i;
 271
 272         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, 0xCAFEBABE);
 273         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_HARD_RESET);
 274
 275         for (i = 0; i < MALI_REG_POLL_COUNT_SLOW; ++i)
 276         {
 277                 if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR) == 0)
 278                 {
 279                         break;
 280                 }
 281         }
 282         if (MALI_REG_POLL_COUNT_SLOW == i)
 283         {
 284                 MALI_PRINT_ERROR(("Reset request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS)));
 285                 return _MALI_OSK_ERR_FAULT;
 286         }
 287
 288         return _MALI_OSK_ERR_OK;
 289 }
 290
 291 _mali_osk_errcode_t mali_mmu_reset(struct mali_mmu_core *mmu)
 292 {
 293         _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT;
 294         mali_bool stall_success;
 295         MALI_DEBUG_ASSERT_POINTER(mmu);
 296
 297         stall_success = mali_mmu_enable_stall(mmu);
 298
 299         /* The stall can not fail in current hw-state */
 300         MALI_DEBUG_ASSERT(stall_success);
 301
 302         MALI_DEBUG_PRINT(3, ("Mali MMU: mali_kernel_mmu_reset: %s\n", mmu->hw_core.description));
 303
 304         if (_MALI_OSK_ERR_OK == mali_mmu_raw_reset(mmu))
 305         {
 306                 mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR);
 307                 /* no session is active, so just activate the empty page directory */
 308                 mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, mali_empty_page_directory);
 309                 mali_mmu_enable_paging(mmu);
 310                 err = _MALI_OSK_ERR_OK;
 311         }
 312         mali_mmu_disable_stall(mmu);
 313
 314         return err;
 315 }
 316
 317 mali_bool mali_mmu_zap_tlb(struct mali_mmu_core *mmu)
 318 {
 319         mali_bool stall_success = mali_mmu_enable_stall(mmu);
 320
 321         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE);
 322
 323         if (MALI_FALSE == stall_success)
 324         {
 325                 /* False means that it is in Pagefault state. Not possible to disable_stall then */
 326                 return MALI_FALSE;
 327         }
 328
 329         mali_mmu_disable_stall(mmu);
 330         return MALI_TRUE;
 331 }
 332
 333 void mali_mmu_zap_tlb_without_stall(struct mali_mmu_core *mmu)
 334 {
 335         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE);
 336 }
 337
 338
 339 void mali_mmu_invalidate_page(struct mali_mmu_core *mmu, u32 mali_address)
 340 {
 341         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_ZAP_ONE_LINE, MALI_MMU_PDE_ENTRY(mali_address));
 342 }
 343
 344 static void mali_mmu_activate_address_space(struct mali_mmu_core *mmu, u32 page_directory)
 345 {
 346         /* The MMU must be in stalled or page fault mode, for this writing to work */
 347         MALI_DEBUG_ASSERT( 0 != ( mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS)
 348                           & (MALI_MMU_STATUS_BIT_STALL_ACTIVE|MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) ) );
 349         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, page_directory);
 350         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE);
 351
 352 }
 353
 354 mali_bool mali_mmu_activate_page_directory(struct mali_mmu_core *mmu, struct mali_page_directory *pagedir)
 355 {
 356         mali_bool stall_success;
 357         MALI_DEBUG_ASSERT_POINTER(mmu);
 358
 359         MALI_DEBUG_PRINT(5, ("Asked to activate page directory 0x%x on MMU %s\n", pagedir, mmu->hw_core.description));
 360         stall_success = mali_mmu_enable_stall(mmu);
 361
 362         if ( MALI_FALSE==stall_success ) return MALI_FALSE;
 363         mali_mmu_activate_address_space(mmu, pagedir->page_directory);
 364         mali_mmu_disable_stall(mmu);
 365         return MALI_TRUE;
 366 }
 367
 368 void mali_mmu_activate_empty_page_directory(struct mali_mmu_core* mmu)
 369 {
 370         mali_bool stall_success;
 371
 372         MALI_DEBUG_ASSERT_POINTER(mmu);
 373         MALI_DEBUG_PRINT(3, ("Activating the empty page directory on MMU %s\n", mmu->hw_core.description));
 374
 375         stall_success = mali_mmu_enable_stall(mmu);
 376         /* This function can only be called when the core is idle, so it could not fail. */
 377         MALI_DEBUG_ASSERT( stall_success );
 378         mali_mmu_activate_address_space(mmu, mali_empty_page_directory);
 379         mali_mmu_disable_stall(mmu);
 380 }
 381
 382 void mali_mmu_activate_fault_flush_page_directory(struct mali_mmu_core* mmu)
 383 {
 384         mali_bool stall_success;
 385         MALI_DEBUG_ASSERT_POINTER(mmu);
 386
 387         MALI_DEBUG_PRINT(3, ("Activating the page fault flush page directory on MMU %s\n", mmu->hw_core.description));
 388         stall_success = mali_mmu_enable_stall(mmu);
 389         /* This function is expect to fail the stalling, since it might be in PageFault mode when it is called */
 390         mali_mmu_activate_address_space(mmu, mali_page_fault_flush_page_directory);
 391         if ( MALI_TRUE==stall_success ) mali_mmu_disable_stall(mmu);
 392 }
 393
 394 /* Is called when we want the mmu to give an interrupt */
 395 static void mali_mmu_probe_trigger(void *data)
 396 {
 397         struct mali_mmu_core *mmu = (struct mali_mmu_core *)data;
 398         mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT, MALI_MMU_INTERRUPT_PAGE_FAULT|MALI_MMU_INTERRUPT_READ_BUS_ERROR);
 399 }
 400
 401 /* Is called when the irq probe wants the mmu to acknowledge an interrupt from the hw */
 402 static _mali_osk_errcode_t mali_mmu_probe_ack(void *data)
 403 {
 404         struct mali_mmu_core *mmu = (struct mali_mmu_core *)data;
 405         u32 int_stat;
 406
 407         int_stat = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_STATUS);
 408
 409         MALI_DEBUG_PRINT(2, ("mali_mmu_probe_irq_acknowledge: intstat 0x%x\n", int_stat));
 410         if (int_stat & MALI_MMU_INTERRUPT_PAGE_FAULT)
 411         {
 412                 MALI_DEBUG_PRINT(2, ("Probe: Page fault detect: PASSED\n"));
 413                 mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_PAGE_FAULT);
 414         }
 415         else
 416         {
 417                 MALI_DEBUG_PRINT(1, ("Probe: Page fault detect: FAILED\n"));
 418         }
 419
 420         if (int_stat & MALI_MMU_INTERRUPT_READ_BUS_ERROR)
 421         {
 422                 MALI_DEBUG_PRINT(2, ("Probe: Bus read error detect: PASSED\n"));
 423                 mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_READ_BUS_ERROR);
 424         }
 425         else
 426         {
 427                 MALI_DEBUG_PRINT(1, ("Probe: Bus read error detect: FAILED\n"));
 428         }
 429
 430         if ( (int_stat & (MALI_MMU_INTERRUPT_PAGE_FAULT|MALI_MMU_INTERRUPT_READ_BUS_ERROR)) ==
 431                          (MALI_MMU_INTERRUPT_PAGE_FAULT|MALI_MMU_INTERRUPT_READ_BUS_ERROR))
 432         {
 433                 return _MALI_OSK_ERR_OK;
 434         }
 435
 436         return _MALI_OSK_ERR_FAULT;
 437 }
 438
 439 #if 0
 440 void mali_mmu_print_state(struct mali_mmu_core *mmu)
 441 {
 442         MALI_DEBUG_PRINT(2, ("MMU: State of %s is 0x%08x\n", mmu->hw_core.description, mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS)));
 443 }
 444 #endif