drivers/gpu/arm/mali400/mali/common/mali_kernel_memory_engine.c

   1 /*
   2  * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_kernel_common.h"
  12 #include "mali_kernel_memory_engine.h"
  13 #include "mali_osk.h"
  14 #include "mali_osk_list.h"
  15
  16 typedef struct memory_engine
  17 {
  18         mali_kernel_mem_address_manager * mali_address;
  19         mali_kernel_mem_address_manager * process_address;
  20 } memory_engine;
  21
  22 mali_allocation_engine mali_allocation_engine_create(mali_kernel_mem_address_manager * mali_address_manager, mali_kernel_mem_address_manager * process_address_manager)
  23 {
  24         memory_engine * engine;
  25
  26         /* Mali Address Manager need not support unmap_physical */
  27         MALI_DEBUG_ASSERT_POINTER(mali_address_manager);
  28         MALI_DEBUG_ASSERT_POINTER(mali_address_manager->allocate);
  29         MALI_DEBUG_ASSERT_POINTER(mali_address_manager->release);
  30         MALI_DEBUG_ASSERT_POINTER(mali_address_manager->map_physical);
  31
  32         /* Process Address Manager must support unmap_physical for OS allocation
  33          * error path handling */
  34         MALI_DEBUG_ASSERT_POINTER(process_address_manager);
  35         MALI_DEBUG_ASSERT_POINTER(process_address_manager->allocate);
  36         MALI_DEBUG_ASSERT_POINTER(process_address_manager->release);
  37         MALI_DEBUG_ASSERT_POINTER(process_address_manager->map_physical);
  38         MALI_DEBUG_ASSERT_POINTER(process_address_manager->unmap_physical);
  39
  40
  41         engine = (memory_engine*)_mali_osk_malloc(sizeof(memory_engine));
  42         if (NULL == engine) return NULL;
  43
  44         engine->mali_address = mali_address_manager;
  45         engine->process_address = process_address_manager;
  46
  47         return (mali_allocation_engine)engine;
  48 }
  49
  50 void mali_allocation_engine_destroy(mali_allocation_engine engine)
  51 {
  52         MALI_DEBUG_ASSERT_POINTER(engine);
  53         _mali_osk_free(engine);
  54 }
  55
  56 _mali_osk_errcode_t mali_allocation_engine_allocate_memory(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor, mali_physical_memory_allocator * physical_allocators, _mali_osk_list_t *tracking_list )
  57 {
  58         memory_engine * engine = (memory_engine*)mem_engine;
  59
  60         MALI_DEBUG_ASSERT_POINTER(engine);
  61         MALI_DEBUG_ASSERT_POINTER(descriptor);
  62         MALI_DEBUG_ASSERT_POINTER(physical_allocators);
  63         /* ASSERT that the list member has been initialized, even if it won't be
  64          * used for tracking. We need it to be initialized to see if we need to
  65          * delete it from a list in the release function. */
  66         MALI_DEBUG_ASSERT( NULL != descriptor->list.next && NULL != descriptor->list.prev );
  67
  68         if (_MALI_OSK_ERR_OK == engine->mali_address->allocate(descriptor))
  69         {
  70                 _mali_osk_errcode_t res = _MALI_OSK_ERR_OK;
  71                 if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
  72                 {
  73                         res = engine->process_address->allocate(descriptor);
  74                 }
  75                 if ( _MALI_OSK_ERR_OK == res )
  76                 {
  77                         /* address space setup OK, commit physical memory to the allocation */
  78                         mali_physical_memory_allocator * active_allocator = physical_allocators;
  79                         struct mali_physical_memory_allocation * active_allocation_tracker = &descriptor->physical_allocation;
  80                         u32 offset = 0;
  81
  82                         while ( NULL != active_allocator )
  83                         {
  84                                 switch (active_allocator->allocate(active_allocator->ctx, mem_engine, descriptor, &offset, active_allocation_tracker))
  85                                 {
  86                                         case MALI_MEM_ALLOC_FINISHED:
  87                                                 if ( NULL != tracking_list )
  88                                                 {
  89                                                         /* Insert into the memory session list */
  90                                                         /* ASSERT that it is not already part of a list */
  91                                                         MALI_DEBUG_ASSERT( _mali_osk_list_empty( &descriptor->list ) );
  92                                                         _mali_osk_list_add( &descriptor->list, tracking_list );
  93                                                 }
  94
  95                                                 MALI_SUCCESS; /* all done */
  96                                         case MALI_MEM_ALLOC_NONE:
  97                                                 /* reuse current active_allocation_tracker */
  98                                                 MALI_DEBUG_PRINT( 4, ("Memory Engine Allocate: No allocation on %s, resorting to %s\n",
  99                                                                                           ( active_allocator->name ) ? active_allocator->name : "UNNAMED",
 100                                                                                           ( active_allocator->next ) ? (( active_allocator->next->name )? active_allocator->next->name : "UNNAMED") : "NONE") );
 101                                                 active_allocator = active_allocator->next;
 102                                                 break;
 103                                         case MALI_MEM_ALLOC_PARTIAL:
 104                                                 if (NULL != active_allocator->next)
 105                                                 {
 106                                                         /* need a new allocation tracker */
 107                                                         active_allocation_tracker->next = _mali_osk_calloc(1, sizeof(mali_physical_memory_allocation));
 108                                                         if (NULL != active_allocation_tracker->next)
 109                                                         {
 110                                                                 active_allocation_tracker = active_allocation_tracker->next;
 111                                                                 MALI_DEBUG_PRINT( 2, ("Memory Engine Allocate: Partial allocation on %s, resorting to %s\n",
 112                                                                                                           ( active_allocator->name ) ? active_allocator->name : "UNNAMED",
 113                                                                                                           ( active_allocator->next ) ? (( active_allocator->next->name )? active_allocator->next->name : "UNNAMED") : "NONE") );
 114                                                                 active_allocator = active_allocator->next;
 115                                                                 break;
 116                                                         }
 117                                                 }
 118                                            /* FALL THROUGH */
 119                                         case MALI_MEM_ALLOC_INTERNAL_FAILURE:
 120                                            active_allocator = NULL; /* end the while loop */
 121                                            break;
 122                                 }
 123                         }
 124
 125                         MALI_PRINT(("Memory allocate failed, could not allocate size %d kB.\n", descriptor->size/1024));
 126
 127                         /* allocation failure, start cleanup */
 128                         /* loop over any potential partial allocations */
 129                         active_allocation_tracker = &descriptor->physical_allocation;
 130                         while (NULL != active_allocation_tracker)
 131                         {
 132                                 /* handle blank trackers which will show up during failure */
 133                                 if (NULL != active_allocation_tracker->release)
 134                                 {
 135                                         active_allocation_tracker->release(active_allocation_tracker->ctx, active_allocation_tracker->handle);
 136                                 }
 137                                 active_allocation_tracker = active_allocation_tracker->next;
 138                         }
 139
 140                         /* free the allocation tracker objects themselves, skipping the tracker stored inside the descriptor itself */
 141                         for ( active_allocation_tracker = descriptor->physical_allocation.next; active_allocation_tracker != NULL; )
 142                         {
 143                                 void * buf = active_allocation_tracker;
 144                                 active_allocation_tracker = active_allocation_tracker->next;
 145                                 _mali_osk_free(buf);
 146                         }
 147
 148                         /* release the address spaces */
 149
 150                         if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
 151                         {
 152                                 engine->process_address->release(descriptor);
 153                         }
 154                 }
 155                 engine->mali_address->release(descriptor);
 156         }
 157
 158         MALI_ERROR(_MALI_OSK_ERR_FAULT);
 159 }
 160
 161 void mali_allocation_engine_release_memory(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor)
 162 {
 163         mali_allocation_engine_release_pt1_mali_pagetables_unmap(mem_engine, descriptor);
 164         mali_allocation_engine_release_pt2_physical_memory_free(mem_engine, descriptor);
 165 }
 166
 167 void mali_allocation_engine_release_pt1_mali_pagetables_unmap(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor)
 168 {
 169         memory_engine * engine = (memory_engine*)mem_engine;
 170
 171         MALI_DEBUG_ASSERT_POINTER(engine);
 172         MALI_DEBUG_ASSERT_POINTER(descriptor);
 173
 174         /* Calling: mali_address_manager_release()  */
 175         /* This function is allowed to be called several times, and it only does the release on the first call. */
 176         engine->mali_address->release(descriptor);
 177 }
 178
 179 void mali_allocation_engine_release_pt2_physical_memory_free(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor)
 180 {
 181         memory_engine * engine = (memory_engine*)mem_engine;
 182         mali_physical_memory_allocation * active_allocation_tracker;
 183
 184         /* Remove this from a tracking list in session_data->memory_head */
 185         if ( ! _mali_osk_list_empty( &descriptor->list ) )
 186         {
 187                 _mali_osk_list_del( &descriptor->list );
 188                 /* Clear the list for debug mode, catch use-after-free */
 189                 MALI_DEBUG_CODE( descriptor->list.next = descriptor->list.prev = NULL; )
 190         }
 191
 192         active_allocation_tracker = &descriptor->physical_allocation;
 193         while (NULL != active_allocation_tracker)
 194         {
 195                 MALI_DEBUG_ASSERT_POINTER(active_allocation_tracker->release);
 196                 active_allocation_tracker->release(active_allocation_tracker->ctx, active_allocation_tracker->handle);
 197                 active_allocation_tracker = active_allocation_tracker->next;
 198         }
 199
 200         /* free the allocation tracker objects themselves, skipping the tracker stored inside the descriptor itself */
 201         for ( active_allocation_tracker = descriptor->physical_allocation.next; active_allocation_tracker != NULL; )
 202         {
 203                 void * buf = active_allocation_tracker;
 204                 active_allocation_tracker = active_allocation_tracker->next;
 205                 _mali_osk_free(buf);
 206         }
 207
 208         if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
 209         {
 210                 engine->process_address->release(descriptor);
 211         }
 212 }
 213
 214 _mali_osk_errcode_t mali_allocation_engine_map_physical(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor, u32 offset, u32 phys, u32 cpu_usage_adjust, u32 size)
 215 {
 216         _mali_osk_errcode_t err;
 217         memory_engine * engine = (memory_engine*)mem_engine;
 218         _mali_osk_mem_mapregion_flags_t unmap_flags = (_mali_osk_mem_mapregion_flags_t)0;
 219
 220         MALI_DEBUG_ASSERT_POINTER(engine);
 221         MALI_DEBUG_ASSERT_POINTER(descriptor);
 222
 223         MALI_DEBUG_PRINT(7, ("Mapping phys 0x%08X length 0x%08X at offset 0x%08X\n", phys, size, offset));
 224
 225         MALI_DEBUG_ASSERT_POINTER(engine->mali_address);
 226         MALI_DEBUG_ASSERT_POINTER(engine->mali_address->map_physical);
 227
 228         /* Handle process address manager first, because we may need them to
 229          * allocate the physical page */
 230         if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
 231         {
 232                 /* Handle OS-allocated specially, since an adjustment may be required */
 233                 if ( MALI_MEMORY_ALLOCATION_OS_ALLOCATED_PHYSADDR_MAGIC == phys )
 234                 {
 235                         MALI_DEBUG_ASSERT( _MALI_OSK_CPU_PAGE_SIZE == size );
 236
 237                         /* Set flags to use on error path */
 238                         unmap_flags |= _MALI_OSK_MEM_MAPREGION_FLAG_OS_ALLOCATED_PHYSADDR;
 239
 240                         err = engine->process_address->map_physical(descriptor, offset, &phys, size);
 241                         /* Adjust for cpu physical address to mali physical address */
 242                         phys -= cpu_usage_adjust;
 243                 }
 244                 else
 245                 {
 246                         u32 cpu_phys;
 247                         /* Adjust mali physical address to cpu physical address */
 248                         cpu_phys = phys + cpu_usage_adjust;
 249                         err = engine->process_address->map_physical(descriptor, offset, &cpu_phys, size);
 250                 }
 251
 252                 if ( _MALI_OSK_ERR_OK != err )
 253                 {
 254                         MALI_DEBUG_PRINT(2, ("Map failed: %s %d\n", __FUNCTION__, __LINE__));
 255                         MALI_ERROR( err );
 256                 }
 257         }
 258
 259         MALI_DEBUG_PRINT(7, ("Mapping phys 0x%08X length 0x%08X at offset 0x%08X to CPUVA 0x%08X\n", phys, size, offset, (u32)(descriptor->mapping) + offset));
 260
 261         /* Mali address manager must use the physical address - no point in asking
 262          * it to allocate another one for us */
 263         MALI_DEBUG_ASSERT( MALI_MEMORY_ALLOCATION_OS_ALLOCATED_PHYSADDR_MAGIC != phys );
 264
 265         err = engine->mali_address->map_physical(descriptor, offset, &phys, size);
 266
 267         if ( _MALI_OSK_ERR_OK != err )
 268         {
 269                 if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
 270                 {
 271                         MALI_DEBUG_PRINT( 2, ("Process address manager succeeded, but Mali Address manager failed for phys=0x%08X size=0x%08X, offset=0x%08X. Will unmap.\n", phys, size, offset));
 272                         engine->process_address->unmap_physical(descriptor, offset, size, unmap_flags);
 273                 }
 274                 MALI_DEBUG_PRINT(2, ("Map mali failed: %s %d\n", __FUNCTION__, __LINE__));
 275                 MALI_ERROR( err );
 276         }
 277
 278         MALI_SUCCESS;
 279 }
 280
 281 void mali_allocation_engine_unmap_physical(mali_allocation_engine mem_engine, mali_memory_allocation * descriptor, u32 offset, u32 size, _mali_osk_mem_mapregion_flags_t unmap_flags )
 282 {
 283         memory_engine * engine = (memory_engine*)mem_engine;
 284
 285         MALI_DEBUG_ASSERT_POINTER(engine);
 286         MALI_DEBUG_ASSERT_POINTER(descriptor);
 287
 288         MALI_DEBUG_PRINT(7, ("UnMapping length 0x%08X at offset 0x%08X\n", size, offset));
 289
 290         MALI_DEBUG_ASSERT_POINTER(engine->mali_address);
 291         MALI_DEBUG_ASSERT_POINTER(engine->process_address);
 292
 293         if ( descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_INTO_USERSPACE )
 294         {
 295                 /* Mandetory for process_address manager to have an unmap function*/
 296                 engine->process_address->unmap_physical( descriptor, offset, size, unmap_flags );
 297         }
 298
 299         /* Optional for mali_address manager to have an unmap function*/
 300         if ( NULL != engine->mali_address->unmap_physical )
 301         {
 302                 engine->mali_address->unmap_physical( descriptor, offset, size, unmap_flags );
 303         }
 304 }
 305
 306
 307 _mali_osk_errcode_t mali_allocation_engine_allocate_page_tables(mali_allocation_engine engine, mali_page_table_block * descriptor, mali_physical_memory_allocator * physical_provider)
 308 {
 309         mali_physical_memory_allocator * active_allocator = physical_provider;
 310
 311         MALI_DEBUG_ASSERT_POINTER(descriptor);
 312         MALI_DEBUG_ASSERT_POINTER(physical_provider);
 313
 314         while ( NULL != active_allocator )
 315         {
 316                 switch (active_allocator->allocate_page_table_block(active_allocator->ctx, descriptor))
 317                 {
 318                         case MALI_MEM_ALLOC_FINISHED:
 319                                 MALI_SUCCESS; /* all done */
 320                         case MALI_MEM_ALLOC_NONE:
 321                                 /* try next */
 322                                 MALI_DEBUG_PRINT( 2, ("Memory Engine Allocate PageTables: No allocation on %s, resorting to %s\n",
 323                                                                           ( active_allocator->name ) ? active_allocator->name : "UNNAMED",
 324                                                                           ( active_allocator->next ) ? (( active_allocator->next->name )? active_allocator->next->name : "UNNAMED") : "NONE") );
 325                                 active_allocator = active_allocator->next;
 326                                 break;
 327                         case MALI_MEM_ALLOC_PARTIAL:
 328                                 MALI_DEBUG_PRINT(1, ("Invalid return value from allocate_page_table_block call: MALI_MEM_ALLOC_PARTIAL\n"));
 329                                 /* FALL THROUGH */
 330                         case MALI_MEM_ALLOC_INTERNAL_FAILURE:
 331                                 MALI_DEBUG_PRINT(1, ("Aborting due to allocation failure\n"));
 332                                 active_allocator = NULL; /* end the while loop */
 333                                 break;
 334                 }
 335         }
 336
 337         MALI_ERROR(_MALI_OSK_ERR_FAULT);
 338 }
 339
 340
 341 void mali_allocation_engine_report_allocators( mali_physical_memory_allocator * physical_provider )
 342 {
 343         mali_physical_memory_allocator * active_allocator = physical_provider;
 344         MALI_DEBUG_ASSERT_POINTER(physical_provider);
 345
 346         MALI_DEBUG_PRINT( 1, ("Mali memory allocators will be used in this order of preference (lowest numbered first) :\n"));
 347         while ( NULL != active_allocator )
 348         {
 349                 if ( NULL != active_allocator->name )
 350                 {
 351                         MALI_DEBUG_PRINT( 1, ("\t%d: %s\n", active_allocator->alloc_order, active_allocator->name) );
 352                 }
 353                 else
 354                 {
 355                         MALI_DEBUG_PRINT( 1, ("\t%d: (UNNAMED ALLOCATOR)\n", active_allocator->alloc_order) );
 356                 }
 357                 active_allocator = active_allocator->next;
 358         }
 359
 360 }
 361
 362 u32 mali_allocation_engine_memory_usage(mali_physical_memory_allocator *allocator)
 363 {
 364         u32 sum = 0;
 365         while(NULL != allocator)
 366         {
 367                 /* Only count allocators that have set up a stat function. */
 368                 if(allocator->stat)
 369                         sum += allocator->stat(allocator);
 370
 371                 allocator = allocator->next;
 372         }
 373
 374         return sum;
 375 }