drivers/gpu/arm/mali400/mali/common/mali_kernel_mem_os.c

   1 /*
   2  * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_kernel_common.h"
  12 #include "mali_kernel_memory_engine.h"
  13 #include "mali_osk.h"
  14
  15 typedef struct os_allocation
  16 {
  17         u32 num_pages;
  18         u32 offset_start;
  19         mali_allocation_engine * engine;
  20         mali_memory_allocation * descriptor;
  21 } os_allocation;
  22
  23 typedef struct os_allocator
  24 {
  25         _mali_osk_lock_t *mutex;
  26
  27         /**
  28          * Maximum number of pages to allocate from the OS
  29          */
  30         u32 num_pages_max;
  31
  32         /**
  33          * Number of pages allocated from the OS
  34          */
  35         u32 num_pages_allocated;
  36
  37         /** CPU Usage adjustment (add to mali physical address to get cpu physical address) */
  38         u32 cpu_usage_adjust;
  39 } os_allocator;
  40
  41 static mali_physical_memory_allocation_result os_allocator_allocate(void* ctx, mali_allocation_engine * engine,  mali_memory_allocation * descriptor, u32* offset, mali_physical_memory_allocation * alloc_info);
  42 static mali_physical_memory_allocation_result os_allocator_allocate_page_table_block(void * ctx, mali_page_table_block * block);
  43 static void os_allocator_release(void * ctx, void * handle);
  44 static void os_allocator_page_table_block_release( mali_page_table_block *page_table_block );
  45 static void os_allocator_destroy(mali_physical_memory_allocator * allocator);
  46 static u32 os_allocator_stat(mali_physical_memory_allocator * allocator);
  47
  48 mali_physical_memory_allocator * mali_os_allocator_create(u32 max_allocation, u32 cpu_usage_adjust, const char *name)
  49 {
  50         mali_physical_memory_allocator * allocator;
  51         os_allocator * info;
  52
  53         max_allocation = (max_allocation + _MALI_OSK_CPU_PAGE_SIZE-1) & ~(_MALI_OSK_CPU_PAGE_SIZE-1);
  54
  55         MALI_DEBUG_PRINT(2, ("Mali OS memory allocator created with max allocation size of 0x%X bytes, cpu_usage_adjust 0x%08X\n", max_allocation, cpu_usage_adjust));
  56
  57         allocator = _mali_osk_malloc(sizeof(mali_physical_memory_allocator));
  58         if (NULL != allocator)
  59         {
  60                 info = _mali_osk_malloc(sizeof(os_allocator));
  61                 if (NULL != info)
  62                 {
  63                         info->num_pages_max = max_allocation / _MALI_OSK_CPU_PAGE_SIZE;
  64                         info->num_pages_allocated = 0;
  65                         info->cpu_usage_adjust = cpu_usage_adjust;
  66
  67                         info->mutex = _mali_osk_lock_init( _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE | _MALI_OSK_LOCKFLAG_ORDERED, 0, _MALI_OSK_LOCK_ORDER_MEM_INFO);
  68             if (NULL != info->mutex)
  69             {
  70                             allocator->allocate = os_allocator_allocate;
  71                             allocator->allocate_page_table_block = os_allocator_allocate_page_table_block;
  72                             allocator->destroy = os_allocator_destroy;
  73                                 allocator->stat = os_allocator_stat;
  74                             allocator->ctx = info;
  75                                 allocator->name = name;
  76
  77                             return allocator;
  78             }
  79             _mali_osk_free(info);
  80                 }
  81                 _mali_osk_free(allocator);
  82         }
  83
  84         return NULL;
  85 }
  86
  87 static u32 os_allocator_stat(mali_physical_memory_allocator * allocator)
  88 {
  89         os_allocator * info;
  90         info = (os_allocator*)allocator->ctx;
  91         return info->num_pages_allocated * _MALI_OSK_MALI_PAGE_SIZE;
  92 }
  93
  94 static void os_allocator_destroy(mali_physical_memory_allocator * allocator)
  95 {
  96         os_allocator * info;
  97         MALI_DEBUG_ASSERT_POINTER(allocator);
  98         MALI_DEBUG_ASSERT_POINTER(allocator->ctx);
  99         info = (os_allocator*)allocator->ctx;
 100         _mali_osk_lock_term(info->mutex);
 101         _mali_osk_free(info);
 102         _mali_osk_free(allocator);
 103 }
 104
 105 static mali_physical_memory_allocation_result os_allocator_allocate(void* ctx, mali_allocation_engine * engine,  mali_memory_allocation * descriptor, u32* offset, mali_physical_memory_allocation * alloc_info)
 106 {
 107         mali_physical_memory_allocation_result result = MALI_MEM_ALLOC_NONE;
 108         u32 left;
 109         os_allocator * info;
 110         os_allocation * allocation;
 111         int pages_allocated = 0;
 112         _mali_osk_errcode_t err = _MALI_OSK_ERR_OK;
 113
 114         MALI_DEBUG_ASSERT_POINTER(ctx);
 115         MALI_DEBUG_ASSERT_POINTER(engine);
 116         MALI_DEBUG_ASSERT_POINTER(descriptor);
 117         MALI_DEBUG_ASSERT_POINTER(offset);
 118         MALI_DEBUG_ASSERT_POINTER(alloc_info);
 119
 120         info = (os_allocator*)ctx;
 121         left = descriptor->size - *offset;
 122
 123         if (_MALI_OSK_ERR_OK != _mali_osk_lock_wait(info->mutex, _MALI_OSK_LOCKMODE_RW)) return MALI_MEM_ALLOC_INTERNAL_FAILURE;
 124
 125         /** @note this code may not work on Linux, or may require a more complex Linux implementation */
 126         allocation = _mali_osk_malloc(sizeof(os_allocation));
 127         if (NULL != allocation)
 128         {
 129                 u32 os_mem_max_usage = info->num_pages_max * _MALI_OSK_CPU_PAGE_SIZE;
 130                 allocation->offset_start = *offset;
 131                 allocation->num_pages = ((left + _MALI_OSK_CPU_PAGE_SIZE - 1) & ~(_MALI_OSK_CPU_PAGE_SIZE - 1)) >> _MALI_OSK_CPU_PAGE_ORDER;
 132                 MALI_DEBUG_PRINT(6, ("Allocating page array of size %d bytes\n", allocation->num_pages * sizeof(struct page*)));
 133 /* MALI_SEC */
 134 /*
 135 while (left > 0 && ((info->num_pages_allocated + pages_allocated) < info->num_pages_max) && _mali_osk_mem_check_allocated(os_mem_max_usage))
 136 */
 137                 while (left > 0)
 138                 {
 139                         err = mali_allocation_engine_map_physical(engine, descriptor, *offset, MALI_MEMORY_ALLOCATION_OS_ALLOCATED_PHYSADDR_MAGIC, info->cpu_usage_adjust, _MALI_OSK_CPU_PAGE_SIZE);
 140                         if ( _MALI_OSK_ERR_OK != err)
 141                         {
 142                                 if (  _MALI_OSK_ERR_NOMEM == err)
 143                                 {
 144                                         /* 'Partial' allocation (or, out-of-memory on first page) */
 145                                         break;
 146                                 }
 147
 148                                 MALI_DEBUG_PRINT(1, ("Mapping of physical memory failed\n"));
 149
 150                                 /* Fatal error, cleanup any previous pages allocated. */
 151                                 if ( pages_allocated > 0 )
 152                                 {
 153                                         mali_allocation_engine_unmap_physical( engine, descriptor, allocation->offset_start, _MALI_OSK_CPU_PAGE_SIZE*pages_allocated, _MALI_OSK_MEM_MAPREGION_FLAG_OS_ALLOCATED_PHYSADDR );
 154                                         /* (*offset) doesn't need to be restored; it will not be used by the caller on failure */
 155                                 }
 156
 157                                 pages_allocated = 0;
 158
 159                                 result = MALI_MEM_ALLOC_INTERNAL_FAILURE;
 160                                 break;
 161                         }
 162
 163                         /* Loop iteration */
 164                         if (left < _MALI_OSK_CPU_PAGE_SIZE) left = 0;
 165                         else left -= _MALI_OSK_CPU_PAGE_SIZE;
 166
 167                         pages_allocated++;
 168
 169                         *offset += _MALI_OSK_CPU_PAGE_SIZE;
 170                 }
 171
 172                 if (left) MALI_PRINT(("Out of memory. Mali memory allocated: %d kB  Configured maximum OS memory usage: %d kB\n",
 173                                  (info->num_pages_allocated * _MALI_OSK_CPU_PAGE_SIZE)/1024, (info->num_pages_max* _MALI_OSK_CPU_PAGE_SIZE)/1024));
 174
 175                 /* Loop termination; decide on result */
 176                 if (pages_allocated)
 177                 {
 178                         MALI_DEBUG_PRINT(6, ("Allocated %d pages\n", pages_allocated));
 179                         if (left) result = MALI_MEM_ALLOC_PARTIAL;
 180                         else result = MALI_MEM_ALLOC_FINISHED;
 181
 182             /* Some OS do not perform a full cache flush (including all outer caches) for uncached mapped memory.
 183              * They zero the memory through a cached mapping, then flush the inner caches but not the outer caches.
 184              * This is required for MALI to have the correct view of the memory.
 185              */
 186             _mali_osk_cache_ensure_uncached_range_flushed( (void *)descriptor, allocation->offset_start, pages_allocated *_MALI_OSK_CPU_PAGE_SIZE );
 187                         allocation->num_pages = pages_allocated;
 188                         allocation->engine = engine;         /* Necessary to make the engine's unmap call */
 189                         allocation->descriptor = descriptor; /* Necessary to make the engine's unmap call */
 190                         info->num_pages_allocated += pages_allocated;
 191
 192                         MALI_DEBUG_PRINT(6, ("%d out of %d pages now allocated\n", info->num_pages_allocated, info->num_pages_max));
 193
 194                         alloc_info->ctx = info;
 195                         alloc_info->handle = allocation;
 196                         alloc_info->release = os_allocator_release;
 197                 }
 198                 else
 199                 {
 200                         MALI_DEBUG_PRINT(6, ("Releasing pages array due to no pages allocated\n"));
 201                         _mali_osk_free( allocation );
 202                 }
 203         }
 204
 205         _mali_osk_lock_signal(info->mutex, _MALI_OSK_LOCKMODE_RW);
 206
 207         return result;
 208 }
 209
 210 static void os_allocator_release(void * ctx, void * handle)
 211 {
 212         os_allocator * info;
 213         os_allocation * allocation;
 214         mali_allocation_engine * engine;
 215         mali_memory_allocation * descriptor;
 216
 217         MALI_DEBUG_ASSERT_POINTER(ctx);
 218         MALI_DEBUG_ASSERT_POINTER(handle);
 219
 220         info = (os_allocator*)ctx;
 221         allocation = (os_allocation*)handle;
 222         engine = allocation->engine;
 223         descriptor = allocation->descriptor;
 224
 225         MALI_DEBUG_ASSERT_POINTER( engine );
 226         MALI_DEBUG_ASSERT_POINTER( descriptor );
 227
 228         if (_MALI_OSK_ERR_OK != _mali_osk_lock_wait(info->mutex, _MALI_OSK_LOCKMODE_RW))
 229         {
 230                 MALI_DEBUG_PRINT(1, ("allocator release: Failed to get mutex\n"));
 231                 return;
 232         }
 233
 234         MALI_DEBUG_PRINT(6, ("Releasing %d os pages\n", allocation->num_pages));
 235
 236         MALI_DEBUG_ASSERT( allocation->num_pages <= info->num_pages_allocated);
 237         info->num_pages_allocated -= allocation->num_pages;
 238
 239         mali_allocation_engine_unmap_physical( engine, descriptor, allocation->offset_start, _MALI_OSK_CPU_PAGE_SIZE*allocation->num_pages, _MALI_OSK_MEM_MAPREGION_FLAG_OS_ALLOCATED_PHYSADDR );
 240
 241         _mali_osk_lock_signal(info->mutex, _MALI_OSK_LOCKMODE_RW);
 242
 243         _mali_osk_free(allocation);
 244 }
 245
 246 static mali_physical_memory_allocation_result os_allocator_allocate_page_table_block(void * ctx, mali_page_table_block * block)
 247 {
 248 /* MALI_SEC  6->10 */
 249 #ifndef CONFIG_FORCE_MAX_ZONEORDER
 250         int allocation_order = 10;
 251 #else
 252         int allocation_order = CONFIG_FORCE_MAX_ZONEORDER - 1;
 253 #endif
 254         void *virt = NULL;
 255         u32 size = _MALI_OSK_CPU_PAGE_SIZE << allocation_order;
 256         os_allocator * info;
 257
 258         u32 cpu_phys_base;
 259
 260         MALI_DEBUG_ASSERT_POINTER(ctx);
 261         info = (os_allocator*)ctx;
 262
 263         /* Ensure we don't allocate more than we're supposed to from the ctx */
 264         if (_MALI_OSK_ERR_OK != _mali_osk_lock_wait(info->mutex, _MALI_OSK_LOCKMODE_RW)) return MALI_MEM_ALLOC_INTERNAL_FAILURE;
 265
 266         /* if the number of pages to be requested lead to exceeding the memory
 267          * limit in info->num_pages_max, reduce the size that is to be requested. */
 268         while ( (info->num_pages_allocated + (1 << allocation_order) > info->num_pages_max)
 269                 && _mali_osk_mem_check_allocated(info->num_pages_max * _MALI_OSK_CPU_PAGE_SIZE) )
 270         {
 271                 if ( allocation_order > 0 ) {
 272                         --allocation_order;
 273                 } else {
 274                         /* return OOM */
 275                         _mali_osk_lock_signal(info->mutex, _MALI_OSK_LOCKMODE_RW);
 276                         return MALI_MEM_ALLOC_NONE;
 277                 }
 278         }
 279
 280         /* try to allocate 2^(allocation_order) pages, if that fails, try
 281          * allocation_order-1 to allocation_order 0 (inclusive) */
 282         while ( allocation_order >= 0 )
 283         {
 284                 size = _MALI_OSK_CPU_PAGE_SIZE << allocation_order;
 285                 virt = _mali_osk_mem_allocioregion( &cpu_phys_base, size );
 286
 287                 if (NULL != virt) break;
 288
 289                 --allocation_order;
 290         }
 291
 292         if ( NULL == virt )
 293         {
 294                 MALI_DEBUG_PRINT(1, ("Failed to allocate consistent memory. Is CONSISTENT_DMA_SIZE set too low?\n"));
 295                 /* return OOM */
 296                 _mali_osk_lock_signal(info->mutex, _MALI_OSK_LOCKMODE_RW);
 297                 return MALI_MEM_ALLOC_NONE;
 298         }
 299
 300         MALI_DEBUG_PRINT(5, ("os_allocator_allocate_page_table_block: Allocation of order %i succeeded\n",
 301                                 allocation_order));
 302
 303         /* we now know the size of the allocation since we know for what
 304          * allocation_order the allocation succeeded */
 305         size = _MALI_OSK_CPU_PAGE_SIZE << allocation_order;
 306
 307
 308         block->release = os_allocator_page_table_block_release;
 309         block->ctx = ctx;
 310         block->handle = (void*)allocation_order;
 311         block->size = size;
 312         block->phys_base = cpu_phys_base - info->cpu_usage_adjust;
 313         block->mapping = virt;
 314
 315         info->num_pages_allocated += (1 << allocation_order);
 316
 317         _mali_osk_lock_signal(info->mutex, _MALI_OSK_LOCKMODE_RW);
 318
 319         return MALI_MEM_ALLOC_FINISHED;
 320 }
 321
 322 static void os_allocator_page_table_block_release( mali_page_table_block *page_table_block )
 323 {
 324         os_allocator * info;
 325         u32 allocation_order;
 326         u32 pages_allocated;
 327
 328         MALI_DEBUG_ASSERT_POINTER( page_table_block );
 329
 330         info = (os_allocator*)page_table_block->ctx;
 331
 332         MALI_DEBUG_ASSERT_POINTER( info );
 333
 334         allocation_order = (u32)page_table_block->handle;
 335
 336         pages_allocated = 1 << allocation_order;
 337
 338         MALI_DEBUG_ASSERT( pages_allocated * _MALI_OSK_CPU_PAGE_SIZE == page_table_block->size );
 339
 340         if (_MALI_OSK_ERR_OK != _mali_osk_lock_wait(info->mutex, _MALI_OSK_LOCKMODE_RW))
 341         {
 342                 MALI_DEBUG_PRINT(1, ("allocator release: Failed to get mutex\n"));
 343                 return;
 344         }
 345
 346         MALI_DEBUG_ASSERT( pages_allocated <= info->num_pages_allocated);
 347         info->num_pages_allocated -= pages_allocated;
 348
 349         /* Adjust phys_base from mali physical address to CPU physical address */
 350         _mali_osk_mem_freeioregion( page_table_block->phys_base + info->cpu_usage_adjust, page_table_block->size, page_table_block->mapping );
 351
 352         _mali_osk_lock_signal(info->mutex, _MALI_OSK_LOCKMODE_RW);
 353 }