2 * Copyright (C) 2010-2012 ARM Limited. All rights reserved.
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
11 #include <linux/list.h>
13 #include <linux/mm_types.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/slab.h>
17 #include <linux/version.h>
18 #include <linux/platform_device.h>
21 #include "mali_osk_mali.h"
22 #include "mali_kernel_linux.h"
23 #include "mali_scheduler.h"
24 #include "mali_kernel_descriptor_mapping.h"
26 #include "mali_memory.h"
27 #include "mali_memory_dma_buf.h"
28 #include "mali_memory_os_alloc.h"
29 #include "mali_memory_block_alloc.h"
31 /* session->memory_lock must be held when calling this function */
32 static void mali_mem_release(mali_mem_allocation *descriptor)
34 MALI_DEBUG_ASSERT_POINTER(descriptor);
35 MALI_DEBUG_ASSERT_LOCK_HELD(descriptor->session->memory_lock);
37 MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
39 switch (descriptor->type) {
41 mali_mem_os_release(descriptor);
43 case MALI_MEM_DMA_BUF:
44 #if defined(CONFIG_DMA_SHARED_BUFFER)
45 mali_mem_dma_buf_release(descriptor);
49 #if defined(CONFIG_MALI400_UMP)
50 mali_mem_ump_release(descriptor);
53 case MALI_MEM_EXTERNAL:
54 mali_mem_external_release(descriptor);
57 mali_mem_block_release(descriptor);
62 static void mali_mem_vma_open(struct vm_area_struct * vma)
64 mali_mem_allocation *descriptor = (mali_mem_allocation*)vma->vm_private_data;
65 MALI_DEBUG_PRINT(4, ("Open called on vma %p\n", vma));
67 descriptor->cpu_mapping.ref++;
72 static void mali_mem_vma_close(struct vm_area_struct *vma)
74 mali_mem_allocation *descriptor;
75 struct mali_session_data *session;
76 mali_mem_virt_cpu_mapping *mapping;
78 MALI_DEBUG_PRINT(3, ("Close called on vma %p\n", vma));
80 descriptor = (mali_mem_allocation*)vma->vm_private_data;
83 MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
85 mapping = &descriptor->cpu_mapping;
86 BUG_ON(0 == mapping->ref);
89 if (0 != mapping->ref) {
90 MALI_DEBUG_PRINT(3, ("Ignoring this close, %d references still exists\n", mapping->ref));
94 session = descriptor->session;
96 mali_descriptor_mapping_free(session->descriptor_mapping, descriptor->id);
98 _mali_osk_mutex_wait(session->memory_lock);
99 mali_mem_release(descriptor);
100 _mali_osk_mutex_signal(session->memory_lock);
102 mali_mem_descriptor_destroy(descriptor);
105 static int mali_kernel_memory_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf)
107 void __user * address;
108 mali_mem_allocation *descriptor;
110 address = vmf->virtual_address;
111 descriptor = (mali_mem_allocation *)vma->vm_private_data;
113 MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
116 * We always fail the call since all memory is pre-faulted when assigned to the process.
117 * Only the Mali cores can use page faults to extend buffers.
120 MALI_DEBUG_PRINT(1, ("Page-fault in Mali memory region caused by the CPU.\n"));
121 MALI_DEBUG_PRINT(1, ("Tried to access %p (process local virtual address) which is not currently mapped to any Mali memory.\n", (void*)address));
123 MALI_IGNORE(address);
124 MALI_IGNORE(descriptor);
126 return VM_FAULT_SIGBUS;
129 struct vm_operations_struct mali_kernel_vm_ops = {
130 .open = mali_mem_vma_open,
131 .close = mali_mem_vma_close,
132 .fault = mali_kernel_memory_cpu_page_fault_handler
135 /** @note munmap handler is done by vma close handler */
136 int mali_mmap(struct file *filp, struct vm_area_struct *vma)
138 struct mali_session_data *session;
139 mali_mem_allocation *descriptor;
140 u32 size = vma->vm_end - vma->vm_start;
141 u32 mali_addr = vma->vm_pgoff << PAGE_SHIFT;
143 session = (struct mali_session_data *)filp->private_data;
144 if (NULL == session) {
145 MALI_PRINT_ERROR(("mmap called without any session data available\n"));
149 MALI_DEBUG_PRINT(4, ("MMap() handler: start=0x%08X, phys=0x%08X, size=0x%08X vma->flags 0x%08x\n",
150 (unsigned int)vma->vm_start, (unsigned int)(vma->vm_pgoff << PAGE_SHIFT),
151 (unsigned int)(vma->vm_end - vma->vm_start), vma->vm_flags));
153 /* Set some bits which indicate that, the memory is IO memory, meaning
154 * that no paging is to be performed and the memory should not be
155 * included in crash dumps. And that the memory is reserved, meaning
156 * that it's present and can never be paged out (see also previous
159 vma->vm_flags |= VM_IO;
160 vma->vm_flags |= VM_DONTCOPY;
161 vma->vm_flags |= VM_PFNMAP;
162 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0)
163 vma->vm_flags |= VM_RESERVED;
165 vma->vm_flags |= VM_DONTDUMP;
166 vma->vm_flags |= VM_DONTEXPAND;
169 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
170 vma->vm_ops = &mali_kernel_vm_ops; /* Operations used on any memory system */
172 descriptor = mali_mem_block_alloc(mali_addr, size, vma, session);
173 if (NULL == descriptor) {
174 descriptor = mali_mem_os_alloc(mali_addr, size, vma, session);
175 if (NULL == descriptor) {
176 MALI_DEBUG_PRINT(3, ("MMAP failed\n"));
181 MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
183 vma->vm_private_data = (void*)descriptor;
185 /* Put on descriptor map */
186 if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_allocate_mapping(session->descriptor_mapping, descriptor, &descriptor->id)) {
187 _mali_osk_mutex_wait(session->memory_lock);
188 mali_mem_os_release(descriptor);
189 _mali_osk_mutex_signal(session->memory_lock);
197 /* Prepare memory descriptor */
198 mali_mem_allocation *mali_mem_descriptor_create(struct mali_session_data *session, mali_mem_type type)
200 mali_mem_allocation *descriptor;
202 descriptor = (mali_mem_allocation*)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL);
203 if (NULL == descriptor) {
204 MALI_DEBUG_PRINT(3,("mali_ukk_mem_mmap: descriptor was NULL\n"));
208 MALI_DEBUG_CODE(descriptor->magic = MALI_MEM_ALLOCATION_VALID_MAGIC);
210 descriptor->flags = 0;
211 descriptor->type = type;
212 descriptor->session = session;
217 void mali_mem_descriptor_destroy(mali_mem_allocation *descriptor)
219 MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
220 MALI_DEBUG_CODE(descriptor->magic = MALI_MEM_ALLOCATION_FREED_MAGIC);
225 _mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor)
227 u32 size = descriptor->size;
228 struct mali_session_data *session = descriptor->session;
230 MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
232 /* Map dma-buf into this session's page tables */
234 if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
235 size += MALI_MMU_PAGE_SIZE;
238 return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_mapping.addr, size);
241 void mali_mem_mali_map_free(mali_mem_allocation *descriptor)
243 u32 size = descriptor->size;
244 struct mali_session_data *session = descriptor->session;
246 MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic);
248 if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) {
249 size += MALI_MMU_PAGE_SIZE;
252 /* Umap and flush L2 */
253 mali_mmu_pagedir_unmap(session->page_directory, descriptor->mali_mapping.addr, descriptor->size);
255 mali_scheduler_zap_all_active(session);
258 u32 _mali_ukk_report_memory_usage(void)
262 sum += mali_mem_block_allocator_stat();
263 sum += mali_mem_os_stat();
269 * Per-session memory descriptor mapping table sizes
271 #define MALI_MEM_DESCRIPTORS_INIT 64
272 #define MALI_MEM_DESCRIPTORS_MAX 65536
274 _mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data * session_data)
276 MALI_DEBUG_PRINT(5, ("Memory session begin\n"));
278 /* Create descriptor mapping table */
279 session_data->descriptor_mapping = mali_descriptor_mapping_create(MALI_MEM_DESCRIPTORS_INIT, MALI_MEM_DESCRIPTORS_MAX);
281 if (NULL == session_data->descriptor_mapping) {
282 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
285 session_data->memory_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED,
286 _MALI_OSK_LOCK_ORDER_MEM_SESSION);
288 if (NULL == session_data->memory_lock) {
289 mali_descriptor_mapping_destroy(session_data->descriptor_mapping);
290 _mali_osk_free(session_data);
291 MALI_ERROR(_MALI_OSK_ERR_FAULT);
294 MALI_DEBUG_PRINT(5, ("MMU session begin: success\n"));
298 /** @brief Callback function that releases memory
300 * session->memory_lock must be held when calling this function.
302 static void descriptor_table_cleanup_callback(int descriptor_id, void* map_target)
304 mali_mem_allocation *descriptor;
306 descriptor = (mali_mem_allocation*)map_target;
308 MALI_DEBUG_ASSERT_LOCK_HELD(descriptor->session->memory_lock);
310 MALI_DEBUG_PRINT(3, ("Cleanup of descriptor %d mapping to 0x%x in descriptor table\n", descriptor_id, map_target));
311 MALI_DEBUG_ASSERT(descriptor);
313 mali_mem_release(descriptor);
314 mali_mem_descriptor_destroy(descriptor);
317 void mali_memory_session_end(struct mali_session_data *session)
319 MALI_DEBUG_PRINT(3, ("MMU session end\n"));
321 if (NULL == session) {
322 MALI_DEBUG_PRINT(1, ("No session data found during session end\n"));
326 /* Lock the session so we can modify the memory list */
327 _mali_osk_mutex_wait(session->memory_lock);
329 /* Free all allocations still in the descriptor map, and terminate the map */
330 if (NULL != session->descriptor_mapping) {
331 mali_descriptor_mapping_call_for_each(session->descriptor_mapping, descriptor_table_cleanup_callback);
332 mali_descriptor_mapping_destroy(session->descriptor_mapping);
333 session->descriptor_mapping = NULL;
336 _mali_osk_mutex_signal(session->memory_lock);
339 _mali_osk_mutex_term(session->memory_lock);
344 _mali_osk_errcode_t mali_memory_initialize(void)
346 return mali_mem_os_init();
349 void mali_memory_terminate(void)
352 mali_mem_block_allocator_destroy(NULL);