2 * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
11 #include <linux/fs.h> /* file system operations */
12 #include <asm/uaccess.h> /* user space access */
13 #include <linux/dma-buf.h>
14 #include <linux/scatterlist.h>
15 #include <linux/rbtree.h>
16 #include <linux/platform_device.h>
20 #include "mali_kernel_common.h"
21 #include "mali_session.h"
22 #include "mali_kernel_linux.h"
24 #include "mali_kernel_memory_engine.h"
25 #include "mali_memory.h"
28 struct mali_dma_buf_attachment {
30 struct dma_buf_attachment *attachment;
32 _mali_osk_atomic_t ref;
33 struct rb_node rb_node;
36 static struct rb_root mali_dma_bufs = RB_ROOT;
37 static DEFINE_SPINLOCK(mali_dma_bufs_lock);
39 static inline struct mali_dma_buf_attachment *mali_dma_buf_lookup(struct rb_root *root, struct dma_buf *target)
41 struct rb_node *node = root->rb_node;
42 struct mali_dma_buf_attachment *res;
44 spin_lock(&mali_dma_bufs_lock);
47 res = rb_entry(node, struct mali_dma_buf_attachment, rb_node);
49 if (target < res->buf) node = node->rb_left;
50 else if (target > res->buf) node = node->rb_right;
53 _mali_osk_atomic_inc(&res->ref);
54 spin_unlock(&mali_dma_bufs_lock);
58 spin_unlock(&mali_dma_bufs_lock);
63 static void mali_dma_buf_add(struct rb_root *root, struct mali_dma_buf_attachment *new)
65 struct rb_node **node = &root->rb_node;
66 struct rb_node *parent = NULL;
67 struct mali_dma_buf_attachment *res;
69 spin_lock(&mali_dma_bufs_lock);
73 res = rb_entry(*node, struct mali_dma_buf_attachment, rb_node);
75 if (new->buf < res->buf) node = &(*node)->rb_left;
76 else node = &(*node)->rb_right;
79 rb_link_node(&new->rb_node, parent, node);
80 rb_insert_color(&new->rb_node, &mali_dma_bufs);
82 spin_unlock(&mali_dma_bufs_lock);
88 static void mali_dma_buf_release(void *ctx, void *handle)
90 struct mali_dma_buf_attachment *mem;
93 mem = (struct mali_dma_buf_attachment *)handle;
95 MALI_DEBUG_ASSERT_POINTER(mem);
96 MALI_DEBUG_ASSERT_POINTER(mem->attachment);
97 MALI_DEBUG_ASSERT_POINTER(mem->buf);
99 spin_lock(&mali_dma_bufs_lock);
100 ref = _mali_osk_atomic_dec_return(&mem->ref);
104 rb_erase(&mem->rb_node, &mali_dma_bufs);
105 spin_unlock(&mali_dma_bufs_lock);
107 MALI_DEBUG_ASSERT(0 == _mali_osk_atomic_read(&mem->ref));
109 dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL);
111 dma_buf_detach(mem->buf, mem->attachment);
112 dma_buf_put(mem->buf);
118 spin_unlock(&mali_dma_bufs_lock);
122 /* Callback from memory engine which will map into Mali virtual address space */
123 static mali_physical_memory_allocation_result mali_dma_buf_commit(void* ctx, mali_allocation_engine * engine, mali_memory_allocation * descriptor, u32* offset, mali_physical_memory_allocation * alloc_info)
125 struct mali_session_data *session;
126 struct mali_page_directory *pagedir;
127 struct mali_dma_buf_attachment *mem;
128 struct scatterlist *sg;
132 MALI_DEBUG_ASSERT_POINTER(ctx);
133 MALI_DEBUG_ASSERT_POINTER(engine);
134 MALI_DEBUG_ASSERT_POINTER(descriptor);
135 MALI_DEBUG_ASSERT_POINTER(offset);
136 MALI_DEBUG_ASSERT_POINTER(alloc_info);
138 /* Mapping dma-buf with an offset is not supported. */
139 MALI_DEBUG_ASSERT(0 == *offset);
141 virt = descriptor->mali_address;
142 session = (struct mali_session_data *)descriptor->mali_addr_mapping_info;
143 pagedir = mali_session_get_page_directory(session);
145 MALI_DEBUG_ASSERT_POINTER(session);
147 mem = (struct mali_dma_buf_attachment *)ctx;
149 MALI_DEBUG_ASSERT_POINTER(mem);
151 mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL);
152 if (IS_ERR_OR_NULL(mem->sgt))
154 MALI_PRINT_ERROR(("Failed to map dma-buf attachment\n"));
155 return MALI_MEM_ALLOC_INTERNAL_FAILURE;
158 for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i)
160 u32 size = sg_dma_len(sg);
161 dma_addr_t phys = sg_dma_address(sg);
163 /* sg must be page aligned. */
164 MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE);
166 mali_mmu_pagedir_update(pagedir, virt, phys, size, MALI_CACHE_STANDARD);
172 if (descriptor->flags & MALI_MEMORY_ALLOCATION_FLAG_MAP_GUARD_PAGE)
175 MALI_DEBUG_PRINT(7, ("Mapping in extra guard page\n"));
177 guard_phys = sg_dma_address(mem->sgt->sgl);
178 mali_mmu_pagedir_update(mali_session_get_page_directory(session), virt, guard_phys, MALI_MMU_PAGE_SIZE, MALI_CACHE_STANDARD);
181 MALI_DEBUG_ASSERT(*offset == descriptor->size);
183 alloc_info->ctx = NULL;
184 alloc_info->handle = mem;
185 alloc_info->next = NULL;
186 alloc_info->release = mali_dma_buf_release;
188 return MALI_MEM_ALLOC_FINISHED;
191 int mali_attach_dma_buf(struct mali_session_data *session, _mali_uk_attach_dma_buf_s __user *user_arg)
193 mali_physical_memory_allocator external_memory_allocator;
195 struct mali_dma_buf_attachment *mem;
196 _mali_uk_attach_dma_buf_s args;
197 mali_memory_allocation *descriptor;
201 /* Get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */
202 if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_attach_dma_buf_s)))
210 buf = dma_buf_get(fd);
211 if (IS_ERR_OR_NULL(buf))
213 MALI_DEBUG_PRINT(2, ("Failed to get dma-buf from fd: %d\n", fd));
217 /* Currently, mapping of the full buffer are supported. */
218 if (args.size != buf->size)
220 MALI_DEBUG_PRINT(2, ("dma-buf size doesn't match mapping size.\n"));
226 mem = mali_dma_buf_lookup(&mali_dma_bufs, buf);
229 /* dma-buf is not already attached to Mali */
230 mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment));
233 MALI_PRINT_ERROR(("Failed to allocate dma-buf tracing struct\n"));
237 _mali_osk_atomic_init(&mem->ref, 1);
240 mem->attachment = dma_buf_attach(mem->buf, &mali_platform_device->dev);
241 if (NULL == mem->attachment)
243 MALI_DEBUG_PRINT(2, ("Failed to attach to dma-buf %d\n", fd));
244 dma_buf_put(mem->buf);
249 mali_dma_buf_add(&mali_dma_bufs, mem);
253 /* dma-buf is already attached to Mali */
254 /* Give back the reference we just took, mali_dma_buf_lookup grabbed a new reference for us. */
258 /* Map dma-buf into this session's page tables */
260 /* Set up Mali memory descriptor */
261 descriptor = _mali_osk_calloc(1, sizeof(mali_memory_allocation));
262 if (NULL == descriptor)
264 MALI_PRINT_ERROR(("Failed to allocate descriptor dma-buf %d\n", fd));
265 mali_dma_buf_release(NULL, mem);
269 descriptor->size = args.size;
270 descriptor->mapping = NULL;
271 descriptor->mali_address = args.mali_address;
272 descriptor->mali_addr_mapping_info = (void*)session;
273 descriptor->process_addr_mapping_info = NULL; /* do not map to process address space */
274 descriptor->lock = session->memory_lock;
276 if (args.flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE)
278 descriptor->flags = MALI_MEMORY_ALLOCATION_FLAG_MAP_GUARD_PAGE;
280 _mali_osk_list_init( &descriptor->list );
282 /* Get descriptor mapping for memory. */
283 if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_allocate_mapping(session->descriptor_mapping, descriptor, &md))
285 MALI_PRINT_ERROR(("Failed to create descriptor mapping for dma-buf %d\n", fd));
286 _mali_osk_free(descriptor);
287 mali_dma_buf_release(NULL, mem);
291 external_memory_allocator.allocate = mali_dma_buf_commit;
292 external_memory_allocator.allocate_page_table_block = NULL;
293 external_memory_allocator.ctx = mem;
294 external_memory_allocator.name = "DMA-BUF Memory";
295 external_memory_allocator.next = NULL;
297 /* Map memory into session's Mali virtual address space. */
298 _mali_osk_lock_wait(session->memory_lock, _MALI_OSK_LOCKMODE_RW);
299 if (_MALI_OSK_ERR_OK != mali_allocation_engine_allocate_memory(mali_mem_get_memory_engine(), descriptor, &external_memory_allocator, NULL))
301 _mali_osk_lock_signal(session->memory_lock, _MALI_OSK_LOCKMODE_RW);
303 MALI_PRINT_ERROR(("Failed to map dma-buf %d into Mali address space\n", fd));
304 mali_descriptor_mapping_free(session->descriptor_mapping, md);
305 mali_dma_buf_release(NULL, mem);
308 _mali_osk_lock_signal(session->memory_lock, _MALI_OSK_LOCKMODE_RW);
310 /* Return stuff to user space */
311 if (0 != put_user(md, &user_arg->cookie))
314 MALI_PRINT_ERROR(("Failed to return descriptor to user space for dma-buf %d\n", fd));
315 mali_descriptor_mapping_free(session->descriptor_mapping, md);
316 mali_dma_buf_release(NULL, mem);
323 int mali_release_dma_buf(struct mali_session_data *session, _mali_uk_release_dma_buf_s __user *user_arg)
325 _mali_uk_release_dma_buf_s args;
326 mali_memory_allocation *descriptor;
328 /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */
329 if ( 0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_release_dma_buf_s)) )
334 if (_MALI_OSK_ERR_OK != mali_descriptor_mapping_get(session->descriptor_mapping, args.cookie, (void**)&descriptor))
336 MALI_DEBUG_PRINT(1, ("Invalid memory descriptor %d used to release dma-buf\n", args.cookie));
340 descriptor = mali_descriptor_mapping_free(session->descriptor_mapping, args.cookie);
342 if (NULL != descriptor)
344 _mali_osk_lock_wait( session->memory_lock, _MALI_OSK_LOCKMODE_RW );
346 /* Will call back to mali_dma_buf_release() which will release the dma-buf attachment. */
347 mali_allocation_engine_release_memory(mali_mem_get_memory_engine(), descriptor);
349 _mali_osk_lock_signal( session->memory_lock, _MALI_OSK_LOCKMODE_RW );
351 _mali_osk_free(descriptor);
354 /* Return the error that _mali_ukk_map_external_ump_mem produced */
358 int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg)
360 _mali_uk_dma_buf_get_size_s args;
364 /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */
365 if ( 0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s)) )
370 /* Do DMA-BUF stuff */
373 buf = dma_buf_get(fd);
374 if (IS_ERR_OR_NULL(buf))
376 MALI_DEBUG_PRINT(2, ("Failed to get dma-buf from fd: %d\n", fd));
380 if (0 != put_user(buf->size, &user_arg->size))