1 /**************************************************************************
3 * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
34 * Locking may look a bit complicated but isn't really:
36 * The buffer usage atomic_t needs to be protected by dev->struct_mutex
37 * when there is a chance that it can be zero before or after the operation.
39 * dev->struct_mutex also protects all lists and list heads,
40 * Hash tables and hash heads.
42 * bo->mutex protects the buffer object itself excluding the usage field.
43 * bo->mutex does also protect the buffer list heads, so to manipulate those,
44 * we need both the bo->mutex and the dev->struct_mutex.
46 * Locking order is bo->mutex, dev->struct_mutex. Therefore list traversal
47 * is a bit complicated. When dev->struct_mutex is released to grab bo->mutex,
48 * the list traversal will, in general, need to be restarted.
52 static void drm_bo_destroy_locked(struct drm_buffer_object *bo);
53 static int drm_bo_setup_vm_locked(struct drm_buffer_object *bo);
54 static void drm_bo_takedown_vm_locked(struct drm_buffer_object *bo);
55 static void drm_bo_unmap_virtual(struct drm_buffer_object *bo);
57 static inline uint64_t drm_bo_type_flags(unsigned type)
59 return (1ULL << (24 + type));
63 * bo locked. dev->struct_mutex locked.
66 void drm_bo_add_to_pinned_lru(struct drm_buffer_object *bo)
68 struct drm_mem_type_manager *man;
70 DRM_ASSERT_LOCKED(&bo->dev->struct_mutex);
71 DRM_ASSERT_LOCKED(&bo->mutex);
73 man = &bo->dev->bm.man[bo->pinned_mem_type];
74 list_add_tail(&bo->pinned_lru, &man->pinned);
77 void drm_bo_add_to_lru(struct drm_buffer_object *bo)
79 struct drm_mem_type_manager *man;
81 DRM_ASSERT_LOCKED(&bo->dev->struct_mutex);
83 if (!(bo->mem.proposed_flags & (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT))
84 || bo->mem.mem_type != bo->pinned_mem_type) {
85 man = &bo->dev->bm.man[bo->mem.mem_type];
86 list_add_tail(&bo->lru, &man->lru);
88 INIT_LIST_HEAD(&bo->lru);
92 static int drm_bo_vm_pre_move(struct drm_buffer_object *bo, int old_is_pci)
94 #ifdef DRM_ODD_MM_COMPAT
97 if (!bo->map_list.map)
100 ret = drm_bo_lock_kmm(bo);
103 drm_bo_unmap_virtual(bo);
105 drm_bo_finish_unmap(bo);
107 if (!bo->map_list.map)
110 drm_bo_unmap_virtual(bo);
115 static void drm_bo_vm_post_move(struct drm_buffer_object *bo)
117 #ifdef DRM_ODD_MM_COMPAT
120 if (!bo->map_list.map)
123 ret = drm_bo_remap_bound(bo);
125 DRM_ERROR("Failed to remap a bound buffer object.\n"
126 "\tThis might cause a sigbus later.\n");
128 drm_bo_unlock_kmm(bo);
133 * Call bo->mutex locked.
136 static int drm_bo_add_ttm(struct drm_buffer_object *bo)
138 struct drm_device *dev = bo->dev;
140 uint32_t page_flags = 0;
142 DRM_ASSERT_LOCKED(&bo->mutex);
145 if (bo->mem.proposed_flags & DRM_BO_FLAG_WRITE)
146 page_flags |= DRM_TTM_PAGE_WRITE;
149 case drm_bo_type_device:
150 case drm_bo_type_kernel:
151 bo->ttm = drm_ttm_create(dev, bo->num_pages << PAGE_SHIFT,
152 page_flags, dev->bm.dummy_read_page);
156 case drm_bo_type_user:
157 bo->ttm = drm_ttm_create(dev, bo->num_pages << PAGE_SHIFT,
158 page_flags | DRM_TTM_PAGE_USER,
159 dev->bm.dummy_read_page);
163 ret = drm_ttm_set_user(bo->ttm, current,
171 DRM_ERROR("Illegal buffer object type\n");
179 static int drm_bo_handle_move_mem(struct drm_buffer_object *bo,
180 struct drm_bo_mem_reg *mem,
181 int evict, int no_wait)
183 struct drm_device *dev = bo->dev;
184 struct drm_buffer_manager *bm = &dev->bm;
185 int old_is_pci = drm_mem_reg_is_pci(dev, &bo->mem);
186 int new_is_pci = drm_mem_reg_is_pci(dev, mem);
187 struct drm_mem_type_manager *old_man = &bm->man[bo->mem.mem_type];
188 struct drm_mem_type_manager *new_man = &bm->man[mem->mem_type];
191 if (old_is_pci || new_is_pci ||
192 ((mem->flags ^ bo->mem.flags) & DRM_BO_FLAG_CACHED))
193 ret = drm_bo_vm_pre_move(bo, old_is_pci);
198 * Create and bind a ttm if required.
201 if (!(new_man->flags & _DRM_FLAG_MEMTYPE_FIXED) && (bo->ttm == NULL)) {
202 ret = drm_bo_add_ttm(bo);
206 if (mem->mem_type != DRM_BO_MEM_LOCAL) {
207 ret = drm_ttm_bind(bo->ttm, mem);
212 if (bo->mem.mem_type == DRM_BO_MEM_LOCAL) {
214 struct drm_bo_mem_reg *old_mem = &bo->mem;
215 uint64_t save_flags = old_mem->flags;
216 uint64_t save_proposed_flags = old_mem->proposed_flags;
220 old_mem->proposed_flags = save_proposed_flags;
221 DRM_FLAG_MASKED(save_flags, mem->flags,
222 DRM_BO_MASK_MEMTYPE);
228 if (!(old_man->flags & _DRM_FLAG_MEMTYPE_FIXED) &&
229 !(new_man->flags & _DRM_FLAG_MEMTYPE_FIXED))
230 ret = drm_bo_move_ttm(bo, evict, no_wait, mem);
231 else if (dev->driver->bo_driver->move)
232 ret = dev->driver->bo_driver->move(bo, evict, no_wait, mem);
234 ret = drm_bo_move_memcpy(bo, evict, no_wait, mem);
240 if (old_is_pci || new_is_pci)
241 drm_bo_vm_post_move(bo);
243 if (bo->priv_flags & _DRM_BO_FLAG_EVICTED) {
245 dev->driver->bo_driver->invalidate_caches(dev,
248 DRM_ERROR("Can not flush read caches\n");
251 DRM_FLAG_MASKED(bo->priv_flags,
252 (evict) ? _DRM_BO_FLAG_EVICTED : 0,
253 _DRM_BO_FLAG_EVICTED);
256 bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) +
257 bm->man[bo->mem.mem_type].gpu_offset;
263 if (old_is_pci || new_is_pci)
264 drm_bo_vm_post_move(bo);
266 new_man = &bm->man[bo->mem.mem_type];
267 if ((new_man->flags & _DRM_FLAG_MEMTYPE_FIXED) && bo->ttm) {
268 drm_ttm_unbind(bo->ttm);
269 drm_ttm_destroy(bo->ttm);
277 * Call bo->mutex locked.
278 * Returns -EBUSY if the buffer is currently rendered to or from. 0 otherwise.
281 static int drm_bo_busy(struct drm_buffer_object *bo, int check_unfenced)
283 struct drm_fence_object *fence = bo->fence;
285 if (check_unfenced && (bo->priv_flags & _DRM_BO_FLAG_UNFENCED))
289 if (drm_fence_object_signaled(fence, bo->fence_type)) {
290 drm_fence_usage_deref_unlocked(&bo->fence);
293 drm_fence_object_flush(fence, DRM_FENCE_TYPE_EXE);
294 if (drm_fence_object_signaled(fence, bo->fence_type)) {
295 drm_fence_usage_deref_unlocked(&bo->fence);
303 static int drm_bo_check_unfenced(struct drm_buffer_object *bo)
307 mutex_lock(&bo->mutex);
308 ret = (bo->priv_flags & _DRM_BO_FLAG_UNFENCED);
309 mutex_unlock(&bo->mutex);
315 * Call bo->mutex locked.
316 * Wait until the buffer is idle.
319 int drm_bo_wait(struct drm_buffer_object *bo, int lazy, int interruptible,
320 int no_wait, int check_unfenced)
324 DRM_ASSERT_LOCKED(&bo->mutex);
325 while(unlikely(drm_bo_busy(bo, check_unfenced))) {
329 if (check_unfenced && (bo->priv_flags & _DRM_BO_FLAG_UNFENCED)) {
330 mutex_unlock(&bo->mutex);
331 wait_event(bo->event_queue, !drm_bo_check_unfenced(bo));
332 mutex_lock(&bo->mutex);
333 bo->priv_flags |= _DRM_BO_FLAG_UNLOCKED;
337 struct drm_fence_object *fence;
338 uint32_t fence_type = bo->fence_type;
340 drm_fence_reference_unlocked(&fence, bo->fence);
341 mutex_unlock(&bo->mutex);
343 ret = drm_fence_object_wait(fence, lazy, !interruptible,
346 drm_fence_usage_deref_unlocked(&fence);
347 mutex_lock(&bo->mutex);
348 bo->priv_flags |= _DRM_BO_FLAG_UNLOCKED;
356 EXPORT_SYMBOL(drm_bo_wait);
358 static int drm_bo_expire_fence(struct drm_buffer_object *bo, int allow_errors)
360 struct drm_device *dev = bo->dev;
361 struct drm_buffer_manager *bm = &dev->bm;
365 unsigned long _end = jiffies + 3 * DRM_HZ;
368 ret = drm_bo_wait(bo, 0, 0, 0, 0);
369 if (ret && allow_errors)
372 } while (ret && !time_after_eq(jiffies, _end));
376 DRM_ERROR("Detected GPU lockup or "
377 "fence driver was taken down. "
378 "Evicting buffer.\n");
382 drm_fence_usage_deref_unlocked(&bo->fence);
388 * Call dev->struct_mutex locked.
389 * Attempts to remove all private references to a buffer by expiring its
390 * fence object and removing from lru lists and memory managers.
393 static void drm_bo_cleanup_refs(struct drm_buffer_object *bo, int remove_all)
395 struct drm_device *dev = bo->dev;
396 struct drm_buffer_manager *bm = &dev->bm;
398 DRM_ASSERT_LOCKED(&dev->struct_mutex);
400 atomic_inc(&bo->usage);
401 mutex_unlock(&dev->struct_mutex);
402 mutex_lock(&bo->mutex);
404 DRM_FLAG_MASKED(bo->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
406 if (bo->fence && drm_fence_object_signaled(bo->fence,
408 drm_fence_usage_deref_unlocked(&bo->fence);
410 if (bo->fence && remove_all)
411 (void)drm_bo_expire_fence(bo, 0);
413 mutex_lock(&dev->struct_mutex);
415 if (!atomic_dec_and_test(&bo->usage))
419 list_del_init(&bo->lru);
420 if (bo->mem.mm_node) {
421 drm_mm_put_block(bo->mem.mm_node);
422 if (bo->pinned_node == bo->mem.mm_node)
423 bo->pinned_node = NULL;
424 bo->mem.mm_node = NULL;
426 list_del_init(&bo->pinned_lru);
427 if (bo->pinned_node) {
428 drm_mm_put_block(bo->pinned_node);
429 bo->pinned_node = NULL;
431 list_del_init(&bo->ddestroy);
432 mutex_unlock(&bo->mutex);
433 drm_bo_destroy_locked(bo);
437 if (list_empty(&bo->ddestroy)) {
438 drm_fence_object_flush(bo->fence, bo->fence_type);
439 list_add_tail(&bo->ddestroy, &bm->ddestroy);
440 schedule_delayed_work(&bm->wq,
441 ((DRM_HZ / 100) < 1) ? 1 : DRM_HZ / 100);
445 mutex_unlock(&bo->mutex);
450 * Verify that refcount is 0 and that there are no internal references
451 * to the buffer object. Then destroy it.
454 static void drm_bo_destroy_locked(struct drm_buffer_object *bo)
456 struct drm_device *dev = bo->dev;
457 struct drm_buffer_manager *bm = &dev->bm;
459 DRM_ASSERT_LOCKED(&dev->struct_mutex);
461 if (list_empty(&bo->lru) && bo->mem.mm_node == NULL &&
462 list_empty(&bo->pinned_lru) && bo->pinned_node == NULL &&
463 list_empty(&bo->ddestroy) && atomic_read(&bo->usage) == 0) {
464 if (bo->fence != NULL) {
465 DRM_ERROR("Fence was non-zero.\n");
466 drm_bo_cleanup_refs(bo, 0);
470 #ifdef DRM_ODD_MM_COMPAT
471 BUG_ON(!list_empty(&bo->vma_list));
472 BUG_ON(!list_empty(&bo->p_mm_list));
476 drm_ttm_unbind(bo->ttm);
477 drm_ttm_destroy(bo->ttm);
481 atomic_dec(&bm->count);
483 drm_ctl_free(bo, sizeof(*bo), DRM_MEM_BUFOBJ);
489 * Some stuff is still trying to reference the buffer object.
490 * Get rid of those references.
493 drm_bo_cleanup_refs(bo, 0);
499 * Call dev->struct_mutex locked.
502 static void drm_bo_delayed_delete(struct drm_device *dev, int remove_all)
504 struct drm_buffer_manager *bm = &dev->bm;
506 struct drm_buffer_object *entry, *nentry;
507 struct list_head *list, *next;
509 list_for_each_safe(list, next, &bm->ddestroy) {
510 entry = list_entry(list, struct drm_buffer_object, ddestroy);
513 if (next != &bm->ddestroy) {
514 nentry = list_entry(next, struct drm_buffer_object,
516 atomic_inc(&nentry->usage);
519 drm_bo_cleanup_refs(entry, remove_all);
522 atomic_dec(&nentry->usage);
526 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
527 static void drm_bo_delayed_workqueue(void *data)
529 static void drm_bo_delayed_workqueue(struct work_struct *work)
532 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
533 struct drm_device *dev = (struct drm_device *) data;
534 struct drm_buffer_manager *bm = &dev->bm;
536 struct drm_buffer_manager *bm =
537 container_of(work, struct drm_buffer_manager, wq.work);
538 struct drm_device *dev = container_of(bm, struct drm_device, bm);
541 DRM_DEBUG("Delayed delete Worker\n");
543 mutex_lock(&dev->struct_mutex);
544 if (!bm->initialized) {
545 mutex_unlock(&dev->struct_mutex);
548 drm_bo_delayed_delete(dev, 0);
549 if (bm->initialized && !list_empty(&bm->ddestroy)) {
550 schedule_delayed_work(&bm->wq,
551 ((DRM_HZ / 100) < 1) ? 1 : DRM_HZ / 100);
553 mutex_unlock(&dev->struct_mutex);
556 void drm_bo_usage_deref_locked(struct drm_buffer_object **bo)
558 struct drm_buffer_object *tmp_bo = *bo;
561 DRM_ASSERT_LOCKED(&tmp_bo->dev->struct_mutex);
563 if (atomic_dec_and_test(&tmp_bo->usage))
564 drm_bo_destroy_locked(tmp_bo);
566 EXPORT_SYMBOL(drm_bo_usage_deref_locked);
568 static void drm_bo_base_deref_locked(struct drm_file *file_priv,
569 struct drm_user_object *uo)
571 struct drm_buffer_object *bo =
572 drm_user_object_entry(uo, struct drm_buffer_object, base);
574 DRM_ASSERT_LOCKED(&bo->dev->struct_mutex);
576 drm_bo_takedown_vm_locked(bo);
577 drm_bo_usage_deref_locked(&bo);
580 void drm_bo_usage_deref_unlocked(struct drm_buffer_object **bo)
582 struct drm_buffer_object *tmp_bo = *bo;
583 struct drm_device *dev = tmp_bo->dev;
586 if (atomic_dec_and_test(&tmp_bo->usage)) {
587 mutex_lock(&dev->struct_mutex);
588 if (atomic_read(&tmp_bo->usage) == 0)
589 drm_bo_destroy_locked(tmp_bo);
590 mutex_unlock(&dev->struct_mutex);
593 EXPORT_SYMBOL(drm_bo_usage_deref_unlocked);
595 void drm_putback_buffer_objects(struct drm_device *dev)
597 struct drm_buffer_manager *bm = &dev->bm;
598 struct list_head *list = &bm->unfenced;
599 struct drm_buffer_object *entry, *next;
601 mutex_lock(&dev->struct_mutex);
602 list_for_each_entry_safe(entry, next, list, lru) {
603 atomic_inc(&entry->usage);
604 mutex_unlock(&dev->struct_mutex);
606 mutex_lock(&entry->mutex);
607 BUG_ON(!(entry->priv_flags & _DRM_BO_FLAG_UNFENCED));
608 mutex_lock(&dev->struct_mutex);
610 list_del_init(&entry->lru);
611 DRM_FLAG_MASKED(entry->priv_flags, 0, _DRM_BO_FLAG_UNFENCED);
612 wake_up_all(&entry->event_queue);
615 * FIXME: Might want to put back on head of list
616 * instead of tail here.
619 drm_bo_add_to_lru(entry);
620 mutex_unlock(&entry->mutex);
621 drm_bo_usage_deref_locked(&entry);
623 mutex_unlock(&dev->struct_mutex);
625 EXPORT_SYMBOL(drm_putback_buffer_objects);
629 * Note. The caller has to register (if applicable)
630 * and deregister fence object usage.
633 int drm_fence_buffer_objects(struct drm_device *dev,
634 struct list_head *list,
635 uint32_t fence_flags,
636 struct drm_fence_object *fence,
637 struct drm_fence_object **used_fence)
639 struct drm_buffer_manager *bm = &dev->bm;
640 struct drm_buffer_object *entry;
641 uint32_t fence_type = 0;
642 uint32_t fence_class = ~0;
647 mutex_lock(&dev->struct_mutex);
650 list = &bm->unfenced;
653 fence_class = fence->fence_class;
655 list_for_each_entry(entry, list, lru) {
656 BUG_ON(!(entry->priv_flags & _DRM_BO_FLAG_UNFENCED));
657 fence_type |= entry->new_fence_type;
658 if (fence_class == ~0)
659 fence_class = entry->new_fence_class;
660 else if (entry->new_fence_class != fence_class) {
661 DRM_ERROR("Unmatching fence classes on unfenced list: "
664 entry->new_fence_class);
677 if ((fence_type & fence->type) != fence_type ||
678 (fence->fence_class != fence_class)) {
679 DRM_ERROR("Given fence doesn't match buffers "
680 "on unfenced list.\n");
685 mutex_unlock(&dev->struct_mutex);
686 ret = drm_fence_object_create(dev, fence_class, fence_type,
687 fence_flags | DRM_FENCE_FLAG_EMIT,
689 mutex_lock(&dev->struct_mutex);
698 entry = list_entry(l, struct drm_buffer_object, lru);
699 atomic_inc(&entry->usage);
700 mutex_unlock(&dev->struct_mutex);
701 mutex_lock(&entry->mutex);
702 mutex_lock(&dev->struct_mutex);
704 if (entry->priv_flags & _DRM_BO_FLAG_UNFENCED) {
707 drm_fence_usage_deref_locked(&entry->fence);
708 entry->fence = drm_fence_reference_locked(fence);
709 entry->fence_class = entry->new_fence_class;
710 entry->fence_type = entry->new_fence_type;
711 DRM_FLAG_MASKED(entry->priv_flags, 0,
712 _DRM_BO_FLAG_UNFENCED);
713 wake_up_all(&entry->event_queue);
714 drm_bo_add_to_lru(entry);
716 mutex_unlock(&entry->mutex);
717 drm_bo_usage_deref_locked(&entry);
720 DRM_DEBUG("Fenced %d buffers\n", count);
722 mutex_unlock(&dev->struct_mutex);
726 EXPORT_SYMBOL(drm_fence_buffer_objects);
732 static int drm_bo_evict(struct drm_buffer_object *bo, unsigned mem_type,
736 struct drm_device *dev = bo->dev;
737 struct drm_bo_mem_reg evict_mem;
740 * Someone might have modified the buffer before we took the
745 bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED;
747 if (unlikely(bo->mem.flags &
748 (DRM_BO_FLAG_NO_MOVE | DRM_BO_FLAG_NO_EVICT)))
750 if (unlikely(bo->priv_flags & _DRM_BO_FLAG_UNFENCED))
752 if (unlikely(bo->mem.mem_type != mem_type))
754 ret = drm_bo_wait(bo, 0, 1, no_wait, 0);
758 } while(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED);
761 evict_mem.mm_node = NULL;
764 evict_mem.proposed_flags = dev->driver->bo_driver->evict_flags(bo);
766 mutex_lock(&dev->struct_mutex);
767 list_del_init(&bo->lru);
768 mutex_unlock(&dev->struct_mutex);
770 ret = drm_bo_mem_space(bo, &evict_mem, no_wait);
774 DRM_ERROR("Failed to find memory space for "
775 "buffer 0x%p eviction.\n", bo);
779 ret = drm_bo_handle_move_mem(bo, &evict_mem, 1, no_wait);
783 DRM_ERROR("Buffer eviction failed\n");
787 DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_EVICTED,
788 _DRM_BO_FLAG_EVICTED);
791 mutex_lock(&dev->struct_mutex);
792 if (evict_mem.mm_node) {
793 if (evict_mem.mm_node != bo->pinned_node)
794 drm_mm_put_block(evict_mem.mm_node);
795 evict_mem.mm_node = NULL;
797 drm_bo_add_to_lru(bo);
798 BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED);
800 mutex_unlock(&dev->struct_mutex);
806 * Repeatedly evict memory from the LRU for @mem_type until we create enough
807 * space, or we've evicted everything and there isn't enough space.
809 static int drm_bo_mem_force_space(struct drm_device *dev,
810 struct drm_bo_mem_reg *mem,
811 uint32_t mem_type, int no_wait)
813 struct drm_mm_node *node;
814 struct drm_buffer_manager *bm = &dev->bm;
815 struct drm_buffer_object *entry;
816 struct drm_mem_type_manager *man = &bm->man[mem_type];
817 struct list_head *lru;
818 unsigned long num_pages = mem->num_pages;
821 mutex_lock(&dev->struct_mutex);
823 node = drm_mm_search_free(&man->manager, num_pages,
824 mem->page_alignment, 1);
829 if (lru->next == lru)
832 entry = list_entry(lru->next, struct drm_buffer_object, lru);
833 atomic_inc(&entry->usage);
834 mutex_unlock(&dev->struct_mutex);
835 mutex_lock(&entry->mutex);
836 ret = drm_bo_evict(entry, mem_type, no_wait);
837 mutex_unlock(&entry->mutex);
838 drm_bo_usage_deref_unlocked(&entry);
841 mutex_lock(&dev->struct_mutex);
845 mutex_unlock(&dev->struct_mutex);
849 node = drm_mm_get_block(node, num_pages, mem->page_alignment);
850 if (unlikely(!node)) {
851 mutex_unlock(&dev->struct_mutex);
855 mutex_unlock(&dev->struct_mutex);
857 mem->mem_type = mem_type;
861 static int drm_bo_mt_compatible(struct drm_mem_type_manager *man,
864 uint64_t mask, uint32_t *res_mask)
866 uint64_t cur_flags = drm_bo_type_flags(mem_type);
869 if ((man->flags & _DRM_FLAG_MEMTYPE_FIXED) && disallow_fixed)
871 if (man->flags & _DRM_FLAG_MEMTYPE_CACHED)
872 cur_flags |= DRM_BO_FLAG_CACHED;
873 if (man->flags & _DRM_FLAG_MEMTYPE_MAPPABLE)
874 cur_flags |= DRM_BO_FLAG_MAPPABLE;
875 if (man->flags & _DRM_FLAG_MEMTYPE_CSELECT)
876 DRM_FLAG_MASKED(cur_flags, mask, DRM_BO_FLAG_CACHED);
878 if ((cur_flags & mask & DRM_BO_MASK_MEM) == 0)
881 if (mem_type == DRM_BO_MEM_LOCAL) {
882 *res_mask = cur_flags;
886 flag_diff = (mask ^ cur_flags);
887 if (flag_diff & DRM_BO_FLAG_CACHED_MAPPED)
888 cur_flags |= DRM_BO_FLAG_CACHED_MAPPED;
890 if ((flag_diff & DRM_BO_FLAG_CACHED) &&
891 (!(mask & DRM_BO_FLAG_CACHED) ||
892 (mask & DRM_BO_FLAG_FORCE_CACHING)))
895 if ((flag_diff & DRM_BO_FLAG_MAPPABLE) &&
896 ((mask & DRM_BO_FLAG_MAPPABLE) ||
897 (mask & DRM_BO_FLAG_FORCE_MAPPABLE)))
900 *res_mask = cur_flags;
905 * Creates space for memory region @mem according to its type.
907 * This function first searches for free space in compatible memory types in
908 * the priority order defined by the driver. If free space isn't found, then
909 * drm_bo_mem_force_space is attempted in priority order to evict and find
912 int drm_bo_mem_space(struct drm_buffer_object *bo,
913 struct drm_bo_mem_reg *mem, int no_wait)
915 struct drm_device *dev = bo->dev;
916 struct drm_buffer_manager *bm = &dev->bm;
917 struct drm_mem_type_manager *man;
919 uint32_t num_prios = dev->driver->bo_driver->num_mem_type_prio;
920 const uint32_t *prios = dev->driver->bo_driver->mem_type_prio;
922 uint32_t mem_type = DRM_BO_MEM_LOCAL;
927 struct drm_mm_node *node = NULL;
931 for (i = 0; i < num_prios; ++i) {
933 man = &bm->man[mem_type];
935 type_ok = drm_bo_mt_compatible(man,
936 bo->type == drm_bo_type_user,
937 mem_type, mem->proposed_flags,
943 if (mem_type == DRM_BO_MEM_LOCAL)
946 if ((mem_type == bo->pinned_mem_type) &&
947 (bo->pinned_node != NULL)) {
948 node = bo->pinned_node;
952 mutex_lock(&dev->struct_mutex);
953 if (man->has_type && man->use_type) {
955 node = drm_mm_search_free(&man->manager, mem->num_pages,
956 mem->page_alignment, 1);
958 node = drm_mm_get_block(node, mem->num_pages,
959 mem->page_alignment);
961 mutex_unlock(&dev->struct_mutex);
966 if ((type_ok && (mem_type == DRM_BO_MEM_LOCAL)) || node) {
968 mem->mem_type = mem_type;
969 mem->flags = cur_flags;
976 num_prios = dev->driver->bo_driver->num_mem_busy_prio;
977 prios = dev->driver->bo_driver->mem_busy_prio;
979 for (i = 0; i < num_prios; ++i) {
981 man = &bm->man[mem_type];
986 if (!drm_bo_mt_compatible(man,
987 bo->type == drm_bo_type_user,
993 ret = drm_bo_mem_force_space(dev, mem, mem_type, no_wait);
995 if (ret == 0 && mem->mm_node) {
996 mem->flags = cur_flags;
1004 ret = (has_eagain) ? -EAGAIN : -ENOMEM;
1007 EXPORT_SYMBOL(drm_bo_mem_space);
1010 * drm_bo_propose_flags:
1012 * @bo: the buffer object getting new flags
1014 * @new_flags: the new set of proposed flag bits
1016 * @new_mask: the mask of bits changed in new_flags
1018 * Modify the proposed_flag bits in @bo
1020 static int drm_bo_modify_proposed_flags (struct drm_buffer_object *bo,
1021 uint64_t new_flags, uint64_t new_mask)
1023 uint32_t new_access;
1025 /* Copy unchanging bits from existing proposed_flags */
1026 DRM_FLAG_MASKED(new_flags, bo->mem.proposed_flags, ~new_mask);
1028 if (bo->type == drm_bo_type_user &&
1029 ((new_flags & (DRM_BO_FLAG_CACHED | DRM_BO_FLAG_FORCE_CACHING)) !=
1030 (DRM_BO_FLAG_CACHED | DRM_BO_FLAG_FORCE_CACHING))) {
1031 DRM_ERROR("User buffers require cache-coherent memory.\n");
1035 if (bo->type != drm_bo_type_kernel && (new_mask & DRM_BO_FLAG_NO_EVICT) && !DRM_SUSER(DRM_CURPROC)) {
1036 DRM_ERROR("DRM_BO_FLAG_NO_EVICT is only available to priviliged processes.\n");
1040 if (likely(new_mask & DRM_BO_MASK_MEM) &&
1041 (bo->mem.flags & DRM_BO_FLAG_NO_EVICT) &&
1042 !DRM_SUSER(DRM_CURPROC)) {
1043 if (likely(bo->mem.flags & new_flags & new_mask &
1045 new_flags = (new_flags & ~DRM_BO_MASK_MEM) |
1046 (bo->mem.flags & DRM_BO_MASK_MEM);
1048 DRM_ERROR("Incompatible memory type specification "
1049 "for NO_EVICT buffer.\n");
1054 if ((new_flags & DRM_BO_FLAG_NO_MOVE)) {
1055 DRM_ERROR("DRM_BO_FLAG_NO_MOVE is not properly implemented yet.\n");
1059 new_access = new_flags & (DRM_BO_FLAG_EXE | DRM_BO_FLAG_WRITE |
1062 if (new_access == 0) {
1063 DRM_ERROR("Invalid buffer object rwx properties\n");
1067 bo->mem.proposed_flags = new_flags;
1072 * Call dev->struct_mutex locked.
1075 struct drm_buffer_object *drm_lookup_buffer_object(struct drm_file *file_priv,
1076 uint32_t handle, int check_owner)
1078 struct drm_user_object *uo;
1079 struct drm_buffer_object *bo;
1081 uo = drm_lookup_user_object(file_priv, handle);
1083 if (!uo || (uo->type != drm_buffer_type)) {
1084 DRM_ERROR("Could not find buffer object 0x%08x\n", handle);
1088 if (check_owner && file_priv != uo->owner) {
1089 if (!drm_lookup_ref_object(file_priv, uo, _DRM_REF_USE))
1093 bo = drm_user_object_entry(uo, struct drm_buffer_object, base);
1094 atomic_inc(&bo->usage);
1097 EXPORT_SYMBOL(drm_lookup_buffer_object);
1100 * Call bo->mutex locked.
1101 * Returns -EBUSY if the buffer is currently rendered to or from. 0 otherwise.
1102 * Doesn't do any fence flushing as opposed to the drm_bo_busy function.
1105 static int drm_bo_quick_busy(struct drm_buffer_object *bo, int check_unfenced)
1107 struct drm_fence_object *fence = bo->fence;
1109 if (check_unfenced && (bo->priv_flags & _DRM_BO_FLAG_UNFENCED))
1113 if (drm_fence_object_signaled(fence, bo->fence_type)) {
1114 drm_fence_usage_deref_unlocked(&bo->fence);
1122 int drm_bo_evict_cached(struct drm_buffer_object *bo)
1126 BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNFENCED);
1127 if (bo->mem.mm_node)
1128 ret = drm_bo_evict(bo, DRM_BO_MEM_TT, 1);
1132 EXPORT_SYMBOL(drm_bo_evict_cached);
1134 * Wait until a buffer is unmapped.
1137 static int drm_bo_wait_unmapped(struct drm_buffer_object *bo, int no_wait)
1141 if (likely(atomic_read(&bo->mapped)) == 0)
1144 if (unlikely(no_wait))
1148 mutex_unlock(&bo->mutex);
1149 ret = wait_event_interruptible(bo->event_queue,
1150 atomic_read(&bo->mapped) == 0);
1151 mutex_lock(&bo->mutex);
1152 bo->priv_flags |= _DRM_BO_FLAG_UNLOCKED;
1154 if (ret == -ERESTARTSYS)
1156 } while((ret == 0) && atomic_read(&bo->mapped) > 0);
1162 * Fill in the ioctl reply argument with buffer info.
1166 void drm_bo_fill_rep_arg(struct drm_buffer_object *bo,
1167 struct drm_bo_info_rep *rep)
1172 rep->handle = bo->base.hash.key;
1173 rep->flags = bo->mem.flags;
1174 rep->size = bo->num_pages * PAGE_SIZE;
1175 rep->offset = bo->offset;
1178 * drm_bo_type_device buffers have user-visible
1179 * handles which can be used to share across
1180 * processes. Hand that back to the application
1182 if (bo->type == drm_bo_type_device)
1183 rep->arg_handle = bo->map_list.user_token;
1185 rep->arg_handle = 0;
1187 rep->proposed_flags = bo->mem.proposed_flags;
1188 rep->buffer_start = bo->buffer_start;
1189 rep->fence_flags = bo->fence_type;
1191 rep->page_alignment = bo->mem.page_alignment;
1193 if ((bo->priv_flags & _DRM_BO_FLAG_UNFENCED) || drm_bo_quick_busy(bo, 1)) {
1194 DRM_FLAG_MASKED(rep->rep_flags, DRM_BO_REP_BUSY,
1198 EXPORT_SYMBOL(drm_bo_fill_rep_arg);
1201 * Wait for buffer idle and register that we've mapped the buffer.
1202 * Mapping is registered as a drm_ref_object with type _DRM_REF_TYPE1,
1203 * so that if the client dies, the mapping is automatically
1207 static int drm_buffer_object_map(struct drm_file *file_priv, uint32_t handle,
1208 uint32_t map_flags, unsigned hint,
1209 struct drm_bo_info_rep *rep)
1211 struct drm_buffer_object *bo;
1212 struct drm_device *dev = file_priv->minor->dev;
1214 int no_wait = hint & DRM_BO_HINT_DONT_BLOCK;
1216 mutex_lock(&dev->struct_mutex);
1217 bo = drm_lookup_buffer_object(file_priv, handle, 1);
1218 mutex_unlock(&dev->struct_mutex);
1223 mutex_lock(&bo->mutex);
1225 bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED;
1227 ret = drm_bo_wait(bo, 0, 1, no_wait, 1);
1231 if (bo->mem.flags & DRM_BO_FLAG_CACHED_MAPPED)
1232 drm_bo_evict_cached(bo);
1234 } while (unlikely(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED));
1236 atomic_inc(&bo->mapped);
1237 mutex_lock(&dev->struct_mutex);
1238 ret = drm_add_ref_object(file_priv, &bo->base, _DRM_REF_TYPE1);
1239 mutex_unlock(&dev->struct_mutex);
1241 if (atomic_dec_and_test(&bo->mapped))
1242 wake_up_all(&bo->event_queue);
1245 drm_bo_fill_rep_arg(bo, rep);
1248 mutex_unlock(&bo->mutex);
1249 drm_bo_usage_deref_unlocked(&bo);
1254 static int drm_buffer_object_unmap(struct drm_file *file_priv, uint32_t handle)
1256 struct drm_device *dev = file_priv->minor->dev;
1257 struct drm_buffer_object *bo;
1258 struct drm_ref_object *ro;
1261 mutex_lock(&dev->struct_mutex);
1263 bo = drm_lookup_buffer_object(file_priv, handle, 1);
1269 ro = drm_lookup_ref_object(file_priv, &bo->base, _DRM_REF_TYPE1);
1275 drm_remove_ref_object(file_priv, ro);
1276 drm_bo_usage_deref_locked(&bo);
1278 mutex_unlock(&dev->struct_mutex);
1283 * Call struct-sem locked.
1286 static void drm_buffer_user_object_unmap(struct drm_file *file_priv,
1287 struct drm_user_object *uo,
1288 enum drm_ref_type action)
1290 struct drm_buffer_object *bo =
1291 drm_user_object_entry(uo, struct drm_buffer_object, base);
1294 * We DON'T want to take the bo->lock here, because we want to
1295 * hold it when we wait for unmapped buffer.
1298 BUG_ON(action != _DRM_REF_TYPE1);
1300 if (atomic_dec_and_test(&bo->mapped))
1301 wake_up_all(&bo->event_queue);
1306 * Note that new_mem_flags are NOT transferred to the bo->mem.proposed_flags.
1309 int drm_bo_move_buffer(struct drm_buffer_object *bo, uint64_t new_mem_flags,
1310 int no_wait, int move_unfenced)
1312 struct drm_device *dev = bo->dev;
1313 struct drm_buffer_manager *bm = &dev->bm;
1315 struct drm_bo_mem_reg mem;
1317 BUG_ON(bo->fence != NULL);
1319 mem.num_pages = bo->num_pages;
1320 mem.size = mem.num_pages << PAGE_SHIFT;
1321 mem.proposed_flags = new_mem_flags;
1322 mem.page_alignment = bo->mem.page_alignment;
1324 mutex_lock(&bm->evict_mutex);
1325 mutex_lock(&dev->struct_mutex);
1326 list_del_init(&bo->lru);
1327 mutex_unlock(&dev->struct_mutex);
1330 * Determine where to move the buffer.
1332 ret = drm_bo_mem_space(bo, &mem, no_wait);
1336 ret = drm_bo_handle_move_mem(bo, &mem, 0, no_wait);
1339 mutex_lock(&dev->struct_mutex);
1340 if (ret || !move_unfenced) {
1342 if (mem.mm_node != bo->pinned_node)
1343 drm_mm_put_block(mem.mm_node);
1346 drm_bo_add_to_lru(bo);
1347 if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED) {
1348 wake_up_all(&bo->event_queue);
1349 DRM_FLAG_MASKED(bo->priv_flags, 0,
1350 _DRM_BO_FLAG_UNFENCED);
1353 list_add_tail(&bo->lru, &bm->unfenced);
1354 DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED,
1355 _DRM_BO_FLAG_UNFENCED);
1357 mutex_unlock(&dev->struct_mutex);
1358 mutex_unlock(&bm->evict_mutex);
1362 static int drm_bo_mem_compat(struct drm_bo_mem_reg *mem)
1364 uint32_t flag_diff = (mem->proposed_flags ^ mem->flags);
1366 if ((mem->proposed_flags & mem->flags & DRM_BO_MASK_MEM) == 0)
1368 if ((flag_diff & DRM_BO_FLAG_CACHED) &&
1369 (/* !(mem->proposed_flags & DRM_BO_FLAG_CACHED) ||*/
1370 (mem->proposed_flags & DRM_BO_FLAG_FORCE_CACHING)))
1373 if ((flag_diff & DRM_BO_FLAG_MAPPABLE) &&
1374 ((mem->proposed_flags & DRM_BO_FLAG_MAPPABLE) ||
1375 (mem->proposed_flags & DRM_BO_FLAG_FORCE_MAPPABLE)))
1381 * drm_buffer_object_validate:
1383 * @bo: the buffer object to modify
1385 * @fence_class: the new fence class covering this buffer
1387 * @move_unfenced: a boolean indicating whether switching the
1388 * memory space of this buffer should cause the buffer to
1389 * be placed on the unfenced list.
1391 * @no_wait: whether this function should return -EBUSY instead
1394 * Change buffer access parameters. This can involve moving
1395 * the buffer to the correct memory type, pinning the buffer
1396 * or changing the class/type of fence covering this buffer
1398 * Must be called with bo locked.
1401 static int drm_buffer_object_validate(struct drm_buffer_object *bo,
1402 uint32_t fence_class,
1403 int move_unfenced, int no_wait,
1406 struct drm_device *dev = bo->dev;
1407 struct drm_buffer_manager *bm = &dev->bm;
1411 ret = drm_bo_move_buffer(bo, bo->mem.proposed_flags, no_wait,
1415 DRM_ERROR("Failed moving buffer.\n");
1417 DRM_ERROR("Out of aperture space or "
1418 "DRM memory quota.\n");
1427 if (bo->mem.proposed_flags & (DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE)) {
1428 bo->pinned_mem_type = bo->mem.mem_type;
1429 mutex_lock(&dev->struct_mutex);
1430 list_del_init(&bo->pinned_lru);
1431 drm_bo_add_to_pinned_lru(bo);
1433 if (bo->pinned_node != bo->mem.mm_node) {
1434 if (bo->pinned_node != NULL)
1435 drm_mm_put_block(bo->pinned_node);
1436 bo->pinned_node = bo->mem.mm_node;
1439 mutex_unlock(&dev->struct_mutex);
1441 } else if (bo->pinned_node != NULL) {
1443 mutex_lock(&dev->struct_mutex);
1445 if (bo->pinned_node != bo->mem.mm_node)
1446 drm_mm_put_block(bo->pinned_node);
1448 list_del_init(&bo->pinned_lru);
1449 bo->pinned_node = NULL;
1450 mutex_unlock(&dev->struct_mutex);
1455 * We might need to add a TTM.
1458 if (bo->mem.mem_type == DRM_BO_MEM_LOCAL && bo->ttm == NULL) {
1459 ret = drm_bo_add_ttm(bo);
1464 * Validation has succeeded, move the access and other
1465 * non-mapping-related flag bits from the proposed flags to
1469 DRM_FLAG_MASKED(bo->mem.flags, bo->mem.proposed_flags, ~DRM_BO_MASK_MEMTYPE);
1472 * Finally, adjust lru to be sure.
1475 mutex_lock(&dev->struct_mutex);
1477 if (move_unfenced) {
1478 list_add_tail(&bo->lru, &bm->unfenced);
1479 DRM_FLAG_MASKED(bo->priv_flags, _DRM_BO_FLAG_UNFENCED,
1480 _DRM_BO_FLAG_UNFENCED);
1482 drm_bo_add_to_lru(bo);
1483 if (bo->priv_flags & _DRM_BO_FLAG_UNFENCED) {
1484 wake_up_all(&bo->event_queue);
1485 DRM_FLAG_MASKED(bo->priv_flags, 0,
1486 _DRM_BO_FLAG_UNFENCED);
1489 mutex_unlock(&dev->struct_mutex);
1495 * This function is called with bo->mutex locked, but may release it
1496 * temporarily to wait for events.
1499 static int drm_bo_prepare_for_validate(struct drm_buffer_object *bo,
1503 uint32_t fence_class,
1507 struct drm_device *dev = bo->dev;
1508 struct drm_bo_driver *driver = dev->driver->bo_driver;
1513 DRM_DEBUG("Proposed flags 0x%016llx, Old flags 0x%016llx\n",
1514 (unsigned long long) bo->mem.proposed_flags,
1515 (unsigned long long) bo->mem.flags);
1517 ret = drm_bo_modify_proposed_flags (bo, flags, mask);
1521 ret = drm_bo_wait_unmapped(bo, no_wait);
1525 ret = driver->fence_type(bo, &fence_class, &ftype);
1528 DRM_ERROR("Driver did not support given buffer permissions.\n");
1533 * We're switching command submission mechanism,
1534 * or cannot simply rely on the hardware serializing for us.
1535 * Insert a driver-dependant barrier or wait for buffer idle.
1538 if ((fence_class != bo->fence_class) ||
1539 ((ftype ^ bo->fence_type) & bo->fence_type)) {
1542 if (driver->command_stream_barrier) {
1543 ret = driver->command_stream_barrier(bo,
1548 if (ret && ret != -EAGAIN)
1549 ret = drm_bo_wait(bo, 0, 1, no_wait, 1);
1555 bo->new_fence_class = fence_class;
1556 bo->new_fence_type = ftype;
1559 * Check whether we need to move buffer.
1563 if (!drm_bo_mem_compat(&bo->mem)) {
1565 ret = drm_bo_wait(bo, 0, 1, no_wait, 1);
1572 * drm_bo_do_validate:
1574 * @bo: the buffer object
1576 * @flags: access rights, mapping parameters and cacheability. See
1577 * the DRM_BO_FLAG_* values in drm.h
1579 * @mask: Which flag values to change; this allows callers to modify
1580 * things without knowing the current state of other flags.
1582 * @hint: changes the proceedure for this operation, see the DRM_BO_HINT_*
1585 * @fence_class: a driver-specific way of doing fences. Presumably,
1586 * this would be used if the driver had more than one submission and
1587 * fencing mechanism. At this point, there isn't any use of this
1588 * from the user mode code.
1590 * @rep: To be stuffed with the reply from validation
1592 * 'validate' a buffer object. This changes where the buffer is
1593 * located, along with changing access modes.
1596 int drm_bo_do_validate(struct drm_buffer_object *bo,
1597 uint64_t flags, uint64_t mask, uint32_t hint,
1598 uint32_t fence_class,
1599 struct drm_bo_info_rep *rep)
1602 int no_wait = (hint & DRM_BO_HINT_DONT_BLOCK) != 0;
1605 mutex_lock(&bo->mutex);
1608 bo->priv_flags &= ~_DRM_BO_FLAG_UNLOCKED;
1610 ret = drm_bo_prepare_for_validate(bo, flags, mask, hint,
1611 fence_class, no_wait,
1616 } while(unlikely(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED));
1618 ret = drm_buffer_object_validate(bo,
1620 !(hint & DRM_BO_HINT_DONT_FENCE),
1624 BUG_ON(bo->priv_flags & _DRM_BO_FLAG_UNLOCKED);
1627 drm_bo_fill_rep_arg(bo, rep);
1629 mutex_unlock(&bo->mutex);
1633 EXPORT_SYMBOL(drm_bo_do_validate);
1636 * drm_bo_handle_validate
1638 * @file_priv: the drm file private, used to get a handle to the user context
1640 * @handle: the buffer object handle
1642 * @flags: access rights, mapping parameters and cacheability. See
1643 * the DRM_BO_FLAG_* values in drm.h
1645 * @mask: Which flag values to change; this allows callers to modify
1646 * things without knowing the current state of other flags.
1648 * @hint: changes the proceedure for this operation, see the DRM_BO_HINT_*
1651 * @fence_class: a driver-specific way of doing fences. Presumably,
1652 * this would be used if the driver had more than one submission and
1653 * fencing mechanism. At this point, there isn't any use of this
1654 * from the user mode code.
1656 * @rep: To be stuffed with the reply from validation
1658 * @bp_rep: To be stuffed with the buffer object pointer
1660 * Perform drm_bo_do_validate on a buffer referenced by a user-space handle instead
1661 * of a pointer to a buffer object. Optionally return a pointer to the buffer object.
1662 * This is a convenience wrapper only.
1665 int drm_bo_handle_validate(struct drm_file *file_priv, uint32_t handle,
1666 uint64_t flags, uint64_t mask,
1668 uint32_t fence_class,
1669 struct drm_bo_info_rep *rep,
1670 struct drm_buffer_object **bo_rep)
1672 struct drm_device *dev = file_priv->minor->dev;
1673 struct drm_buffer_object *bo;
1676 mutex_lock(&dev->struct_mutex);
1677 bo = drm_lookup_buffer_object(file_priv, handle, 1);
1678 mutex_unlock(&dev->struct_mutex);
1683 if (bo->base.owner != file_priv)
1684 mask &= ~(DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE);
1686 ret = drm_bo_do_validate(bo, flags, mask, hint, fence_class, rep);
1691 drm_bo_usage_deref_unlocked(&bo);
1695 EXPORT_SYMBOL(drm_bo_handle_validate);
1698 static int drm_bo_handle_info(struct drm_file *file_priv, uint32_t handle,
1699 struct drm_bo_info_rep *rep)
1701 struct drm_device *dev = file_priv->minor->dev;
1702 struct drm_buffer_object *bo;
1704 mutex_lock(&dev->struct_mutex);
1705 bo = drm_lookup_buffer_object(file_priv, handle, 1);
1706 mutex_unlock(&dev->struct_mutex);
1711 mutex_lock(&bo->mutex);
1714 * FIXME: Quick busy here?
1718 drm_bo_fill_rep_arg(bo, rep);
1719 mutex_unlock(&bo->mutex);
1720 drm_bo_usage_deref_unlocked(&bo);
1724 static int drm_bo_handle_wait(struct drm_file *file_priv, uint32_t handle,
1726 struct drm_bo_info_rep *rep)
1728 struct drm_device *dev = file_priv->minor->dev;
1729 struct drm_buffer_object *bo;
1730 int no_wait = hint & DRM_BO_HINT_DONT_BLOCK;
1733 mutex_lock(&dev->struct_mutex);
1734 bo = drm_lookup_buffer_object(file_priv, handle, 1);
1735 mutex_unlock(&dev->struct_mutex);
1740 mutex_lock(&bo->mutex);
1741 ret = drm_bo_wait(bo, hint & DRM_BO_HINT_WAIT_LAZY, 1, no_wait, 1);
1745 drm_bo_fill_rep_arg(bo, rep);
1747 mutex_unlock(&bo->mutex);
1748 drm_bo_usage_deref_unlocked(&bo);
1752 int drm_buffer_object_create(struct drm_device *dev,
1754 enum drm_bo_type type,
1757 uint32_t page_alignment,
1758 unsigned long buffer_start,
1759 struct drm_buffer_object **buf_obj)
1761 struct drm_buffer_manager *bm = &dev->bm;
1762 struct drm_buffer_object *bo;
1764 unsigned long num_pages;
1766 size += buffer_start & ~PAGE_MASK;
1767 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1768 if (num_pages == 0) {
1769 DRM_ERROR("Illegal buffer object size.\n");
1773 bo = drm_ctl_calloc(1, sizeof(*bo), DRM_MEM_BUFOBJ);
1778 mutex_init(&bo->mutex);
1779 mutex_lock(&bo->mutex);
1781 atomic_set(&bo->usage, 1);
1782 atomic_set(&bo->mapped, 0);
1783 DRM_INIT_WAITQUEUE(&bo->event_queue);
1784 INIT_LIST_HEAD(&bo->lru);
1785 INIT_LIST_HEAD(&bo->pinned_lru);
1786 INIT_LIST_HEAD(&bo->ddestroy);
1787 #ifdef DRM_ODD_MM_COMPAT
1788 INIT_LIST_HEAD(&bo->p_mm_list);
1789 INIT_LIST_HEAD(&bo->vma_list);
1793 bo->num_pages = num_pages;
1794 bo->mem.mem_type = DRM_BO_MEM_LOCAL;
1795 bo->mem.num_pages = bo->num_pages;
1796 bo->mem.mm_node = NULL;
1797 bo->mem.page_alignment = page_alignment;
1798 bo->buffer_start = buffer_start & PAGE_MASK;
1800 bo->mem.flags = (DRM_BO_FLAG_MEM_LOCAL | DRM_BO_FLAG_CACHED |
1801 DRM_BO_FLAG_MAPPABLE);
1802 bo->mem.proposed_flags = 0;
1803 atomic_inc(&bm->count);
1805 * Use drm_bo_modify_proposed_flags to error-check the proposed flags
1807 ret = drm_bo_modify_proposed_flags (bo, flags, flags);
1812 * For drm_bo_type_device buffers, allocate
1813 * address space from the device so that applications
1814 * can mmap the buffer from there
1816 if (bo->type == drm_bo_type_device) {
1817 mutex_lock(&dev->struct_mutex);
1818 ret = drm_bo_setup_vm_locked(bo);
1819 mutex_unlock(&dev->struct_mutex);
1824 mutex_unlock(&bo->mutex);
1825 ret = drm_bo_do_validate(bo, 0, 0, hint | DRM_BO_HINT_DONT_FENCE,
1828 goto out_err_unlocked;
1834 mutex_unlock(&bo->mutex);
1836 drm_bo_usage_deref_unlocked(&bo);
1839 EXPORT_SYMBOL(drm_buffer_object_create);
1842 static int drm_bo_add_user_object(struct drm_file *file_priv,
1843 struct drm_buffer_object *bo, int shareable)
1845 struct drm_device *dev = file_priv->minor->dev;
1848 mutex_lock(&dev->struct_mutex);
1849 ret = drm_add_user_object(file_priv, &bo->base, shareable);
1853 bo->base.remove = drm_bo_base_deref_locked;
1854 bo->base.type = drm_buffer_type;
1855 bo->base.ref_struct_locked = NULL;
1856 bo->base.unref = drm_buffer_user_object_unmap;
1859 mutex_unlock(&dev->struct_mutex);
1863 int drm_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1865 struct drm_bo_create_arg *arg = data;
1866 struct drm_bo_create_req *req = &arg->d.req;
1867 struct drm_bo_info_rep *rep = &arg->d.rep;
1868 struct drm_buffer_object *entry;
1869 enum drm_bo_type bo_type;
1872 DRM_DEBUG("drm_bo_create_ioctl: %dkb, %dkb align\n",
1873 (int)(req->size / 1024), req->page_alignment * 4);
1875 if (!dev->bm.initialized) {
1876 DRM_ERROR("Buffer object manager is not initialized.\n");
1881 * If the buffer creation request comes in with a starting address,
1882 * that points at the desired user pages to map. Otherwise, create
1883 * a drm_bo_type_device buffer, which uses pages allocated from the kernel
1885 bo_type = (req->buffer_start) ? drm_bo_type_user : drm_bo_type_device;
1888 * User buffers cannot be shared
1890 if (bo_type == drm_bo_type_user)
1891 req->flags &= ~DRM_BO_FLAG_SHAREABLE;
1893 ret = drm_buffer_object_create(file_priv->minor->dev,
1894 req->size, bo_type, req->flags,
1895 req->hint, req->page_alignment,
1896 req->buffer_start, &entry);
1900 ret = drm_bo_add_user_object(file_priv, entry,
1901 req->flags & DRM_BO_FLAG_SHAREABLE);
1903 drm_bo_usage_deref_unlocked(&entry);
1907 mutex_lock(&entry->mutex);
1908 drm_bo_fill_rep_arg(entry, rep);
1909 mutex_unlock(&entry->mutex);
1915 int drm_bo_setstatus_ioctl(struct drm_device *dev,
1916 void *data, struct drm_file *file_priv)
1918 struct drm_bo_map_wait_idle_arg *arg = data;
1919 struct drm_bo_info_req *req = &arg->d.req;
1920 struct drm_bo_info_rep *rep = &arg->d.rep;
1921 struct drm_buffer_object *bo;
1924 if (!dev->bm.initialized) {
1925 DRM_ERROR("Buffer object manager is not initialized.\n");
1929 ret = drm_bo_read_lock(&dev->bm.bm_lock, 1);
1933 mutex_lock(&dev->struct_mutex);
1934 bo = drm_lookup_buffer_object(file_priv, req->handle, 1);
1935 mutex_unlock(&dev->struct_mutex);
1940 if (bo->base.owner != file_priv)
1941 req->mask &= ~(DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE);
1943 ret = drm_bo_do_validate(bo, req->flags, req->mask,
1944 req->hint | DRM_BO_HINT_DONT_FENCE,
1945 bo->fence_class, rep);
1947 drm_bo_usage_deref_unlocked(&bo);
1949 (void) drm_bo_read_unlock(&dev->bm.bm_lock);
1954 int drm_bo_map_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1956 struct drm_bo_map_wait_idle_arg *arg = data;
1957 struct drm_bo_info_req *req = &arg->d.req;
1958 struct drm_bo_info_rep *rep = &arg->d.rep;
1960 if (!dev->bm.initialized) {
1961 DRM_ERROR("Buffer object manager is not initialized.\n");
1965 ret = drm_buffer_object_map(file_priv, req->handle, req->mask,
1973 int drm_bo_unmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1975 struct drm_bo_handle_arg *arg = data;
1977 if (!dev->bm.initialized) {
1978 DRM_ERROR("Buffer object manager is not initialized.\n");
1982 ret = drm_buffer_object_unmap(file_priv, arg->handle);
1987 int drm_bo_reference_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1989 struct drm_bo_reference_info_arg *arg = data;
1990 struct drm_bo_handle_arg *req = &arg->d.req;
1991 struct drm_bo_info_rep *rep = &arg->d.rep;
1992 struct drm_user_object *uo;
1995 if (!dev->bm.initialized) {
1996 DRM_ERROR("Buffer object manager is not initialized.\n");
2000 ret = drm_user_object_ref(file_priv, req->handle,
2001 drm_buffer_type, &uo);
2005 ret = drm_bo_handle_info(file_priv, req->handle, rep);
2012 int drm_bo_unreference_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
2014 struct drm_bo_handle_arg *arg = data;
2017 if (!dev->bm.initialized) {
2018 DRM_ERROR("Buffer object manager is not initialized.\n");
2022 ret = drm_user_object_unref(file_priv, arg->handle, drm_buffer_type);
2026 int drm_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
2028 struct drm_bo_reference_info_arg *arg = data;
2029 struct drm_bo_handle_arg *req = &arg->d.req;
2030 struct drm_bo_info_rep *rep = &arg->d.rep;
2033 if (!dev->bm.initialized) {
2034 DRM_ERROR("Buffer object manager is not initialized.\n");
2038 ret = drm_bo_handle_info(file_priv, req->handle, rep);
2045 int drm_bo_wait_idle_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
2047 struct drm_bo_map_wait_idle_arg *arg = data;
2048 struct drm_bo_info_req *req = &arg->d.req;
2049 struct drm_bo_info_rep *rep = &arg->d.rep;
2051 if (!dev->bm.initialized) {
2052 DRM_ERROR("Buffer object manager is not initialized.\n");
2056 ret = drm_bo_handle_wait(file_priv, req->handle,
2064 static int drm_bo_leave_list(struct drm_buffer_object *bo,
2069 struct drm_device *dev = bo->dev;
2072 mutex_lock(&bo->mutex);
2074 ret = drm_bo_expire_fence(bo, allow_errors);
2079 DRM_FLAG_MASKED(bo->mem.flags, 0, DRM_BO_FLAG_NO_MOVE);
2080 mutex_lock(&dev->struct_mutex);
2081 list_del_init(&bo->pinned_lru);
2082 if (bo->pinned_node == bo->mem.mm_node)
2083 bo->pinned_node = NULL;
2084 if (bo->pinned_node != NULL) {
2085 drm_mm_put_block(bo->pinned_node);
2086 bo->pinned_node = NULL;
2088 mutex_unlock(&dev->struct_mutex);
2091 if (bo->mem.flags & DRM_BO_FLAG_NO_EVICT) {
2092 DRM_ERROR("A DRM_BO_NO_EVICT buffer present at "
2093 "cleanup. Removing flag and evicting.\n");
2094 bo->mem.flags &= ~DRM_BO_FLAG_NO_EVICT;
2095 bo->mem.proposed_flags &= ~DRM_BO_FLAG_NO_EVICT;
2098 if (bo->mem.mem_type == mem_type)
2099 ret = drm_bo_evict(bo, mem_type, 0);
2106 DRM_ERROR("Cleanup eviction failed\n");
2111 mutex_unlock(&bo->mutex);
2116 static struct drm_buffer_object *drm_bo_entry(struct list_head *list,
2120 return list_entry(list, struct drm_buffer_object, pinned_lru);
2122 return list_entry(list, struct drm_buffer_object, lru);
2126 * dev->struct_mutex locked.
2129 static int drm_bo_force_list_clean(struct drm_device *dev,
2130 struct list_head *head,
2136 struct list_head *list, *next, *prev;
2137 struct drm_buffer_object *entry, *nentry;
2142 * The list traversal is a bit odd here, because an item may
2143 * disappear from the list when we release the struct_mutex or
2144 * when we decrease the usage count. Also we're not guaranteed
2145 * to drain pinned lists, so we can't always restart.
2150 list_for_each_safe(list, next, head) {
2153 entry = (nentry != NULL) ? nentry: drm_bo_entry(list, pinned_list);
2154 atomic_inc(&entry->usage);
2156 atomic_dec(&nentry->usage);
2161 * Protect the next item from destruction, so we can check
2162 * its list pointers later on.
2166 nentry = drm_bo_entry(next, pinned_list);
2167 atomic_inc(&nentry->usage);
2169 mutex_unlock(&dev->struct_mutex);
2171 ret = drm_bo_leave_list(entry, mem_type, free_pinned,
2173 mutex_lock(&dev->struct_mutex);
2175 drm_bo_usage_deref_locked(&entry);
2180 * Has the next item disappeared from the list?
2183 do_restart = ((next->prev != list) && (next->prev != prev));
2185 if (nentry != NULL && do_restart)
2186 drm_bo_usage_deref_locked(&nentry);
2194 int drm_bo_clean_mm(struct drm_device *dev, unsigned mem_type, int kern_clean)
2196 struct drm_buffer_manager *bm = &dev->bm;
2197 struct drm_mem_type_manager *man = &bm->man[mem_type];
2200 if (mem_type >= DRM_BO_MEM_TYPES) {
2201 DRM_ERROR("Illegal memory type %d\n", mem_type);
2205 if (!man->has_type) {
2206 DRM_ERROR("Trying to take down uninitialized "
2207 "memory manager type %u\n", mem_type);
2211 if ((man->kern_init_type) && (kern_clean == 0)) {
2212 DRM_ERROR("Trying to take down kernel initialized "
2213 "memory manager type %u\n", mem_type);
2222 BUG_ON(!list_empty(&bm->unfenced));
2223 drm_bo_force_list_clean(dev, &man->lru, mem_type, 1, 0, 0);
2224 drm_bo_force_list_clean(dev, &man->pinned, mem_type, 1, 0, 1);
2226 if (drm_mm_clean(&man->manager)) {
2227 drm_mm_takedown(&man->manager);
2235 EXPORT_SYMBOL(drm_bo_clean_mm);
2238 *Evict all buffers of a particular mem_type, but leave memory manager
2239 *regions for NO_MOVE buffers intact. New buffers cannot be added at this
2240 *point since we have the hardware lock.
2243 static int drm_bo_lock_mm(struct drm_device *dev, unsigned mem_type)
2246 struct drm_buffer_manager *bm = &dev->bm;
2247 struct drm_mem_type_manager *man = &bm->man[mem_type];
2249 if (mem_type == 0 || mem_type >= DRM_BO_MEM_TYPES) {
2250 DRM_ERROR("Illegal memory manager memory type %u.\n", mem_type);
2254 if (!man->has_type) {
2255 DRM_ERROR("Memory type %u has not been initialized.\n",
2260 ret = drm_bo_force_list_clean(dev, &man->lru, mem_type, 0, 1, 0);
2263 ret = drm_bo_force_list_clean(dev, &man->pinned, mem_type, 0, 1, 1);
2268 int drm_bo_init_mm(struct drm_device *dev, unsigned type,
2269 unsigned long p_offset, unsigned long p_size,
2272 struct drm_buffer_manager *bm = &dev->bm;
2274 struct drm_mem_type_manager *man;
2276 if (type >= DRM_BO_MEM_TYPES) {
2277 DRM_ERROR("Illegal memory type %d\n", type);
2281 man = &bm->man[type];
2282 if (man->has_type) {
2283 DRM_ERROR("Memory manager already initialized for type %d\n",
2288 ret = dev->driver->bo_driver->init_mem_type(dev, type, man);
2293 if (type != DRM_BO_MEM_LOCAL) {
2295 DRM_ERROR("Zero size memory manager type %d\n", type);
2298 ret = drm_mm_init(&man->manager, p_offset, p_size);
2304 man->kern_init_type = kern_init;
2307 INIT_LIST_HEAD(&man->lru);
2308 INIT_LIST_HEAD(&man->pinned);
2312 EXPORT_SYMBOL(drm_bo_init_mm);
2315 * This function is intended to be called on drm driver unload.
2316 * If you decide to call it from lastclose, you must protect the call
2317 * from a potentially racing drm_bo_driver_init in firstopen.
2318 * (This may happen on X server restart).
2321 int drm_bo_driver_finish(struct drm_device *dev)
2323 struct drm_buffer_manager *bm = &dev->bm;
2325 unsigned i = DRM_BO_MEM_TYPES;
2326 struct drm_mem_type_manager *man;
2328 mutex_lock(&dev->struct_mutex);
2330 if (!bm->initialized)
2332 bm->initialized = 0;
2336 if (man->has_type) {
2338 if ((i != DRM_BO_MEM_LOCAL) && drm_bo_clean_mm(dev, i, 1)) {
2340 DRM_ERROR("DRM memory manager type %d "
2341 "is not clean.\n", i);
2346 mutex_unlock(&dev->struct_mutex);
2348 if (!cancel_delayed_work(&bm->wq))
2349 flush_scheduled_work();
2351 mutex_lock(&dev->struct_mutex);
2352 drm_bo_delayed_delete(dev, 1);
2353 if (list_empty(&bm->ddestroy))
2354 DRM_DEBUG("Delayed destroy list was clean\n");
2356 if (list_empty(&bm->man[0].lru))
2357 DRM_DEBUG("Swap list was clean\n");
2359 if (list_empty(&bm->man[0].pinned))
2360 DRM_DEBUG("NO_MOVE list was clean\n");
2362 if (list_empty(&bm->unfenced))
2363 DRM_DEBUG("Unfenced list was clean\n");
2365 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
2366 ClearPageReserved(bm->dummy_read_page);
2368 __free_page(bm->dummy_read_page);
2371 mutex_unlock(&dev->struct_mutex);
2376 * This function is intended to be called on drm driver load.
2377 * If you decide to call it from firstopen, you must protect the call
2378 * from a potentially racing drm_bo_driver_finish in lastclose.
2379 * (This may happen on X server restart).
2382 int drm_bo_driver_init(struct drm_device *dev)
2384 struct drm_bo_driver *driver = dev->driver->bo_driver;
2385 struct drm_buffer_manager *bm = &dev->bm;
2388 bm->dummy_read_page = NULL;
2389 drm_bo_init_lock(&bm->bm_lock);
2390 mutex_lock(&dev->struct_mutex);
2394 bm->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
2395 if (!bm->dummy_read_page) {
2400 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
2401 SetPageReserved(bm->dummy_read_page);
2405 * Initialize the system memory buffer type.
2406 * Other types need to be driver / IOCTL initialized.
2408 ret = drm_bo_init_mm(dev, DRM_BO_MEM_LOCAL, 0, 0, 1);
2412 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
2413 INIT_WORK(&bm->wq, &drm_bo_delayed_workqueue, dev);
2415 INIT_DELAYED_WORK(&bm->wq, drm_bo_delayed_workqueue);
2417 bm->initialized = 1;
2419 atomic_set(&bm->count, 0);
2421 INIT_LIST_HEAD(&bm->unfenced);
2422 INIT_LIST_HEAD(&bm->ddestroy);
2424 mutex_unlock(&dev->struct_mutex);
2427 EXPORT_SYMBOL(drm_bo_driver_init);
2429 int drm_mm_init_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
2431 struct drm_mm_init_arg *arg = data;
2432 struct drm_buffer_manager *bm = &dev->bm;
2433 struct drm_bo_driver *driver = dev->driver->bo_driver;
2437 DRM_ERROR("Buffer objects are not supported by this driver\n");
2441 ret = drm_bo_write_lock(&bm->bm_lock, 1, file_priv);
2446 if (arg->magic != DRM_BO_INIT_MAGIC) {
2447 DRM_ERROR("You are using an old libdrm that is not compatible with\n"
2448 "\tthe kernel DRM module. Please upgrade your libdrm.\n");
2451 if (arg->major != DRM_BO_INIT_MAJOR) {
2452 DRM_ERROR("libdrm and kernel DRM buffer object interface major\n"
2453 "\tversion don't match. Got %d, expected %d.\n",
2454 arg->major, DRM_BO_INIT_MAJOR);
2458 mutex_lock(&dev->struct_mutex);
2459 if (!bm->initialized) {
2460 DRM_ERROR("DRM memory manager was not initialized.\n");
2463 if (arg->mem_type == 0) {
2464 DRM_ERROR("System memory buffers already initialized.\n");
2467 ret = drm_bo_init_mm(dev, arg->mem_type,
2468 arg->p_offset, arg->p_size, 0);
2471 mutex_unlock(&dev->struct_mutex);
2472 (void) drm_bo_write_unlock(&bm->bm_lock, file_priv);
2480 int drm_mm_takedown_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
2482 struct drm_mm_type_arg *arg = data;
2483 struct drm_buffer_manager *bm = &dev->bm;
2484 struct drm_bo_driver *driver = dev->driver->bo_driver;
2488 DRM_ERROR("Buffer objects are not supported by this driver\n");
2492 ret = drm_bo_write_lock(&bm->bm_lock, 0, file_priv);
2496 mutex_lock(&dev->struct_mutex);
2498 if (!bm->initialized) {
2499 DRM_ERROR("DRM memory manager was not initialized\n");
2502 if (arg->mem_type == 0) {
2503 DRM_ERROR("No takedown for System memory buffers.\n");
2507 if ((ret = drm_bo_clean_mm(dev, arg->mem_type, 0))) {
2509 DRM_ERROR("Memory manager type %d not clean. "
2510 "Delaying takedown\n", arg->mem_type);
2514 mutex_unlock(&dev->struct_mutex);
2515 (void) drm_bo_write_unlock(&bm->bm_lock, file_priv);
2523 int drm_mm_lock_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
2525 struct drm_mm_type_arg *arg = data;
2526 struct drm_bo_driver *driver = dev->driver->bo_driver;
2530 DRM_ERROR("Buffer objects are not supported by this driver\n");
2534 if (arg->lock_flags & DRM_BO_LOCK_IGNORE_NO_EVICT) {
2535 DRM_ERROR("Lock flag DRM_BO_LOCK_IGNORE_NO_EVICT not supported yet.\n");
2539 if (arg->lock_flags & DRM_BO_LOCK_UNLOCK_BM) {
2540 ret = drm_bo_write_lock(&dev->bm.bm_lock, 1, file_priv);
2545 mutex_lock(&dev->struct_mutex);
2546 ret = drm_bo_lock_mm(dev, arg->mem_type);
2547 mutex_unlock(&dev->struct_mutex);
2549 (void) drm_bo_write_unlock(&dev->bm.bm_lock, file_priv);
2556 int drm_mm_unlock_ioctl(struct drm_device *dev,
2558 struct drm_file *file_priv)
2560 struct drm_mm_type_arg *arg = data;
2561 struct drm_bo_driver *driver = dev->driver->bo_driver;
2565 DRM_ERROR("Buffer objects are not supported by this driver\n");
2569 if (arg->lock_flags & DRM_BO_LOCK_UNLOCK_BM) {
2570 ret = drm_bo_write_unlock(&dev->bm.bm_lock, file_priv);
2578 int drm_mm_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
2580 struct drm_mm_info_arg *arg = data;
2581 struct drm_buffer_manager *bm = &dev->bm;
2582 struct drm_bo_driver *driver = dev->driver->bo_driver;
2583 struct drm_mem_type_manager *man;
2585 int mem_type = arg->mem_type;
2588 DRM_ERROR("Buffer objects are not supported by this driver\n");
2592 if (mem_type >= DRM_BO_MEM_TYPES) {
2593 DRM_ERROR("Illegal memory type %d\n", arg->mem_type);
2597 mutex_lock(&dev->struct_mutex);
2598 if (!bm->initialized) {
2599 DRM_ERROR("DRM memory manager was not initialized\n");
2605 man = &bm->man[arg->mem_type];
2607 arg->p_size = man->size;
2610 mutex_unlock(&dev->struct_mutex);
2615 * buffer object vm functions.
2618 int drm_mem_reg_is_pci(struct drm_device *dev, struct drm_bo_mem_reg *mem)
2620 struct drm_buffer_manager *bm = &dev->bm;
2621 struct drm_mem_type_manager *man = &bm->man[mem->mem_type];
2623 if (!(man->flags & _DRM_FLAG_MEMTYPE_FIXED)) {
2624 if (mem->mem_type == DRM_BO_MEM_LOCAL)
2627 if (man->flags & _DRM_FLAG_MEMTYPE_CMA)
2630 if (mem->flags & DRM_BO_FLAG_CACHED)
2635 EXPORT_SYMBOL(drm_mem_reg_is_pci);
2638 * \c Get the PCI offset for the buffer object memory.
2640 * \param bo The buffer object.
2641 * \param bus_base On return the base of the PCI region
2642 * \param bus_offset On return the byte offset into the PCI region
2643 * \param bus_size On return the byte size of the buffer object or zero if
2644 * the buffer object memory is not accessible through a PCI region.
2645 * \return Failure indication.
2647 * Returns -EINVAL if the buffer object is currently not mappable.
2648 * Otherwise returns zero.
2651 int drm_bo_pci_offset(struct drm_device *dev,
2652 struct drm_bo_mem_reg *mem,
2653 unsigned long *bus_base,
2654 unsigned long *bus_offset, unsigned long *bus_size)
2656 struct drm_buffer_manager *bm = &dev->bm;
2657 struct drm_mem_type_manager *man = &bm->man[mem->mem_type];
2660 if (!(man->flags & _DRM_FLAG_MEMTYPE_MAPPABLE))
2663 if (drm_mem_reg_is_pci(dev, mem)) {
2664 *bus_offset = mem->mm_node->start << PAGE_SHIFT;
2665 *bus_size = mem->num_pages << PAGE_SHIFT;
2666 *bus_base = man->io_offset;
2673 * \c Kill all user-space virtual mappings of this buffer object.
2675 * \param bo The buffer object.
2677 * Call bo->mutex locked.
2680 void drm_bo_unmap_virtual(struct drm_buffer_object *bo)
2682 struct drm_device *dev = bo->dev;
2683 loff_t offset = ((loff_t) bo->map_list.hash.key) << PAGE_SHIFT;
2684 loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT;
2686 if (!dev->dev_mapping)
2689 unmap_mapping_range(dev->dev_mapping, offset, holelen, 1);
2693 * drm_bo_takedown_vm_locked:
2695 * @bo: the buffer object to remove any drm device mapping
2697 * Remove any associated vm mapping on the drm device node that
2698 * would have been created for a drm_bo_type_device buffer
2700 static void drm_bo_takedown_vm_locked(struct drm_buffer_object *bo)
2702 struct drm_map_list *list;
2703 drm_local_map_t *map;
2704 struct drm_device *dev = bo->dev;
2706 DRM_ASSERT_LOCKED(&dev->struct_mutex);
2707 if (bo->type != drm_bo_type_device)
2710 list = &bo->map_list;
2711 if (list->user_token) {
2712 drm_ht_remove_item(&dev->map_hash, &list->hash);
2713 list->user_token = 0;
2715 if (list->file_offset_node) {
2716 drm_mm_put_block(list->file_offset_node);
2717 list->file_offset_node = NULL;
2724 drm_ctl_free(map, sizeof(*map), DRM_MEM_BUFOBJ);
2726 list->user_token = 0ULL;
2727 drm_bo_usage_deref_locked(&bo);
2731 * drm_bo_setup_vm_locked:
2733 * @bo: the buffer to allocate address space for
2735 * Allocate address space in the drm device so that applications
2736 * can mmap the buffer and access the contents. This only
2737 * applies to drm_bo_type_device objects as others are not
2738 * placed in the drm device address space.
2740 static int drm_bo_setup_vm_locked(struct drm_buffer_object *bo)
2742 struct drm_map_list *list = &bo->map_list;
2743 drm_local_map_t *map;
2744 struct drm_device *dev = bo->dev;
2746 DRM_ASSERT_LOCKED(&dev->struct_mutex);
2747 list->map = drm_ctl_calloc(1, sizeof(*map), DRM_MEM_BUFOBJ);
2753 map->type = _DRM_TTM;
2754 map->flags = _DRM_REMOVABLE;
2755 map->size = bo->mem.num_pages * PAGE_SIZE;
2756 atomic_inc(&bo->usage);
2757 map->handle = (void *)bo;
2759 list->file_offset_node = drm_mm_search_free(&dev->offset_manager,
2760 bo->mem.num_pages, 0, 0);
2762 if (unlikely(!list->file_offset_node)) {
2763 drm_bo_takedown_vm_locked(bo);
2767 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
2768 bo->mem.num_pages, 0);
2770 if (unlikely(!list->file_offset_node)) {
2771 drm_bo_takedown_vm_locked(bo);
2775 list->hash.key = list->file_offset_node->start;
2776 if (drm_ht_insert_item(&dev->map_hash, &list->hash)) {
2777 drm_bo_takedown_vm_locked(bo);
2781 list->user_token = ((uint64_t) list->hash.key) << PAGE_SHIFT;
2786 int drm_bo_version_ioctl(struct drm_device *dev, void *data,
2787 struct drm_file *file_priv)
2789 struct drm_bo_version_arg *arg = (struct drm_bo_version_arg *)data;
2791 arg->major = DRM_BO_INIT_MAJOR;
2792 arg->minor = DRM_BO_INIT_MINOR;
2793 arg->patchlevel = DRM_BO_INIT_PATCH;