2 * Copyright (C) 2013 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/spinlock.h>
19 #include <linux/shmem_fs.h>
20 #include <linux/dma-buf.h>
21 #include <linux/pfn_t.h>
24 #include "msm_fence.h"
29 static void msm_gem_vunmap_locked(struct drm_gem_object *obj);
32 static dma_addr_t physaddr(struct drm_gem_object *obj)
34 struct msm_gem_object *msm_obj = to_msm_bo(obj);
35 struct msm_drm_private *priv = obj->dev->dev_private;
36 return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +
40 static bool use_pages(struct drm_gem_object *obj)
42 struct msm_gem_object *msm_obj = to_msm_bo(obj);
43 return !msm_obj->vram_node;
46 /* allocate pages from VRAM carveout, used when no IOMMU: */
47 static struct page **get_pages_vram(struct drm_gem_object *obj, int npages)
49 struct msm_gem_object *msm_obj = to_msm_bo(obj);
50 struct msm_drm_private *priv = obj->dev->dev_private;
55 p = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
57 return ERR_PTR(-ENOMEM);
59 spin_lock(&priv->vram.lock);
60 ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node, npages);
61 spin_unlock(&priv->vram.lock);
67 paddr = physaddr(obj);
68 for (i = 0; i < npages; i++) {
69 p[i] = phys_to_page(paddr);
76 static struct page **get_pages(struct drm_gem_object *obj)
78 struct msm_gem_object *msm_obj = to_msm_bo(obj);
80 if (!msm_obj->pages) {
81 struct drm_device *dev = obj->dev;
83 int npages = obj->size >> PAGE_SHIFT;
86 p = drm_gem_get_pages(obj);
88 p = get_pages_vram(obj, npages);
91 dev_err(dev->dev, "could not get pages: %ld\n",
98 msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
99 if (IS_ERR(msm_obj->sgt)) {
100 void *ptr = ERR_CAST(msm_obj->sgt);
102 dev_err(dev->dev, "failed to allocate sgt\n");
107 /* For non-cached buffers, ensure the new pages are clean
108 * because display controller, GPU, etc. are not coherent:
110 if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
111 dma_map_sg(dev->dev, msm_obj->sgt->sgl,
112 msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
115 return msm_obj->pages;
118 static void put_pages_vram(struct drm_gem_object *obj)
120 struct msm_gem_object *msm_obj = to_msm_bo(obj);
121 struct msm_drm_private *priv = obj->dev->dev_private;
123 spin_lock(&priv->vram.lock);
124 drm_mm_remove_node(msm_obj->vram_node);
125 spin_unlock(&priv->vram.lock);
127 kvfree(msm_obj->pages);
130 static void put_pages(struct drm_gem_object *obj)
132 struct msm_gem_object *msm_obj = to_msm_bo(obj);
134 if (msm_obj->pages) {
135 /* For non-cached buffers, ensure the new pages are clean
136 * because display controller, GPU, etc. are not coherent:
138 if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
139 dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
140 msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
143 sg_free_table(msm_obj->sgt);
148 drm_gem_put_pages(obj, msm_obj->pages, true, false);
152 msm_obj->pages = NULL;
156 struct page **msm_gem_get_pages(struct drm_gem_object *obj)
158 struct msm_gem_object *msm_obj = to_msm_bo(obj);
161 mutex_lock(&msm_obj->lock);
163 if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
164 mutex_unlock(&msm_obj->lock);
165 return ERR_PTR(-EBUSY);
169 mutex_unlock(&msm_obj->lock);
173 void msm_gem_put_pages(struct drm_gem_object *obj)
175 /* when we start tracking the pin count, then do something here */
178 int msm_gem_mmap_obj(struct drm_gem_object *obj,
179 struct vm_area_struct *vma)
181 struct msm_gem_object *msm_obj = to_msm_bo(obj);
183 vma->vm_flags &= ~VM_PFNMAP;
184 vma->vm_flags |= VM_MIXEDMAP;
186 if (msm_obj->flags & MSM_BO_WC) {
187 vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
188 } else if (msm_obj->flags & MSM_BO_UNCACHED) {
189 vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
192 * Shunt off cached objs to shmem file so they have their own
193 * address_space (so unmap_mapping_range does what we want,
194 * in particular in the case of mmap'd dmabufs)
199 vma->vm_file = obj->filp;
201 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
207 int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
211 ret = drm_gem_mmap(filp, vma);
213 DBG("mmap failed: %d", ret);
217 return msm_gem_mmap_obj(vma->vm_private_data, vma);
220 int msm_gem_fault(struct vm_fault *vmf)
222 struct vm_area_struct *vma = vmf->vma;
223 struct drm_gem_object *obj = vma->vm_private_data;
224 struct msm_gem_object *msm_obj = to_msm_bo(obj);
231 * vm_ops.open/drm_gem_mmap_obj and close get and put
232 * a reference on obj. So, we dont need to hold one here.
234 ret = mutex_lock_interruptible(&msm_obj->lock);
238 if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
239 mutex_unlock(&msm_obj->lock);
240 return VM_FAULT_SIGBUS;
243 /* make sure we have pages attached now */
244 pages = get_pages(obj);
246 ret = PTR_ERR(pages);
250 /* We don't use vmf->pgoff since that has the fake offset: */
251 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
253 pfn = page_to_pfn(pages[pgoff]);
255 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
256 pfn, pfn << PAGE_SHIFT);
258 ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
261 mutex_unlock(&msm_obj->lock);
270 * EBUSY is ok: this just means that another thread
271 * already did the job.
273 return VM_FAULT_NOPAGE;
277 return VM_FAULT_SIGBUS;
281 /** get mmap offset */
282 static uint64_t mmap_offset(struct drm_gem_object *obj)
284 struct drm_device *dev = obj->dev;
285 struct msm_gem_object *msm_obj = to_msm_bo(obj);
288 WARN_ON(!mutex_is_locked(&msm_obj->lock));
290 /* Make it mmapable */
291 ret = drm_gem_create_mmap_offset(obj);
294 dev_err(dev->dev, "could not allocate mmap offset\n");
298 return drm_vma_node_offset_addr(&obj->vma_node);
301 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
304 struct msm_gem_object *msm_obj = to_msm_bo(obj);
306 mutex_lock(&msm_obj->lock);
307 offset = mmap_offset(obj);
308 mutex_unlock(&msm_obj->lock);
312 static struct msm_gem_vma *add_vma(struct drm_gem_object *obj,
313 struct msm_gem_address_space *aspace)
315 struct msm_gem_object *msm_obj = to_msm_bo(obj);
316 struct msm_gem_vma *vma;
318 WARN_ON(!mutex_is_locked(&msm_obj->lock));
320 vma = kzalloc(sizeof(*vma), GFP_KERNEL);
322 return ERR_PTR(-ENOMEM);
324 vma->aspace = aspace;
326 list_add_tail(&vma->list, &msm_obj->vmas);
331 static struct msm_gem_vma *lookup_vma(struct drm_gem_object *obj,
332 struct msm_gem_address_space *aspace)
334 struct msm_gem_object *msm_obj = to_msm_bo(obj);
335 struct msm_gem_vma *vma;
337 WARN_ON(!mutex_is_locked(&msm_obj->lock));
339 list_for_each_entry(vma, &msm_obj->vmas, list) {
340 if (vma->aspace == aspace)
347 static void del_vma(struct msm_gem_vma *vma)
352 list_del(&vma->list);
356 /* Called with msm_obj->lock locked */
358 put_iova(struct drm_gem_object *obj)
360 struct msm_gem_object *msm_obj = to_msm_bo(obj);
361 struct msm_gem_vma *vma, *tmp;
363 WARN_ON(!mutex_is_locked(&msm_obj->lock));
365 list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) {
366 msm_gem_unmap_vma(vma->aspace, vma, msm_obj->sgt);
371 /* get iova, taking a reference. Should have a matching put */
372 int msm_gem_get_iova(struct drm_gem_object *obj,
373 struct msm_gem_address_space *aspace, uint64_t *iova)
375 struct msm_gem_object *msm_obj = to_msm_bo(obj);
376 struct msm_gem_vma *vma;
379 mutex_lock(&msm_obj->lock);
381 if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
382 mutex_unlock(&msm_obj->lock);
386 vma = lookup_vma(obj, aspace);
391 vma = add_vma(obj, aspace);
397 pages = get_pages(obj);
399 ret = PTR_ERR(pages);
403 ret = msm_gem_map_vma(aspace, vma, msm_obj->sgt,
404 obj->size >> PAGE_SHIFT);
411 mutex_unlock(&msm_obj->lock);
417 mutex_unlock(&msm_obj->lock);
421 /* get iova without taking a reference, used in places where you have
422 * already done a 'msm_gem_get_iova()'.
424 uint64_t msm_gem_iova(struct drm_gem_object *obj,
425 struct msm_gem_address_space *aspace)
427 struct msm_gem_object *msm_obj = to_msm_bo(obj);
428 struct msm_gem_vma *vma;
430 mutex_lock(&msm_obj->lock);
431 vma = lookup_vma(obj, aspace);
432 mutex_unlock(&msm_obj->lock);
435 return vma ? vma->iova : 0;
438 void msm_gem_put_iova(struct drm_gem_object *obj,
439 struct msm_gem_address_space *aspace)
442 // NOTE: probably don't need a _locked() version.. we wouldn't
443 // normally unmap here, but instead just mark that it could be
444 // unmapped (if the iova refcnt drops to zero), but then later
445 // if another _get_iova_locked() fails we can start unmapping
446 // things that are no longer needed..
449 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
450 struct drm_mode_create_dumb *args)
452 args->pitch = align_pitch(args->width, args->bpp);
453 args->size = PAGE_ALIGN(args->pitch * args->height);
454 return msm_gem_new_handle(dev, file, args->size,
455 MSM_BO_SCANOUT | MSM_BO_WC, &args->handle);
458 int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
459 uint32_t handle, uint64_t *offset)
461 struct drm_gem_object *obj;
464 /* GEM does all our handle to object mapping */
465 obj = drm_gem_object_lookup(file, handle);
471 *offset = msm_gem_mmap_offset(obj);
473 drm_gem_object_unreference_unlocked(obj);
479 void *msm_gem_get_vaddr(struct drm_gem_object *obj)
481 struct msm_gem_object *msm_obj = to_msm_bo(obj);
484 mutex_lock(&msm_obj->lock);
486 if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
487 mutex_unlock(&msm_obj->lock);
488 return ERR_PTR(-EBUSY);
491 /* increment vmap_count *before* vmap() call, so shrinker can
492 * check vmap_count (is_vunmapable()) outside of msm_obj->lock.
493 * This guarantees that we won't try to msm_gem_vunmap() this
494 * same object from within the vmap() call (while we already
495 * hold msm_obj->lock)
497 msm_obj->vmap_count++;
499 if (!msm_obj->vaddr) {
500 struct page **pages = get_pages(obj);
502 ret = PTR_ERR(pages);
505 msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
506 VM_MAP, pgprot_writecombine(PAGE_KERNEL));
507 if (msm_obj->vaddr == NULL) {
513 mutex_unlock(&msm_obj->lock);
514 return msm_obj->vaddr;
517 msm_obj->vmap_count--;
518 mutex_unlock(&msm_obj->lock);
522 void msm_gem_put_vaddr(struct drm_gem_object *obj)
524 struct msm_gem_object *msm_obj = to_msm_bo(obj);
526 mutex_lock(&msm_obj->lock);
527 WARN_ON(msm_obj->vmap_count < 1);
528 msm_obj->vmap_count--;
529 mutex_unlock(&msm_obj->lock);
532 /* Update madvise status, returns true if not purged, else
535 int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
537 struct msm_gem_object *msm_obj = to_msm_bo(obj);
539 mutex_lock(&msm_obj->lock);
541 WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
543 if (msm_obj->madv != __MSM_MADV_PURGED)
544 msm_obj->madv = madv;
546 madv = msm_obj->madv;
548 mutex_unlock(&msm_obj->lock);
550 return (madv != __MSM_MADV_PURGED);
553 void msm_gem_purge(struct drm_gem_object *obj, enum msm_gem_lock subclass)
555 struct drm_device *dev = obj->dev;
556 struct msm_gem_object *msm_obj = to_msm_bo(obj);
558 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
559 WARN_ON(!is_purgeable(msm_obj));
560 WARN_ON(obj->import_attach);
562 mutex_lock_nested(&msm_obj->lock, subclass);
566 msm_gem_vunmap_locked(obj);
570 msm_obj->madv = __MSM_MADV_PURGED;
572 drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
573 drm_gem_free_mmap_offset(obj);
575 /* Our goal here is to return as much of the memory as
576 * is possible back to the system as we are called from OOM.
577 * To do this we must instruct the shmfs to drop all of its
578 * backing pages, *now*.
580 shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1);
582 invalidate_mapping_pages(file_inode(obj->filp)->i_mapping,
585 mutex_unlock(&msm_obj->lock);
588 static void msm_gem_vunmap_locked(struct drm_gem_object *obj)
590 struct msm_gem_object *msm_obj = to_msm_bo(obj);
592 WARN_ON(!mutex_is_locked(&msm_obj->lock));
594 if (!msm_obj->vaddr || WARN_ON(!is_vunmapable(msm_obj)))
597 vunmap(msm_obj->vaddr);
598 msm_obj->vaddr = NULL;
601 void msm_gem_vunmap(struct drm_gem_object *obj, enum msm_gem_lock subclass)
603 struct msm_gem_object *msm_obj = to_msm_bo(obj);
605 mutex_lock_nested(&msm_obj->lock, subclass);
606 msm_gem_vunmap_locked(obj);
607 mutex_unlock(&msm_obj->lock);
610 /* must be called before _move_to_active().. */
611 int msm_gem_sync_object(struct drm_gem_object *obj,
612 struct msm_fence_context *fctx, bool exclusive)
614 struct msm_gem_object *msm_obj = to_msm_bo(obj);
615 struct reservation_object_list *fobj;
616 struct dma_fence *fence;
619 fobj = reservation_object_get_list(msm_obj->resv);
620 if (!fobj || (fobj->shared_count == 0)) {
621 fence = reservation_object_get_excl(msm_obj->resv);
622 /* don't need to wait on our own fences, since ring is fifo */
623 if (fence && (fence->context != fctx->context)) {
624 ret = dma_fence_wait(fence, true);
630 if (!exclusive || !fobj)
633 for (i = 0; i < fobj->shared_count; i++) {
634 fence = rcu_dereference_protected(fobj->shared[i],
635 reservation_object_held(msm_obj->resv));
636 if (fence->context != fctx->context) {
637 ret = dma_fence_wait(fence, true);
646 void msm_gem_move_to_active(struct drm_gem_object *obj,
647 struct msm_gpu *gpu, bool exclusive, struct dma_fence *fence)
649 struct msm_gem_object *msm_obj = to_msm_bo(obj);
650 WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED);
653 reservation_object_add_excl_fence(msm_obj->resv, fence);
655 reservation_object_add_shared_fence(msm_obj->resv, fence);
656 list_del_init(&msm_obj->mm_list);
657 list_add_tail(&msm_obj->mm_list, &gpu->active_list);
660 void msm_gem_move_to_inactive(struct drm_gem_object *obj)
662 struct drm_device *dev = obj->dev;
663 struct msm_drm_private *priv = dev->dev_private;
664 struct msm_gem_object *msm_obj = to_msm_bo(obj);
666 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
669 list_del_init(&msm_obj->mm_list);
670 list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
673 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)
675 struct msm_gem_object *msm_obj = to_msm_bo(obj);
676 bool write = !!(op & MSM_PREP_WRITE);
677 unsigned long remain =
678 op & MSM_PREP_NOSYNC ? 0 : timeout_to_jiffies(timeout);
681 ret = reservation_object_wait_timeout_rcu(msm_obj->resv, write,
684 return remain == 0 ? -EBUSY : -ETIMEDOUT;
688 /* TODO cache maintenance */
693 int msm_gem_cpu_fini(struct drm_gem_object *obj)
695 /* TODO cache maintenance */
699 #ifdef CONFIG_DEBUG_FS
700 static void describe_fence(struct dma_fence *fence, const char *type,
703 if (!dma_fence_is_signaled(fence))
704 seq_printf(m, "\t%9s: %s %s seq %u\n", type,
705 fence->ops->get_driver_name(fence),
706 fence->ops->get_timeline_name(fence),
710 void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
712 struct msm_gem_object *msm_obj = to_msm_bo(obj);
713 struct reservation_object *robj = msm_obj->resv;
714 struct reservation_object_list *fobj;
715 struct dma_fence *fence;
716 struct msm_gem_vma *vma;
717 uint64_t off = drm_vma_node_start(&obj->vma_node);
720 mutex_lock(&msm_obj->lock);
722 switch (msm_obj->madv) {
723 case __MSM_MADV_PURGED:
726 case MSM_MADV_DONTNEED:
729 case MSM_MADV_WILLNEED:
735 seq_printf(m, "%08x: %c %2d (%2d) %08llx %p\t",
736 msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
737 obj->name, kref_read(&obj->refcount),
738 off, msm_obj->vaddr);
740 /* FIXME: we need to print the address space here too */
741 list_for_each_entry(vma, &msm_obj->vmas, list)
742 seq_printf(m, " %08llx", vma->iova);
744 seq_printf(m, " %zu%s\n", obj->size, madv);
747 fobj = rcu_dereference(robj->fence);
749 unsigned int i, shared_count = fobj->shared_count;
751 for (i = 0; i < shared_count; i++) {
752 fence = rcu_dereference(fobj->shared[i]);
753 describe_fence(fence, "Shared", m);
757 fence = rcu_dereference(robj->fence_excl);
759 describe_fence(fence, "Exclusive", m);
762 mutex_unlock(&msm_obj->lock);
765 void msm_gem_describe_objects(struct list_head *list, struct seq_file *m)
767 struct msm_gem_object *msm_obj;
771 list_for_each_entry(msm_obj, list, mm_list) {
772 struct drm_gem_object *obj = &msm_obj->base;
774 msm_gem_describe(obj, m);
779 seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
783 void msm_gem_free_object(struct drm_gem_object *obj)
785 struct drm_device *dev = obj->dev;
786 struct msm_gem_object *msm_obj = to_msm_bo(obj);
788 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
790 /* object should not be on active list: */
791 WARN_ON(is_active(msm_obj));
793 list_del(&msm_obj->mm_list);
795 mutex_lock(&msm_obj->lock);
799 if (obj->import_attach) {
801 dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);
803 /* Don't drop the pages for imported dmabuf, as they are not
804 * ours, just free the array we allocated:
807 kvfree(msm_obj->pages);
809 drm_prime_gem_destroy(obj, msm_obj->sgt);
811 msm_gem_vunmap_locked(obj);
815 if (msm_obj->resv == &msm_obj->_resv)
816 reservation_object_fini(msm_obj->resv);
818 drm_gem_object_release(obj);
820 mutex_unlock(&msm_obj->lock);
824 /* convenience method to construct a GEM buffer object, and userspace handle */
825 int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
826 uint32_t size, uint32_t flags, uint32_t *handle)
828 struct drm_gem_object *obj;
831 obj = msm_gem_new(dev, size, flags);
836 ret = drm_gem_handle_create(file, obj, handle);
838 /* drop reference from allocate - handle holds it now */
839 drm_gem_object_unreference_unlocked(obj);
844 static int msm_gem_new_impl(struct drm_device *dev,
845 uint32_t size, uint32_t flags,
846 struct reservation_object *resv,
847 struct drm_gem_object **obj,
848 bool struct_mutex_locked)
850 struct msm_drm_private *priv = dev->dev_private;
851 struct msm_gem_object *msm_obj;
853 switch (flags & MSM_BO_CACHE_MASK) {
854 case MSM_BO_UNCACHED:
859 dev_err(dev->dev, "invalid cache flag: %x\n",
860 (flags & MSM_BO_CACHE_MASK));
864 msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);
868 mutex_init(&msm_obj->lock);
870 msm_obj->flags = flags;
871 msm_obj->madv = MSM_MADV_WILLNEED;
874 msm_obj->resv = resv;
876 msm_obj->resv = &msm_obj->_resv;
877 reservation_object_init(msm_obj->resv);
880 INIT_LIST_HEAD(&msm_obj->submit_entry);
881 INIT_LIST_HEAD(&msm_obj->vmas);
883 if (struct_mutex_locked) {
884 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
885 list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
887 mutex_lock(&dev->struct_mutex);
888 list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
889 mutex_unlock(&dev->struct_mutex);
892 *obj = &msm_obj->base;
897 static struct drm_gem_object *_msm_gem_new(struct drm_device *dev,
898 uint32_t size, uint32_t flags, bool struct_mutex_locked)
900 struct msm_drm_private *priv = dev->dev_private;
901 struct drm_gem_object *obj = NULL;
902 bool use_vram = false;
905 size = PAGE_ALIGN(size);
907 if (!iommu_present(&platform_bus_type))
909 else if ((flags & MSM_BO_STOLEN) && priv->vram.size)
912 if (WARN_ON(use_vram && !priv->vram.size))
913 return ERR_PTR(-EINVAL);
915 /* Disallow zero sized objects as they make the underlying
916 * infrastructure grumpy
919 return ERR_PTR(-EINVAL);
921 ret = msm_gem_new_impl(dev, size, flags, NULL, &obj, struct_mutex_locked);
926 struct msm_gem_vma *vma;
928 struct msm_gem_object *msm_obj = to_msm_bo(obj);
930 mutex_lock(&msm_obj->lock);
932 vma = add_vma(obj, NULL);
933 mutex_unlock(&msm_obj->lock);
939 to_msm_bo(obj)->vram_node = &vma->node;
941 drm_gem_private_object_init(dev, obj, size);
943 pages = get_pages(obj);
945 ret = PTR_ERR(pages);
949 vma->iova = physaddr(obj);
951 ret = drm_gem_object_init(dev, obj, size);
959 drm_gem_object_unreference_unlocked(obj);
963 struct drm_gem_object *msm_gem_new_locked(struct drm_device *dev,
964 uint32_t size, uint32_t flags)
966 return _msm_gem_new(dev, size, flags, true);
969 struct drm_gem_object *msm_gem_new(struct drm_device *dev,
970 uint32_t size, uint32_t flags)
972 return _msm_gem_new(dev, size, flags, false);
975 struct drm_gem_object *msm_gem_import(struct drm_device *dev,
976 struct dma_buf *dmabuf, struct sg_table *sgt)
978 struct msm_gem_object *msm_obj;
979 struct drm_gem_object *obj;
983 /* if we don't have IOMMU, don't bother pretending we can import: */
984 if (!iommu_present(&platform_bus_type)) {
985 dev_err(dev->dev, "cannot import without IOMMU\n");
986 return ERR_PTR(-EINVAL);
989 size = PAGE_ALIGN(dmabuf->size);
991 ret = msm_gem_new_impl(dev, size, MSM_BO_WC, dmabuf->resv, &obj, false);
995 drm_gem_private_object_init(dev, obj, size);
997 npages = size / PAGE_SIZE;
999 msm_obj = to_msm_bo(obj);
1000 mutex_lock(&msm_obj->lock);
1002 msm_obj->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
1003 if (!msm_obj->pages) {
1004 mutex_unlock(&msm_obj->lock);
1009 ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
1011 mutex_unlock(&msm_obj->lock);
1015 mutex_unlock(&msm_obj->lock);
1019 drm_gem_object_unreference_unlocked(obj);
1020 return ERR_PTR(ret);
1023 static void *_msm_gem_kernel_new(struct drm_device *dev, uint32_t size,
1024 uint32_t flags, struct msm_gem_address_space *aspace,
1025 struct drm_gem_object **bo, uint64_t *iova, bool locked)
1028 struct drm_gem_object *obj = _msm_gem_new(dev, size, flags, locked);
1032 return ERR_CAST(obj);
1035 ret = msm_gem_get_iova(obj, aspace, iova);
1037 drm_gem_object_unreference(obj);
1038 return ERR_PTR(ret);
1042 vaddr = msm_gem_get_vaddr(obj);
1043 if (IS_ERR(vaddr)) {
1044 msm_gem_put_iova(obj, aspace);
1045 drm_gem_object_unreference(obj);
1046 return ERR_CAST(vaddr);
1055 void *msm_gem_kernel_new(struct drm_device *dev, uint32_t size,
1056 uint32_t flags, struct msm_gem_address_space *aspace,
1057 struct drm_gem_object **bo, uint64_t *iova)
1059 return _msm_gem_kernel_new(dev, size, flags, aspace, bo, iova, false);
1062 void *msm_gem_kernel_new_locked(struct drm_device *dev, uint32_t size,
1063 uint32_t flags, struct msm_gem_address_space *aspace,
1064 struct drm_gem_object **bo, uint64_t *iova)
1066 return _msm_gem_kernel_new(dev, size, flags, aspace, bo, iova, true);