2 * Copyright 2008 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
27 #include "radeon_drm.h"
28 #include "radeon_drv.h"
30 static int radeon_gem_ib_init(struct drm_device *dev);
31 static int radeon_gem_ib_destroy(struct drm_device *dev);
32 static int radeon_gem_dma_bufs_init(struct drm_device *dev);
33 static void radeon_gem_dma_bufs_destroy(struct drm_device *dev);
35 int radeon_gem_init_object(struct drm_gem_object *obj)
37 struct drm_radeon_gem_object *obj_priv;
39 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
44 obj->driver_private = obj_priv;
50 void radeon_gem_free_object(struct drm_gem_object *obj)
53 struct drm_radeon_gem_object *obj_priv = obj->driver_private;
55 /* tear down the buffer object - gem holds struct mutex */
56 drm_bo_takedown_vm_locked(obj_priv->bo);
57 drm_bo_usage_deref_locked(&obj_priv->bo);
58 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
61 int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
62 struct drm_file *file_priv)
64 struct drm_radeon_private *dev_priv = dev->dev_private;
65 struct drm_radeon_gem_info *args = data;
67 args->vram_start = dev_priv->mm.vram_offset;
68 args->vram_size = dev_priv->mm.vram_size;
69 args->vram_visible = dev_priv->mm.vram_visible;
71 args->gart_start = dev_priv->mm.gart_start;
72 args->gart_size = dev_priv->mm.gart_size;
77 struct drm_gem_object *radeon_gem_object_alloc(struct drm_device *dev, int size, int alignment,
80 struct drm_gem_object *obj;
81 struct drm_radeon_gem_object *obj_priv;
85 DRM_DEBUG("size 0x%x, alignment %d, initial_domain %d\n", size, alignment, initial_domain);
86 obj = drm_gem_object_alloc(dev, size);
90 obj_priv = obj->driver_private;
91 if (initial_domain == RADEON_GEM_DOMAIN_VRAM)
92 flags = DRM_BO_FLAG_MEM_VRAM | DRM_BO_FLAG_MAPPABLE;
94 flags = DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_MAPPABLE;
96 flags |= DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_EXE;
98 ret = drm_buffer_object_create(dev,
99 size, drm_bo_type_device,
111 int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
112 struct drm_file *file_priv)
114 struct drm_radeon_gem_create *args = data;
115 struct drm_radeon_gem_object *obj_priv;
116 struct drm_gem_object *obj;
121 /* create a gem object to contain this object in */
122 args->size = roundup(args->size, PAGE_SIZE);
124 obj = radeon_gem_object_alloc(dev, args->size, args->alignment, args->initial_domain);
128 obj_priv = obj->driver_private;
129 DRM_DEBUG("obj is %p bo is %p, %d\n", obj, obj_priv->bo, obj_priv->bo->num_pages);
130 ret = drm_gem_handle_create(file_priv, obj, &handle);
131 mutex_lock(&dev->struct_mutex);
132 drm_gem_object_handle_unreference(obj);
133 mutex_unlock(&dev->struct_mutex);
138 args->handle = handle;
142 drm_gem_object_unreference(obj);
147 int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
148 struct drm_file *file_priv)
150 /* transition the BO to a domain - just validate the BO into a certain domain */
151 struct drm_radeon_gem_set_domain *args = data;
152 struct drm_gem_object *obj;
153 struct drm_radeon_gem_object *obj_priv;
155 /* for now if someone requests domain CPU - just make sure the buffer is finished with */
157 /* just do a BO wait for now */
158 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
162 obj_priv = obj->driver_private;
164 mutex_lock(&obj_priv->bo->mutex);
165 ret = drm_bo_wait(obj_priv->bo, 0, 1, 0, 0);
166 mutex_unlock(&obj_priv->bo->mutex);
168 mutex_lock(&dev->struct_mutex);
169 drm_gem_object_unreference(obj);
170 mutex_unlock(&dev->struct_mutex);
174 int radeon_gem_pread_ioctl(struct drm_device *dev, void *data,
175 struct drm_file *file_priv)
180 int radeon_gem_pwrite_ioctl(struct drm_device *dev, void *data,
181 struct drm_file *file_priv)
186 int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
187 struct drm_file *file_priv)
189 struct drm_radeon_gem_mmap *args = data;
190 struct drm_gem_object *obj;
191 struct drm_radeon_gem_object *obj_priv;
195 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
199 offset = args->offset;
201 DRM_DEBUG("got here %p\n", obj);
202 obj_priv = obj->driver_private;
204 DRM_DEBUG("got here %p %p %lld %ld\n", obj, obj_priv->bo, args->size, obj_priv->bo->num_pages);
206 mutex_lock(&dev->struct_mutex);
207 drm_gem_object_unreference(obj);
208 mutex_unlock(&dev->struct_mutex);
212 down_write(¤t->mm->mmap_sem);
213 addr = do_mmap_pgoff(file_priv->filp, 0, args->size,
214 PROT_READ | PROT_WRITE, MAP_SHARED,
215 obj_priv->bo->map_list.hash.key);
216 up_write(¤t->mm->mmap_sem);
218 DRM_DEBUG("got here %p\n", obj);
219 mutex_lock(&dev->struct_mutex);
220 drm_gem_object_unreference(obj);
221 mutex_unlock(&dev->struct_mutex);
222 if (IS_ERR((void *)addr))
225 args->addr_ptr = (uint64_t) addr;
231 int radeon_gem_pin_ioctl(struct drm_device *dev, void *data,
232 struct drm_file *file_priv)
234 struct drm_radeon_gem_pin *args = data;
235 struct drm_gem_object *obj;
236 struct drm_radeon_gem_object *obj_priv;
239 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
243 obj_priv = obj->driver_private;
245 DRM_DEBUG("got here %p %p %d\n", obj, obj_priv->bo, atomic_read(&obj_priv->bo->usage));
246 /* validate into a pin with no fence */
248 if (!(obj_priv->bo->type != drm_bo_type_kernel && !DRM_SUSER(DRM_CURPROC))) {
249 ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
250 DRM_BO_HINT_DONT_FENCE, 0);
254 args->offset = obj_priv->bo->offset;
255 DRM_DEBUG("got here %p %p\n", obj, obj_priv->bo);
257 mutex_lock(&dev->struct_mutex);
258 drm_gem_object_unreference(obj);
259 mutex_unlock(&dev->struct_mutex);
263 int radeon_gem_unpin_ioctl(struct drm_device *dev, void *data,
264 struct drm_file *file_priv)
266 struct drm_radeon_gem_unpin *args = data;
267 struct drm_gem_object *obj;
268 struct drm_radeon_gem_object *obj_priv;
271 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
275 obj_priv = obj->driver_private;
277 /* validate into a pin with no fence */
279 ret = drm_bo_do_validate(obj_priv->bo, DRM_BO_FLAG_NO_EVICT, DRM_BO_FLAG_NO_EVICT,
280 DRM_BO_HINT_DONT_FENCE, 0);
282 mutex_lock(&dev->struct_mutex);
283 drm_gem_object_unreference(obj);
284 mutex_unlock(&dev->struct_mutex);
288 int radeon_gem_busy(struct drm_device *dev, void *data,
289 struct drm_file *file_priv)
294 int radeon_gem_execbuffer(struct drm_device *dev, void *data,
295 struct drm_file *file_priv)
302 int radeon_gem_indirect_ioctl(struct drm_device *dev, void *data,
303 struct drm_file *file_priv)
305 struct drm_radeon_gem_indirect *args = data;
306 struct drm_radeon_private *dev_priv = dev->dev_private;
307 struct drm_gem_object *obj;
308 struct drm_radeon_gem_object *obj_priv;
313 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
317 obj_priv = obj->driver_private;
319 DRM_DEBUG("got here %p %d\n", obj, args->used);
320 //RING_SPACE_TEST_WITH_RETURN(dev_priv);
321 //VB_AGE_TEST_WITH_RETURN(dev_priv);
323 ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
328 /* Wait for the 3D stream to idle before the indirect buffer
329 * containing 2D acceleration commands is processed.
333 RADEON_WAIT_UNTIL_3D_IDLE();
341 int offset = (dev_priv->gart_vm_start +
342 + obj_priv->bo->offset + start);
343 int dwords = (end - start + 3) / sizeof(u32);
346 /* Indirect buffer data must be an even number of
347 * dwords, so if we've been given an odd number we must
348 * pad the data with a Type-2 CP packet.
352 ((char *)dev->agp_buffer_map->handle
353 + buf->offset + start);
354 data[dwords++] = RADEON_CP_PACKET2;
357 /* Fire off the indirect buffer */
360 OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
369 /* we need to fence the buffer */
370 ret = drm_fence_buffer_objects(dev, NULL, 0, NULL, &obj_priv->fence);
373 drm_putback_buffer_objects(dev);
378 /* dereference he fence object */
379 drm_fence_usage_deref_unlocked(&obj_priv->fence);
381 mutex_lock(&dev->struct_mutex);
382 drm_gem_object_unreference(obj);
383 mutex_unlock(&dev->struct_mutex);
390 * Depending on card genertation, chipset bugs, etc... the amount of vram
391 * accessible to the CPU can vary. This function is our best shot at figuring
392 * it out. Returns a value in KB.
394 static uint32_t radeon_get_accessible_vram(struct drm_device *dev)
396 drm_radeon_private_t *dev_priv = dev->dev_private;
400 if (dev_priv->chip_family >= CHIP_R600)
401 aper_size = RADEON_READ(R600_CONFIG_APER_SIZE) / 1024;
403 aper_size = RADEON_READ(RADEON_CONFIG_APER_SIZE) / 1024;
405 /* Set HDP_APER_CNTL only on cards that are known not to be broken,
406 * that is has the 2nd generation multifunction PCI interface
408 if (dev_priv->chip_family == CHIP_RV280 ||
409 dev_priv->chip_family == CHIP_RV350 ||
410 dev_priv->chip_family == CHIP_RV380 ||
411 dev_priv->chip_family == CHIP_R420 ||
412 dev_priv->chip_family == CHIP_RV410 ||
413 dev_priv->chip_family >= CHIP_RS600) {
414 uint32_t temp = RADEON_READ(RADEON_HOST_PATH_CNTL);
415 temp |= RADEON_HDP_APER_CNTL;
416 RADEON_WRITE(RADEON_HOST_PATH_CNTL, temp);
417 return aper_size * 2;
420 /* Older cards have all sorts of funny issues to deal with. First
421 * check if it's a multifunction card by reading the PCI config
422 * header type... Limit those to one aperture size
424 pci_read_config_byte(dev->pdev, 0xe, &byte);
428 /* Single function older card. We read HDP_APER_CNTL to see how the BIOS
429 * have set it up. We don't write this as it's broken on some ASICs but
430 * we expect the BIOS to have done the right thing (might be too optimistic...)
432 if (RADEON_READ(RADEON_HOST_PATH_CNTL) & RADEON_HDP_APER_CNTL)
433 return aper_size * 2;
438 /* code from the DDX - do memory sizing */
439 void radeon_vram_setup(struct drm_device *dev)
441 drm_radeon_private_t *dev_priv = dev->dev_private;
443 uint32_t accessible, bar_size;
445 if ((dev_priv->chip_family <= CHIP_RV515) && (dev_priv->flags & RADEON_IS_IGP)) {
446 uint32_t tom = RADEON_READ(RADEON_NB_TOM);
448 vram = (((tom >> 16) - (tom & 0xffff) + 1) << 6);
449 RADEON_WRITE(RADEON_CONFIG_MEMSIZE, vram * 1024);
451 if (dev_priv->chip_family >= CHIP_R600)
452 vram = RADEON_READ(R600_CONFIG_MEMSIZE) / 1024;
454 vram = RADEON_READ(RADEON_CONFIG_MEMSIZE) / 1024;
456 /* Some production boards of m6 will return 0 if it's 8 MB */
459 RADEON_WRITE(RADEON_CONFIG_MEMSIZE, 0x800000);
464 accessible = radeon_get_accessible_vram(dev);
466 bar_size = drm_get_resource_len(dev, 0) / 1024;
469 if (accessible > bar_size)
470 accessible = bar_size;
472 DRM_INFO("Detected VRAM RAM=%dK, accessible=%uK, BAR=%uK\n",
473 vram, accessible, bar_size);
475 dev_priv->mm.vram_offset = dev_priv->fb_aper_offset;
476 dev_priv->mm.vram_size = vram * 1024;
477 dev_priv->mm.vram_visible = accessible * 1024;
482 static int radeon_gart_init(struct drm_device *dev)
484 drm_radeon_private_t *dev_priv = dev->dev_private;
488 /* setup a 32MB GART */
489 dev_priv->gart_size = dev_priv->mm.gart_size;
490 dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
493 /* setup VRAM vs GART here */
494 if (dev_priv->flags & RADEON_IS_AGP) {
495 base = dev->agp->base;
496 if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
497 base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
498 DRM_INFO("Can't use agp base @0x%08xlx, won't fit\n",
506 base = dev_priv->fb_location + dev_priv->fb_size;
507 if (base < dev_priv->fb_location ||
508 ((base + dev_priv->gart_size) & 0xfffffffful) < base)
509 base = dev_priv->fb_location
510 - dev_priv->gart_size;
512 /* start on the card */
513 dev_priv->gart_vm_start = base & 0xffc00000u;
514 if (dev_priv->gart_vm_start != base)
515 DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
516 base, dev_priv->gart_vm_start);
518 /* if on PCIE we need to allocate an fb object for the PCIE GART table */
519 if (dev_priv->flags & RADEON_IS_PCIE) {
520 ret = drm_buffer_object_create(dev, RADEON_PCIGART_TABLE_SIZE,
522 DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_VRAM | DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
523 0, 1, 0, &dev_priv->mm.pcie_table.bo);
527 DRM_DEBUG("pcie table bo created %p, %x\n", dev_priv->mm.pcie_table.bo, dev_priv->mm.pcie_table.bo->offset);
528 ret = drm_bo_kmap(dev_priv->mm.pcie_table.bo, 0, RADEON_PCIGART_TABLE_SIZE >> PAGE_SHIFT,
529 &dev_priv->mm.pcie_table.kmap);
533 dev_priv->pcigart_offset_set = 2;
534 dev_priv->gart_info.bus_addr = dev_priv->fb_location + dev_priv->mm.pcie_table.bo->offset;
535 dev_priv->gart_info.addr = dev_priv->mm.pcie_table.kmap.virtual;
536 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
537 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_FB;
538 memset(dev_priv->gart_info.addr, 0, RADEON_PCIGART_TABLE_SIZE);
539 } else if (!(dev_priv->flags & RADEON_IS_AGP)) {
540 /* allocate PCI GART table */
541 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
542 ret = drm_ati_alloc_pcigart_table(dev, &dev_priv->gart_info);
544 DRM_ERROR("cannot allocate PCI GART page!\n");
548 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
549 if (dev_priv->flags & RADEON_IS_IGPGART)
550 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
552 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
553 dev_priv->gart_info.addr = dev_priv->gart_info.table_handle->vaddr;
554 dev_priv->gart_info.bus_addr = dev_priv->gart_info.table_handle->busaddr;
557 /* gart values setup - start the GART */
558 if (dev_priv->flags & RADEON_IS_AGP) {
559 radeon_set_pcigart(dev_priv, 0);
561 radeon_set_pcigart(dev_priv, 1);
567 int radeon_alloc_gart_objects(struct drm_device *dev)
569 drm_radeon_private_t *dev_priv = dev->dev_private;
572 ret = drm_buffer_object_create(dev, RADEON_DEFAULT_RING_SIZE,
574 DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
575 DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
576 0, 1, 0, &dev_priv->mm.ring.bo);
578 DRM_ERROR("failed to allocate ring\n");
582 ret = drm_bo_kmap(dev_priv->mm.ring.bo, 0, RADEON_DEFAULT_RING_SIZE >> PAGE_SHIFT,
583 &dev_priv->mm.ring.kmap);
585 DRM_ERROR("failed to map ring\n");
589 ret = drm_buffer_object_create(dev, PAGE_SIZE,
591 DRM_BO_FLAG_WRITE |DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
592 DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
593 0, 1, 0, &dev_priv->mm.ring_read.bo);
595 DRM_ERROR("failed to allocate ring read\n");
599 ret = drm_bo_kmap(dev_priv->mm.ring_read.bo, 0,
600 PAGE_SIZE >> PAGE_SHIFT,
601 &dev_priv->mm.ring_read.kmap);
603 DRM_ERROR("failed to map ring read\n");
607 DRM_DEBUG("Ring ptr %p mapped at %d %p, read ptr %p maped at %d %p\n",
608 dev_priv->mm.ring.bo, dev_priv->mm.ring.bo->offset, dev_priv->mm.ring.kmap.virtual,
609 dev_priv->mm.ring_read.bo, dev_priv->mm.ring_read.bo->offset, dev_priv->mm.ring_read.kmap.virtual);
611 /* init the indirect buffers */
612 radeon_gem_ib_init(dev);
613 radeon_gem_dma_bufs_init(dev);
618 static void radeon_init_memory_map(struct drm_device *dev)
620 drm_radeon_private_t *dev_priv = dev->dev_private;
621 u32 mem_size, aper_size;
623 dev_priv->mc_fb_location = radeon_read_fb_location(dev_priv);
624 radeon_read_agp_location(dev_priv, &dev_priv->mc_agp_loc_lo, &dev_priv->mc_agp_loc_hi);
626 if (dev_priv->chip_family >= CHIP_R600) {
627 mem_size = RADEON_READ(R600_CONFIG_MEMSIZE);
628 aper_size = RADEON_READ(R600_CONFIG_APER_SIZE);
630 mem_size = RADEON_READ(RADEON_CONFIG_MEMSIZE);
631 aper_size = RADEON_READ(RADEON_CONFIG_APER_SIZE);
634 /* M6s report illegal memory size */
636 mem_size = 8 * 1024 * 1024;
638 /* for RN50/M6/M7 - Novell bug 204882 */
639 if (aper_size > mem_size)
640 mem_size = aper_size;
642 if ((dev_priv->chip_family != CHIP_RS600) &&
643 (dev_priv->chip_family != CHIP_RS690) &&
644 (dev_priv->chip_family != CHIP_RS740)) {
645 if (dev_priv->flags & RADEON_IS_IGP)
646 dev_priv->mc_fb_location = RADEON_READ(RADEON_NB_TOM);
650 if (dev_priv->chip_family >= CHIP_R600)
651 aper0_base = RADEON_READ(R600_CONFIG_F0_BASE);
653 aper0_base = RADEON_READ(RADEON_CONFIG_APER_0_BASE);
656 /* Some chips have an "issue" with the memory controller, the
657 * location must be aligned to the size. We just align it down,
658 * too bad if we walk over the top of system memory, we don't
659 * use DMA without a remapped anyway.
660 * Affected chips are rv280, all r3xx, and all r4xx, but not IGP
662 if (dev_priv->chip_family == CHIP_RV280 ||
663 dev_priv->chip_family == CHIP_R300 ||
664 dev_priv->chip_family == CHIP_R350 ||
665 dev_priv->chip_family == CHIP_RV350 ||
666 dev_priv->chip_family == CHIP_RV380 ||
667 dev_priv->chip_family == CHIP_R420 ||
668 dev_priv->chip_family == CHIP_RV410)
669 aper0_base &= ~(mem_size - 1);
671 if (dev_priv->chip_family >= CHIP_R600) {
672 dev_priv->mc_fb_location = (aper0_base >> 24) |
673 (((aper0_base + mem_size - 1) & 0xff000000U) >> 8);
675 dev_priv->mc_fb_location = (aper0_base >> 16) |
676 ((aper0_base + mem_size - 1) & 0xffff0000U);
681 if (dev_priv->chip_family >= CHIP_R600)
682 dev_priv->fb_location = (dev_priv->mc_fb_location & 0xffff) << 24;
684 dev_priv->fb_location = (dev_priv->mc_fb_location & 0xffff) << 16;
686 if (radeon_is_avivo(dev_priv)) {
687 if (dev_priv->chip_family >= CHIP_R600)
688 RADEON_WRITE(R600_HDP_NONSURFACE_BASE, (dev_priv->mc_fb_location << 16) & 0xff0000);
690 RADEON_WRITE(AVIVO_HDP_FB_LOCATION, dev_priv->mc_fb_location);
693 radeon_write_fb_location(dev_priv, dev_priv->mc_fb_location);
695 dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
697 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
698 - dev_priv->fb_location;
702 /* init memory manager - start with all of VRAM and a 32MB GART aperture for now */
703 int radeon_gem_mm_init(struct drm_device *dev)
705 drm_radeon_private_t *dev_priv = dev->dev_private;
709 /* size the mappable VRAM memory for now */
710 radeon_vram_setup(dev);
712 radeon_init_memory_map(dev);
714 #define VRAM_RESERVE_TEXT (64*1024)
715 dev_priv->mm.vram_visible -= VRAM_RESERVE_TEXT;
716 pg_offset = VRAM_RESERVE_TEXT >> PAGE_SHIFT;
717 drm_bo_init_mm(dev, DRM_BO_MEM_VRAM, pg_offset, /*dev_priv->mm.vram_offset >> PAGE_SHIFT,*/
718 ((dev_priv->mm.vram_visible) >> PAGE_SHIFT) - 16,
722 dev_priv->mm.gart_size = (32 * 1024 * 1024);
723 dev_priv->mm.gart_start = 0;
724 ret = radeon_gart_init(dev);
728 drm_bo_init_mm(dev, DRM_BO_MEM_TT, 0,
729 dev_priv->mm.gart_size >> PAGE_SHIFT,
732 /* need to allocate some objects in the GART */
733 /* ring + ring read ptr */
734 ret = radeon_alloc_gart_objects(dev);
738 dev_priv->mm_enabled = true;
742 void radeon_gem_mm_fini(struct drm_device *dev)
744 drm_radeon_private_t *dev_priv = dev->dev_private;
746 radeon_gem_dma_bufs_destroy(dev);
747 radeon_gem_ib_destroy(dev);
749 mutex_lock(&dev->struct_mutex);
752 if (dev_priv->mm.ring_read.bo) {
753 drm_bo_kunmap(&dev_priv->mm.ring_read.kmap);
754 drm_bo_usage_deref_locked(&dev_priv->mm.ring_read.bo);
757 if (dev_priv->mm.ring.bo) {
758 drm_bo_kunmap(&dev_priv->mm.ring.kmap);
759 drm_bo_usage_deref_locked(&dev_priv->mm.ring.bo);
762 if (drm_bo_clean_mm(dev, DRM_BO_MEM_TT, 1)) {
763 DRM_DEBUG("delaying takedown of TTM memory\n");
766 if (dev_priv->flags & RADEON_IS_PCIE) {
767 if (dev_priv->mm.pcie_table.bo) {
768 drm_bo_kunmap(&dev_priv->mm.pcie_table.kmap);
769 drm_bo_usage_deref_locked(&dev_priv->mm.pcie_table.bo);
773 if (drm_bo_clean_mm(dev, DRM_BO_MEM_VRAM, 1)) {
774 DRM_DEBUG("delaying takedown of TTM memory\n");
777 mutex_unlock(&dev->struct_mutex);
779 drm_bo_driver_finish(dev);
780 dev_priv->mm_enabled = false;
783 int radeon_gem_object_pin(struct drm_gem_object *obj,
786 struct drm_radeon_gem_object *obj_priv;
789 obj_priv = obj->driver_private;
791 ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
792 DRM_BO_HINT_DONT_FENCE, 0);
797 #define RADEON_IB_MEMORY (1*1024*1024)
798 #define RADEON_IB_SIZE (65536)
800 #define RADEON_NUM_IB (RADEON_IB_MEMORY / RADEON_IB_SIZE)
802 int radeon_gem_ib_get(struct drm_device *dev, void **ib, uint32_t dwords, uint32_t *card_offset)
806 drm_radeon_private_t *dev_priv = dev->dev_private;
808 for (i = 0; i < RADEON_NUM_IB; i++) {
809 if (!(dev_priv->ib_alloc_bitmap & (1 << i))){
815 /* if all in use we need to wait */
817 for (i = 0; i < RADEON_NUM_IB; i++) {
818 if (dev_priv->ib_alloc_bitmap & (1 << i)) {
819 mutex_lock(&dev_priv->ib_objs[i]->bo->mutex);
820 ret = drm_bo_wait(dev_priv->ib_objs[i]->bo, 0, 1, 0, 0);
821 mutex_unlock(&dev_priv->ib_objs[i]->bo->mutex);
824 dev_priv->ib_alloc_bitmap &= ~(1 << i);
832 DRM_ERROR("Major case fail to allocate IB from freelist %x\n", dev_priv->ib_alloc_bitmap);
837 if (dwords > RADEON_IB_SIZE / sizeof(uint32_t))
840 ret = drm_bo_do_validate(dev_priv->ib_objs[index]->bo, 0,
841 DRM_BO_FLAG_NO_EVICT,
844 DRM_ERROR("Failed to validate IB %d\n", index);
848 *card_offset = dev_priv->gart_vm_start + dev_priv->ib_objs[index]->bo->offset;
849 *ib = dev_priv->ib_objs[index]->kmap.virtual;
850 dev_priv->ib_alloc_bitmap |= (1 << i);
854 static void radeon_gem_ib_free(struct drm_device *dev, void *ib, uint32_t dwords)
856 drm_radeon_private_t *dev_priv = dev->dev_private;
857 struct drm_fence_object *fence;
861 for (i = 0; i < RADEON_NUM_IB; i++) {
863 if (dev_priv->ib_objs[i]->kmap.virtual == ib) {
864 /* emit a fence object */
865 ret = drm_fence_buffer_objects(dev, NULL, 0, NULL, &fence);
868 drm_putback_buffer_objects(dev);
870 /* dereference the fence object */
872 drm_fence_usage_deref_unlocked(&fence);
878 static int radeon_gem_ib_destroy(struct drm_device *dev)
880 drm_radeon_private_t *dev_priv = dev->dev_private;
883 if (dev_priv->ib_objs) {
884 for (i = 0; i < RADEON_NUM_IB; i++) {
885 if (dev_priv->ib_objs[i]) {
886 drm_bo_kunmap(&dev_priv->ib_objs[i]->kmap);
887 drm_bo_usage_deref_unlocked(&dev_priv->ib_objs[i]->bo);
889 drm_free(dev_priv->ib_objs[i], sizeof(struct radeon_mm_obj), DRM_MEM_DRIVER);
891 drm_free(dev_priv->ib_objs, RADEON_NUM_IB*sizeof(struct radeon_mm_obj *), DRM_MEM_DRIVER);
893 dev_priv->ib_objs = NULL;
897 static int radeon_gem_relocate(struct drm_device *dev, struct drm_file *file_priv,
898 uint32_t *reloc, uint32_t *offset)
900 drm_radeon_private_t *dev_priv = dev->dev_private;
901 /* relocate the handle */
902 int domains = reloc[2];
903 struct drm_gem_object *obj;
906 struct drm_radeon_gem_object *obj_priv;
908 obj = drm_gem_object_lookup(dev, file_priv, reloc[1]);
912 obj_priv = obj->driver_private;
913 if (domains == RADEON_GEM_DOMAIN_VRAM) {
914 flags = DRM_BO_FLAG_MEM_VRAM;
916 flags = DRM_BO_FLAG_MEM_TT;
919 ret = drm_bo_do_validate(obj_priv->bo, flags, DRM_BO_MASK_MEM, 0, 0);
923 if (flags == DRM_BO_FLAG_MEM_VRAM)
924 *offset = obj_priv->bo->offset + dev_priv->fb_location;
926 *offset = obj_priv->bo->offset + dev_priv->gart_vm_start;
928 /* BAD BAD BAD - LINKED LIST THE OBJS and UNREF ONCE IB is SUBMITTED */
929 drm_gem_object_unreference(obj);
933 /* allocate 1MB of 64k IBs the the kernel can keep mapped */
934 static int radeon_gem_ib_init(struct drm_device *dev)
936 drm_radeon_private_t *dev_priv = dev->dev_private;
940 dev_priv->ib_objs = drm_calloc(RADEON_NUM_IB, sizeof(struct radeon_mm_obj *), DRM_MEM_DRIVER);
941 if (!dev_priv->ib_objs)
944 for (i = 0; i < RADEON_NUM_IB; i++) {
945 dev_priv->ib_objs[i] = drm_calloc(1, sizeof(struct radeon_mm_obj), DRM_MEM_DRIVER);
946 if (!dev_priv->ib_objs[i])
949 ret = drm_buffer_object_create(dev, RADEON_IB_SIZE,
951 DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
952 DRM_BO_FLAG_MAPPABLE, 0,
953 0, 0, &dev_priv->ib_objs[i]->bo);
957 ret = drm_bo_kmap(dev_priv->ib_objs[i]->bo, 0, RADEON_IB_SIZE >> PAGE_SHIFT,
958 &dev_priv->ib_objs[i]->kmap);
964 dev_priv->ib_alloc_bitmap = 0;
966 dev_priv->cs.ib_get = radeon_gem_ib_get;
967 dev_priv->cs.ib_free = radeon_gem_ib_free;
970 dev_priv->cs.relocate = radeon_gem_relocate;
974 radeon_gem_ib_destroy(dev);
978 #define RADEON_DMA_BUFFER_SIZE (64 * 1024)
979 #define RADEON_DMA_BUFFER_COUNT (16)
983 * Cleanup after an error on one of the addbufs() functions.
985 * \param dev DRM device.
986 * \param entry buffer entry where the error occurred.
988 * Frees any pages and buffers associated with the given entry.
990 static void drm_cleanup_buf_error(struct drm_device * dev,
991 struct drm_buf_entry * entry)
995 if (entry->seg_count) {
996 for (i = 0; i < entry->seg_count; i++) {
997 if (entry->seglist[i]) {
998 drm_pci_free(dev, entry->seglist[i]);
1001 drm_free(entry->seglist,
1003 sizeof(*entry->seglist), DRM_MEM_SEGS);
1005 entry->seg_count = 0;
1008 if (entry->buf_count) {
1009 for (i = 0; i < entry->buf_count; i++) {
1010 if (entry->buflist[i].dev_private) {
1011 drm_free(entry->buflist[i].dev_private,
1012 entry->buflist[i].dev_priv_size,
1016 drm_free(entry->buflist,
1018 sizeof(*entry->buflist), DRM_MEM_BUFS);
1020 entry->buf_count = 0;
1024 static int radeon_gem_addbufs(struct drm_device *dev)
1026 struct drm_radeon_private *dev_priv = dev->dev_private;
1027 struct drm_device_dma *dma = dev->dma;
1028 struct drm_buf_entry *entry;
1029 struct drm_buf *buf;
1030 unsigned long offset;
1031 unsigned long agp_offset;
1040 struct drm_buf **temp_buflist;
1045 count = RADEON_DMA_BUFFER_COUNT;
1046 order = drm_order(RADEON_DMA_BUFFER_SIZE);
1049 alignment = PAGE_ALIGN(size);
1050 page_order = order - PAGE_SHIFT > 0 ? order - PAGE_SHIFT : 0;
1051 total = PAGE_SIZE << page_order;
1054 agp_offset = dev_priv->mm.dma_bufs.bo->offset;
1056 DRM_DEBUG("count: %d\n", count);
1057 DRM_DEBUG("order: %d\n", order);
1058 DRM_DEBUG("size: %d\n", size);
1059 DRM_DEBUG("agp_offset: %lu\n", agp_offset);
1060 DRM_DEBUG("alignment: %d\n", alignment);
1061 DRM_DEBUG("page_order: %d\n", page_order);
1062 DRM_DEBUG("total: %d\n", total);
1064 if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
1066 if (dev->queue_count)
1067 return -EBUSY; /* Not while in use */
1069 spin_lock(&dev->count_lock);
1071 spin_unlock(&dev->count_lock);
1074 atomic_inc(&dev->buf_alloc);
1075 spin_unlock(&dev->count_lock);
1077 mutex_lock(&dev->struct_mutex);
1078 entry = &dma->bufs[order];
1079 if (entry->buf_count) {
1080 mutex_unlock(&dev->struct_mutex);
1081 atomic_dec(&dev->buf_alloc);
1082 return -ENOMEM; /* May only call once for each order */
1085 if (count < 0 || count > 4096) {
1086 mutex_unlock(&dev->struct_mutex);
1087 atomic_dec(&dev->buf_alloc);
1091 entry->buflist = drm_alloc(count * sizeof(*entry->buflist),
1093 if (!entry->buflist) {
1094 mutex_unlock(&dev->struct_mutex);
1095 atomic_dec(&dev->buf_alloc);
1098 memset(entry->buflist, 0, count * sizeof(*entry->buflist));
1100 entry->buf_size = size;
1101 entry->page_order = page_order;
1105 while (entry->buf_count < count) {
1106 buf = &entry->buflist[entry->buf_count];
1107 buf->idx = dma->buf_count + entry->buf_count;
1108 buf->total = alignment;
1112 buf->offset = (dma->byte_count + offset);
1113 buf->bus_address = dev_priv->gart_vm_start + agp_offset + offset;
1114 buf->address = (void *)(agp_offset + offset);
1118 init_waitqueue_head(&buf->dma_wait);
1119 buf->file_priv = NULL;
1121 buf->dev_priv_size = dev->driver->dev_priv_size;
1122 buf->dev_private = drm_alloc(buf->dev_priv_size, DRM_MEM_BUFS);
1123 if (!buf->dev_private) {
1124 /* Set count correctly so we free the proper amount. */
1125 entry->buf_count = count;
1126 drm_cleanup_buf_error(dev, entry);
1127 mutex_unlock(&dev->struct_mutex);
1128 atomic_dec(&dev->buf_alloc);
1132 memset(buf->dev_private, 0, buf->dev_priv_size);
1134 DRM_DEBUG("buffer %d @ %p\n", entry->buf_count, buf->address);
1136 offset += alignment;
1138 byte_count += PAGE_SIZE << page_order;
1141 DRM_DEBUG("byte_count: %d\n", byte_count);
1143 temp_buflist = drm_realloc(dma->buflist,
1144 dma->buf_count * sizeof(*dma->buflist),
1145 (dma->buf_count + entry->buf_count)
1146 * sizeof(*dma->buflist), DRM_MEM_BUFS);
1147 if (!temp_buflist) {
1148 /* Free the entry because it isn't valid */
1149 drm_cleanup_buf_error(dev, entry);
1150 mutex_unlock(&dev->struct_mutex);
1151 atomic_dec(&dev->buf_alloc);
1154 dma->buflist = temp_buflist;
1156 for (i = 0; i < entry->buf_count; i++) {
1157 dma->buflist[i + dma->buf_count] = &entry->buflist[i];
1160 dma->buf_count += entry->buf_count;
1161 dma->seg_count += entry->seg_count;
1162 dma->page_count += byte_count >> PAGE_SHIFT;
1163 dma->byte_count += byte_count;
1165 DRM_DEBUG("dma->buf_count : %d\n", dma->buf_count);
1166 DRM_DEBUG("entry->buf_count : %d\n", entry->buf_count);
1168 mutex_unlock(&dev->struct_mutex);
1170 dma->flags = _DRM_DMA_USE_SG;
1171 atomic_dec(&dev->buf_alloc);
1175 static int radeon_gem_dma_bufs_init(struct drm_device *dev)
1177 struct drm_radeon_private *dev_priv = dev->dev_private;
1178 int size = RADEON_DMA_BUFFER_SIZE * RADEON_DMA_BUFFER_COUNT;
1181 ret = drm_dma_setup(dev);
1185 ret = drm_buffer_object_create(dev, size, drm_bo_type_device,
1186 DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_NO_EVICT |
1187 DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_MAPPABLE, 0,
1188 0, 0, &dev_priv->mm.dma_bufs.bo);
1190 DRM_ERROR("Failed to create DMA bufs\n");
1194 ret = drm_bo_kmap(dev_priv->mm.dma_bufs.bo, 0, size >> PAGE_SHIFT,
1195 &dev_priv->mm.dma_bufs.kmap);
1197 DRM_ERROR("Failed to mmap DMA buffers\n");
1201 radeon_gem_addbufs(dev);
1203 DRM_DEBUG("%x %d\n", dev_priv->mm.dma_bufs.bo->map_list.hash.key, size);
1204 dev->agp_buffer_token = dev_priv->mm.dma_bufs.bo->map_list.hash.key << PAGE_SHIFT;
1205 dev_priv->mm.fake_agp_map.handle = dev_priv->mm.dma_bufs.kmap.virtual;
1206 dev_priv->mm.fake_agp_map.size = size;
1208 dev->agp_buffer_map = &dev_priv->mm.fake_agp_map;
1209 dev_priv->gart_buffers_offset = dev_priv->mm.dma_bufs.bo->offset + dev_priv->gart_vm_start;
1213 static void radeon_gem_dma_bufs_destroy(struct drm_device *dev)
1216 struct drm_radeon_private *dev_priv = dev->dev_private;
1217 drm_dma_takedown(dev);
1219 drm_bo_kunmap(&dev_priv->mm.dma_bufs.kmap);
1220 drm_bo_usage_deref_unlocked(&dev_priv->mm.dma_bufs.bo);
1224 static struct drm_gem_object *gem_object_get(struct drm_device *dev, uint32_t name)
1226 struct drm_gem_object *obj;
1228 spin_lock(&dev->object_name_lock);
1229 obj = idr_find(&dev->object_name_idr, name);
1231 drm_gem_object_reference(obj);
1232 spin_unlock(&dev->object_name_lock);
1236 void radeon_gem_update_offsets(struct drm_device *dev, struct drm_master *master)
1238 drm_radeon_private_t *dev_priv = dev->dev_private;
1239 struct drm_radeon_master_private *master_priv = master->driver_priv;
1240 drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
1241 struct drm_gem_object *obj;
1242 struct drm_radeon_gem_object *obj_priv;
1244 /* update front_pitch_offset and back_pitch_offset */
1245 obj = gem_object_get(dev, sarea_priv->front_handle);
1247 obj_priv = obj->driver_private;
1249 dev_priv->front_offset = obj_priv->bo->offset;
1250 dev_priv->front_pitch_offset = (((sarea_priv->front_pitch / 64) << 22) |
1251 ((obj_priv->bo->offset
1252 + dev_priv->fb_location) >> 10));
1253 drm_gem_object_unreference(obj);
1256 obj = gem_object_get(dev, sarea_priv->back_handle);
1258 obj_priv = obj->driver_private;
1259 dev_priv->back_offset = obj_priv->bo->offset;
1260 dev_priv->back_pitch_offset = (((sarea_priv->back_pitch / 64) << 22) |
1261 ((obj_priv->bo->offset
1262 + dev_priv->fb_location) >> 10));
1263 drm_gem_object_unreference(obj);
1265 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;