linux-core/radeon_gem.c

   1 /*
   2  * Copyright 2008 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Dave Airlie
  23  */
  24 #include "drmP.h"
  25 #include "drm.h"
  26
  27 #include "radeon_drm.h"
  28 #include "radeon_drv.h"
  29
  30 static int radeon_gem_ib_init(struct drm_device *dev);
  31 static int radeon_gem_ib_destroy(struct drm_device *dev);
  32 static int radeon_gem_dma_bufs_init(struct drm_device *dev);
  33 static void radeon_gem_dma_bufs_destroy(struct drm_device *dev);
  34
  35 int radeon_gem_init_object(struct drm_gem_object *obj)
  36 {
  37         struct drm_radeon_gem_object *obj_priv;
  38
  39         obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
  40         if (!obj_priv) {
  41                 return -ENOMEM;
  42         }
  43
  44         obj->driver_private = obj_priv;
  45         obj_priv->obj = obj;
  46         return 0;
  47 }
  48
  49 void radeon_gem_free_object(struct drm_gem_object *obj)
  50 {
  51
  52         struct drm_radeon_gem_object *obj_priv = obj->driver_private;
  53
  54         /* tear down the buffer object - gem holds struct mutex */
  55         drm_bo_takedown_vm_locked(obj_priv->bo);
  56         drm_bo_usage_deref_locked(&obj_priv->bo);
  57         drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
  58 }
  59
  60 int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
  61                           struct drm_file *file_priv)
  62 {
  63         struct drm_radeon_private *dev_priv = dev->dev_private;
  64         struct drm_radeon_gem_info *args = data;
  65
  66         args->vram_start = dev_priv->mm.vram_offset;
  67         args->vram_size = dev_priv->mm.vram_size;
  68         args->vram_visible = dev_priv->mm.vram_visible;
  69
  70         args->gart_start = dev_priv->mm.gart_start;
  71         args->gart_size = dev_priv->mm.gart_useable;
  72
  73         return 0;
  74 }
  75
  76 struct drm_gem_object *radeon_gem_object_alloc(struct drm_device *dev, int size, int alignment,
  77                                                int initial_domain, bool discardable)
  78 {
  79         struct drm_gem_object *obj;
  80         struct drm_radeon_gem_object *obj_priv;
  81         int ret;
  82         uint32_t flags;
  83         uint32_t page_align;
  84
  85         obj = drm_gem_object_alloc(dev, size);
  86         if (!obj)
  87                 return NULL;
  88
  89         obj_priv = obj->driver_private;
  90         flags = DRM_BO_FLAG_MAPPABLE;
  91         if (initial_domain == RADEON_GEM_DOMAIN_VRAM)
  92                 flags |= DRM_BO_FLAG_MEM_VRAM;
  93         else if (initial_domain == RADEON_GEM_DOMAIN_GTT)
  94                 flags |= DRM_BO_FLAG_MEM_TT;
  95         else
  96                 flags |= DRM_BO_FLAG_MEM_LOCAL | DRM_BO_FLAG_CACHED;
  97
  98         flags |= DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_EXE;
  99
 100         if (discardable)
 101                 flags |= DRM_BO_FLAG_DISCARDABLE;
 102
 103         if (alignment == 0)
 104                 alignment = PAGE_SIZE;
 105
 106         page_align = alignment >> PAGE_SHIFT;
 107         /* create a TTM BO */
 108         ret = drm_buffer_object_create(dev,
 109                                        size, drm_bo_type_device,
 110                                        flags, 0, page_align,
 111                                        0, &obj_priv->bo);
 112         if (ret)
 113                 goto fail;
 114
 115         DRM_DEBUG("%p : size 0x%x, alignment %d, initial_domain %d\n", obj_priv->bo, size, alignment, initial_domain);
 116         return obj;
 117 fail:
 118
 119         return NULL;
 120 }
 121
 122 int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
 123                             struct drm_file *file_priv)
 124 {
 125         struct drm_radeon_gem_create *args = data;
 126         struct drm_radeon_gem_object *obj_priv;
 127         struct drm_gem_object *obj;
 128         int ret = 0;
 129         int handle;
 130
 131         /* create a gem object to contain this object in */
 132         args->size = roundup(args->size, PAGE_SIZE);
 133
 134         obj = radeon_gem_object_alloc(dev, args->size, args->alignment, args->initial_domain, args->no_backing_store);
 135         if (!obj)
 136                 return -EINVAL;
 137
 138         obj_priv = obj->driver_private;
 139         DRM_DEBUG("obj is %p bo is %p, %d\n", obj, obj_priv->bo, obj_priv->bo->num_pages);
 140         ret = drm_gem_handle_create(file_priv, obj, &handle);
 141         mutex_lock(&dev->struct_mutex);
 142         drm_gem_object_handle_unreference(obj);
 143         mutex_unlock(&dev->struct_mutex);
 144
 145         if (ret)
 146                 goto fail;
 147
 148         args->handle = handle;
 149
 150         return 0;
 151 fail:
 152         drm_gem_object_unreference(obj);
 153
 154         return ret;
 155 }
 156
 157 int radeon_gem_set_domain(struct drm_gem_object *obj, uint32_t read_domains, uint32_t write_domain, uint32_t *flags_p, bool unfenced)
 158 {
 159         struct drm_radeon_gem_object *obj_priv;
 160         uint32_t flags = 0;
 161         int ret;
 162
 163         obj_priv = obj->driver_private;
 164
 165         /* work out where to validate the buffer to */
 166         if (write_domain) { /* write domains always win */
 167                 if (write_domain == RADEON_GEM_DOMAIN_VRAM)
 168                         flags = DRM_BO_FLAG_MEM_VRAM;
 169                 else if (write_domain == RADEON_GEM_DOMAIN_GTT)
 170                         flags = DRM_BO_FLAG_MEM_TT; // need a can write gart check
 171                 else
 172                         return -EINVAL; // we can't write to system RAM
 173         } else {
 174                 /* okay for a read domain - prefer wherever the object is now or close enough */
 175                 if (read_domains == 0)
 176                         return -EINVAL;
 177
 178                 /* if its already a local memory and CPU is valid do nothing */
 179                 if (read_domains & RADEON_GEM_DOMAIN_CPU) {
 180                         if (obj_priv->bo->mem.mem_type == DRM_BO_MEM_LOCAL)
 181                                 return 0;
 182                         if (read_domains == RADEON_GEM_DOMAIN_CPU)
 183                                 return -EINVAL;
 184                 }
 185
 186                 /* simple case no choice in domains */
 187                 if (read_domains == RADEON_GEM_DOMAIN_VRAM)
 188                         flags = DRM_BO_FLAG_MEM_VRAM;
 189                 else if (read_domains == RADEON_GEM_DOMAIN_GTT)
 190                         flags = DRM_BO_FLAG_MEM_TT;
 191                 else if ((obj_priv->bo->mem.mem_type == DRM_BO_MEM_VRAM) && (read_domains & RADEON_GEM_DOMAIN_VRAM))
 192                         flags = DRM_BO_FLAG_MEM_VRAM;
 193                 else if ((obj_priv->bo->mem.mem_type == DRM_BO_MEM_TT) && (read_domains & RADEON_GEM_DOMAIN_GTT))
 194                         flags = DRM_BO_FLAG_MEM_TT;
 195                 else if ((obj_priv->bo->mem.mem_type == DRM_BO_MEM_LOCAL) && (read_domains & RADEON_GEM_DOMAIN_GTT))
 196                         flags = DRM_BO_FLAG_MEM_TT;
 197
 198                 /* no idea here just set whatever we are input */
 199                 if (flags == 0) {
 200                         if (read_domains & RADEON_GEM_DOMAIN_VRAM)
 201                                 flags |= DRM_BO_FLAG_MEM_VRAM;
 202                         if (read_domains & RADEON_GEM_DOMAIN_GTT)
 203                                 flags |= DRM_BO_FLAG_MEM_TT;
 204                 }
 205         }
 206
 207         /* if this BO is pinned then we ain't moving it anywhere */
 208         if (obj_priv->bo->pinned_mem_type && unfenced)
 209                 return 0;
 210
 211         DRM_DEBUG("validating %p from %d into %x %d %d\n", obj_priv->bo, obj_priv->bo->mem.mem_type, flags, read_domains, write_domain);
 212         ret = drm_bo_do_validate(obj_priv->bo, flags, DRM_BO_MASK_MEM | DRM_BO_FLAG_CACHED,
 213                                  unfenced ? DRM_BO_HINT_DONT_FENCE : 0, 0);
 214         if (ret)
 215                 return ret;
 216
 217         if (flags_p)
 218                 *flags_p = flags;
 219         return 0;
 220
 221 }
 222
 223 int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 224                                 struct drm_file *file_priv)
 225 {
 226         /* transition the BO to a domain - just validate the BO into a certain domain */
 227         struct drm_radeon_gem_set_domain *args = data;
 228         struct drm_gem_object *obj;
 229         struct drm_radeon_gem_object *obj_priv;
 230         int ret;
 231
 232         /* for now if someone requests domain CPU - just make sure the buffer is finished with */
 233
 234         /* just do a BO wait for now */
 235         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 236         if (obj == NULL)
 237                 return -EINVAL;
 238
 239         obj_priv = obj->driver_private;
 240
 241         ret = radeon_gem_set_domain(obj, args->read_domains, args->write_domain, NULL, true);
 242
 243         mutex_lock(&dev->struct_mutex);
 244         drm_gem_object_unreference(obj);
 245         mutex_unlock(&dev->struct_mutex);
 246         return ret;
 247 }
 248
 249 int radeon_gem_pread_ioctl(struct drm_device *dev, void *data,
 250                           struct drm_file *file_priv)
 251 {
 252         return -ENOSYS;
 253 }
 254
 255 int radeon_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 256                           struct drm_file *file_priv)
 257 {
 258         struct drm_radeon_gem_pwrite *args = data;
 259         struct drm_gem_object *obj;
 260         struct drm_radeon_gem_object *obj_priv;
 261         int ret;
 262
 263         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 264         if (obj == NULL)
 265                 return -EINVAL;
 266
 267         obj_priv = obj->driver_private;
 268
 269         /* check where the buffer is first - if not in VRAM
 270            fallback to userspace copying for now */
 271         mutex_lock(&obj_priv->bo->mutex);
 272         if (obj_priv->bo->mem.mem_type != DRM_BO_MEM_VRAM) {
 273                 ret = -EINVAL;
 274                 goto out_unlock;
 275         }
 276
 277         DRM_ERROR("pwriting data->size %lld %llx\n", args->size, args->offset);
 278         ret = -EINVAL;
 279
 280 #if 0
 281         /* so need to grab an IB, copy the data into it in a loop
 282            and send them to VRAM using HDB */
 283         while ((buf = radeon_host_data_blit(dev, cpp, w, dst_pitch_off, &buf_pitch,
 284                                             x, &y, (unsigned int*)&h, &hpass)) != 0) {
 285                 radeon_host_data_blit_copy_pass(dev, cpp, buf, (uint8_t *)src,
 286                                                 hpass, buf_pitch, src_pitch);
 287                 src += hpass * src_pitch;
 288         }
 289 #endif
 290 out_unlock:
 291         mutex_unlock(&obj_priv->bo->mutex);
 292         mutex_lock(&dev->struct_mutex);
 293         drm_gem_object_unreference(obj);
 294         mutex_unlock(&dev->struct_mutex);
 295         return ret;
 296 }
 297
 298 int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
 299                           struct drm_file *file_priv)
 300 {
 301         struct drm_radeon_gem_mmap *args = data;
 302         struct drm_gem_object *obj;
 303         struct drm_radeon_gem_object *obj_priv;
 304         loff_t offset;
 305         unsigned long addr;
 306
 307         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 308         if (obj == NULL)
 309                 return -EINVAL;
 310
 311         offset = args->offset;
 312
 313         DRM_DEBUG("got here %p\n", obj);
 314         obj_priv = obj->driver_private;
 315
 316         DRM_DEBUG("got here %p %p %lld %ld\n", obj, obj_priv->bo, args->size, obj_priv->bo->num_pages);
 317         if (!obj_priv->bo) {
 318                 mutex_lock(&dev->struct_mutex);
 319                 drm_gem_object_unreference(obj);
 320                 mutex_unlock(&dev->struct_mutex);
 321                 return -EINVAL;
 322         }
 323
 324         down_write(&current->mm->mmap_sem);
 325         addr = do_mmap_pgoff(file_priv->filp, 0, args->size,
 326                              PROT_READ | PROT_WRITE, MAP_SHARED,
 327                              obj_priv->bo->map_list.hash.key);
 328         up_write(&current->mm->mmap_sem);
 329
 330         DRM_DEBUG("got here %p %d\n", obj, obj_priv->bo->mem.mem_type);
 331         mutex_lock(&dev->struct_mutex);
 332         drm_gem_object_unreference(obj);
 333         mutex_unlock(&dev->struct_mutex);
 334         if (IS_ERR((void *)addr))
 335                 return addr;
 336
 337         args->addr_ptr = (uint64_t) addr;
 338
 339         return 0;
 340
 341 }
 342
 343 int radeon_gem_pin_ioctl(struct drm_device *dev, void *data,
 344                          struct drm_file *file_priv)
 345 {
 346         struct drm_radeon_gem_pin *args = data;
 347         struct drm_gem_object *obj;
 348         struct drm_radeon_gem_object *obj_priv;
 349         int ret;
 350         int flags = DRM_BO_FLAG_NO_EVICT;
 351         int mask = DRM_BO_FLAG_NO_EVICT;
 352
 353         /* check for valid args */
 354         if (args->pin_domain) {
 355                 mask |= DRM_BO_MASK_MEM;
 356                 if (args->pin_domain == RADEON_GEM_DOMAIN_GTT)
 357                         flags |= DRM_BO_FLAG_MEM_TT;
 358                 else if (args->pin_domain == RADEON_GEM_DOMAIN_VRAM)
 359                         flags |= DRM_BO_FLAG_MEM_VRAM;
 360                 else /* hand back the offset we currently have if no args supplied
 361                       - this is to allow old mesa to work - its a hack */
 362                         flags = 0;
 363         }
 364
 365         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 366         if (obj == NULL)
 367                 return -EINVAL;
 368
 369         obj_priv = obj->driver_private;
 370
 371         /* validate into a pin with no fence */
 372         DRM_DEBUG("got here %p %p %d\n", obj, obj_priv->bo, atomic_read(&obj_priv->bo->usage));
 373         if (flags && !(obj_priv->bo->type != drm_bo_type_kernel && !DRM_SUSER(DRM_CURPROC))) {
 374                 ret = drm_bo_do_validate(obj_priv->bo, flags, mask,
 375                                          DRM_BO_HINT_DONT_FENCE, 0);
 376         } else
 377                 ret = 0;
 378
 379         args->offset = obj_priv->bo->offset;
 380         DRM_DEBUG("got here %p %p %x\n", obj, obj_priv->bo, obj_priv->bo->offset);
 381
 382         mutex_lock(&dev->struct_mutex);
 383         drm_gem_object_unreference(obj);
 384         mutex_unlock(&dev->struct_mutex);
 385         return ret;
 386 }
 387
 388 int radeon_gem_unpin_ioctl(struct drm_device *dev, void *data,
 389                            struct drm_file *file_priv)
 390 {
 391         struct drm_radeon_gem_unpin *args = data;
 392         struct drm_gem_object *obj;
 393         struct drm_radeon_gem_object *obj_priv;
 394         int ret;
 395
 396         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 397         if (obj == NULL)
 398                 return -EINVAL;
 399
 400         obj_priv = obj->driver_private;
 401
 402         /* validate into a pin with no fence */
 403
 404         ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
 405                                  DRM_BO_HINT_DONT_FENCE, 0);
 406
 407         mutex_lock(&dev->struct_mutex);
 408         drm_gem_object_unreference(obj);
 409         mutex_unlock(&dev->struct_mutex);
 410         return ret;
 411 }
 412
 413 int radeon_gem_busy(struct drm_device *dev, void *data,
 414                      struct drm_file *file_priv)
 415 {
 416         return 0;
 417 }
 418
 419 int radeon_gem_wait_rendering(struct drm_device *dev, void *data,
 420                               struct drm_file *file_priv)
 421 {
 422         struct drm_radeon_gem_wait_rendering *args = data;
 423         struct drm_gem_object *obj;
 424         struct drm_radeon_gem_object *obj_priv;
 425         int ret;
 426
 427
 428         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 429         if (obj == NULL)
 430                 return -EINVAL;
 431
 432         obj_priv = obj->driver_private;
 433
 434         mutex_lock(&obj_priv->bo->mutex);
 435         ret = drm_bo_wait(obj_priv->bo, 0, 1, 1, 0);
 436         mutex_unlock(&obj_priv->bo->mutex);
 437
 438         mutex_lock(&dev->struct_mutex);
 439         drm_gem_object_unreference(obj);
 440         mutex_unlock(&dev->struct_mutex);
 441         return ret;
 442 }
 443
 444
 445
 446 /*
 447  * Depending on card genertation, chipset bugs, etc... the amount of vram
 448  * accessible to the CPU can vary. This function is our best shot at figuring
 449  * it out. Returns a value in KB.
 450  */
 451 static uint32_t radeon_get_accessible_vram(struct drm_device *dev)
 452 {
 453         drm_radeon_private_t *dev_priv = dev->dev_private;
 454         uint32_t aper_size;
 455         u8 byte;
 456
 457         if (dev_priv->chip_family >= CHIP_R600)
 458                 aper_size = RADEON_READ(R600_CONFIG_APER_SIZE) / 1024;
 459         else
 460                 aper_size = RADEON_READ(RADEON_CONFIG_APER_SIZE) / 1024;
 461
 462         /* Set HDP_APER_CNTL only on cards that are known not to be broken,
 463          * that is has the 2nd generation multifunction PCI interface
 464          */
 465         if (dev_priv->chip_family == CHIP_RV280 ||
 466             dev_priv->chip_family == CHIP_RV350 ||
 467             dev_priv->chip_family == CHIP_RV380 ||
 468             dev_priv->chip_family == CHIP_R420 ||
 469             dev_priv->chip_family == CHIP_R423 ||
 470             dev_priv->chip_family == CHIP_RV410 ||
 471             radeon_is_avivo(dev_priv)) {
 472                 uint32_t temp = RADEON_READ(RADEON_HOST_PATH_CNTL);
 473                 temp |= RADEON_HDP_APER_CNTL;
 474                 RADEON_WRITE(RADEON_HOST_PATH_CNTL, temp);
 475                 return aper_size * 2;
 476         }
 477
 478         /* Older cards have all sorts of funny issues to deal with. First
 479          * check if it's a multifunction card by reading the PCI config
 480          * header type... Limit those to one aperture size
 481          */
 482         pci_read_config_byte(dev->pdev, 0xe, &byte);
 483         if (byte & 0x80)
 484                 return aper_size;
 485
 486         /* Single function older card. We read HDP_APER_CNTL to see how the BIOS
 487          * have set it up. We don't write this as it's broken on some ASICs but
 488          * we expect the BIOS to have done the right thing (might be too optimistic...)
 489          */
 490         if (RADEON_READ(RADEON_HOST_PATH_CNTL) & RADEON_HDP_APER_CNTL)
 491                 return aper_size * 2;
 492
 493         return aper_size;
 494 }
 495
 496 /* code from the DDX - do memory sizing */
 497 void radeon_vram_setup(struct drm_device *dev)
 498 {
 499         drm_radeon_private_t *dev_priv = dev->dev_private;
 500         uint32_t vram;
 501         uint32_t accessible,  bar_size;
 502
 503         if (!radeon_is_avivo(dev_priv) && (dev_priv->flags & RADEON_IS_IGP)) {
 504                 uint32_t tom = RADEON_READ(RADEON_NB_TOM);
 505
 506                 vram = (((tom >> 16) - (tom & 0xffff) + 1) << 6);
 507                 RADEON_WRITE(RADEON_CONFIG_MEMSIZE, vram * 1024);
 508         } else {
 509                 if (dev_priv->chip_family >= CHIP_R600)
 510                         vram = RADEON_READ(R600_CONFIG_MEMSIZE) / 1024;
 511                 else {
 512                         vram = RADEON_READ(RADEON_CONFIG_MEMSIZE) / 1024;
 513
 514                         /* Some production boards of m6 will return 0 if it's 8 MB */
 515                         if (vram == 0) {
 516                                 vram = 8192;
 517                                 RADEON_WRITE(RADEON_CONFIG_MEMSIZE, 0x800000);
 518                         }
 519                 }
 520         }
 521
 522         accessible = radeon_get_accessible_vram(dev);
 523
 524         bar_size = drm_get_resource_len(dev, 0) / 1024;
 525         if (bar_size == 0)
 526                 bar_size = 0x20000;
 527         if (accessible > bar_size)
 528                 accessible = bar_size;
 529
 530         if (accessible > vram)
 531                 accessible = vram;
 532
 533         DRM_INFO("Detected VRAM RAM=%dK, accessible=%uK, BAR=%uK\n",
 534                  vram, accessible, bar_size);
 535
 536         dev_priv->mm.vram_offset = dev_priv->fb_aper_offset;
 537         dev_priv->mm.vram_size = vram * 1024;
 538         dev_priv->mm.vram_visible = accessible * 1024;
 539
 540
 541 }
 542
 543 static int radeon_gart_init(struct drm_device *dev)
 544 {
 545         drm_radeon_private_t *dev_priv = dev->dev_private;
 546         int ret;
 547         u32 base = 0;
 548
 549         /* setup a 32MB GART */
 550         dev_priv->gart_size = dev_priv->mm.gart_size;
 551
 552         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
 553
 554 #if __OS_HAS_AGP
 555         /* setup VRAM vs GART here */
 556         if (dev_priv->flags & RADEON_IS_AGP) {
 557                 base = dev->agp->base;
 558                 if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
 559                     base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
 560                         DRM_INFO("Can't use agp base @0x%08lx, won't fit\n",
 561                                  dev->agp->base);
 562                         base = 0;
 563                 }
 564         }
 565 #endif
 566
 567         if (base == 0) {
 568                 base = dev_priv->fb_location + dev_priv->fb_size;
 569                 if (base < dev_priv->fb_location ||
 570                     ((base + dev_priv->gart_size) & 0xfffffffful) < base)
 571                         base = dev_priv->fb_location
 572                                 - dev_priv->gart_size;
 573         }
 574         /* start on the card */
 575         dev_priv->gart_vm_start = base & 0xffc00000u;
 576         if (dev_priv->gart_vm_start != base)
 577                 DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
 578                          base, dev_priv->gart_vm_start);
 579
 580         /* if on PCIE we need to allocate an fb object for the PCIE GART table */
 581         if (dev_priv->flags & RADEON_IS_PCIE) {
 582                 ret = drm_buffer_object_create(dev, RADEON_PCIGART_TABLE_SIZE,
 583                                                drm_bo_type_kernel,
 584                                                DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_VRAM | DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
 585                                                0, 1, 0, &dev_priv->mm.pcie_table.bo);
 586                 if (ret)
 587                         return -EINVAL;
 588
 589                 /* subtract from VRAM value reporting to userspace */
 590                 dev_priv->mm.vram_visible -= RADEON_PCIGART_TABLE_SIZE;
 591
 592                 dev_priv->mm.pcie_table_backup = kzalloc(RADEON_PCIGART_TABLE_SIZE, GFP_KERNEL);
 593                 if (!dev_priv->mm.pcie_table_backup)
 594                         return -EINVAL;
 595
 596                 ret = drm_bo_kmap(dev_priv->mm.pcie_table.bo, 0, RADEON_PCIGART_TABLE_SIZE >> PAGE_SHIFT,
 597                                   &dev_priv->mm.pcie_table.kmap);
 598                 if (ret)
 599                         return -EINVAL;
 600
 601                 dev_priv->pcigart_offset_set = 2;
 602                 dev_priv->gart_info.bus_addr =  dev_priv->fb_location + dev_priv->mm.pcie_table.bo->offset;
 603                 dev_priv->gart_info.addr = dev_priv->mm.pcie_table.kmap.virtual;
 604                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
 605                 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_FB;
 606                 memset(dev_priv->gart_info.addr, 0, RADEON_PCIGART_TABLE_SIZE);
 607         } else if (!(dev_priv->flags & RADEON_IS_AGP)) {
 608                 /* allocate PCI GART table */
 609                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
 610                 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
 611                 if (dev_priv->flags & RADEON_IS_IGPGART)
 612                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
 613                 else
 614                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
 615
 616                 ret = drm_ati_alloc_pcigart_table(dev, &dev_priv->gart_info);
 617                 if (ret) {
 618                         DRM_ERROR("cannot allocate PCI GART page!\n");
 619                         return -EINVAL;
 620                 }
 621
 622                 dev_priv->gart_info.addr = dev_priv->gart_info.table_handle->vaddr;
 623                 dev_priv->gart_info.bus_addr = dev_priv->gart_info.table_handle->busaddr;
 624         }
 625
 626         /* gart values setup - start the GART */
 627         if (dev_priv->flags & RADEON_IS_AGP) {
 628                 radeon_set_pcigart(dev_priv, 0);
 629         } else {
 630                 radeon_set_pcigart(dev_priv, 1);
 631         }
 632
 633         return 0;
 634 }
 635
 636 int radeon_alloc_gart_objects(struct drm_device *dev)
 637 {
 638         drm_radeon_private_t *dev_priv = dev->dev_private;
 639         int ret;
 640
 641         ret = drm_buffer_object_create(dev, RADEON_DEFAULT_RING_SIZE,
 642                                        drm_bo_type_kernel,
 643                                        DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
 644                                        DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
 645                                        0, 1, 0, &dev_priv->mm.ring.bo);
 646         if (ret) {
 647                 if (dev_priv->flags & RADEON_IS_AGP)
 648                         DRM_ERROR("failed to allocate ring - most likely an AGP driver bug\n");
 649                 else
 650                         DRM_ERROR("failed to allocate ring\n");
 651                 return -EINVAL;
 652         }
 653
 654         ret = drm_bo_kmap(dev_priv->mm.ring.bo, 0, RADEON_DEFAULT_RING_SIZE >> PAGE_SHIFT,
 655                           &dev_priv->mm.ring.kmap);
 656         if (ret) {
 657                 DRM_ERROR("failed to map ring\n");
 658                 return -EINVAL;
 659         }
 660
 661         ret = drm_buffer_object_create(dev, PAGE_SIZE,
 662                                        drm_bo_type_kernel,
 663                                        DRM_BO_FLAG_WRITE |DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
 664                                        DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
 665                                        0, 1, 0, &dev_priv->mm.ring_read.bo);
 666         if (ret) {
 667                 DRM_ERROR("failed to allocate ring read\n");
 668                 return -EINVAL;
 669         }
 670
 671         ret = drm_bo_kmap(dev_priv->mm.ring_read.bo, 0,
 672                           PAGE_SIZE >> PAGE_SHIFT,
 673                           &dev_priv->mm.ring_read.kmap);
 674         if (ret) {
 675                 DRM_ERROR("failed to map ring read\n");
 676                 return -EINVAL;
 677         }
 678
 679         DRM_DEBUG("Ring ptr %p mapped at %ld %p, read ptr %p maped at %ld %p\n",
 680                   dev_priv->mm.ring.bo, dev_priv->mm.ring.bo->offset, dev_priv->mm.ring.kmap.virtual,
 681                   dev_priv->mm.ring_read.bo, dev_priv->mm.ring_read.bo->offset, dev_priv->mm.ring_read.kmap.virtual);
 682
 683         dev_priv->mm.gart_useable -= RADEON_DEFAULT_RING_SIZE + PAGE_SIZE;
 684
 685         /* init the indirect buffers */
 686         radeon_gem_ib_init(dev);
 687         radeon_gem_dma_bufs_init(dev);
 688         return 0;
 689
 690 }
 691
 692 static bool avivo_get_mc_idle(struct drm_device *dev)
 693 {
 694         drm_radeon_private_t *dev_priv = dev->dev_private;
 695
 696         if (dev_priv->chip_family >= CHIP_R600) {
 697                 /* no idea where this is on r600 yet */
 698                 return true;
 699         } else if (dev_priv->chip_family == CHIP_RV515) {
 700                 if (radeon_read_mc_reg(dev_priv, RV515_MC_STATUS) & RV515_MC_STATUS_IDLE)
 701                         return true;
 702                 else
 703                         return false;
 704         } else if (dev_priv->chip_family == CHIP_RS600) {
 705                 if (radeon_read_mc_reg(dev_priv, RS600_MC_STATUS) & RS600_MC_STATUS_IDLE)
 706                         return true;
 707                 else
 708                         return false;
 709         } else if ((dev_priv->chip_family == CHIP_RS690) ||
 710                    (dev_priv->chip_family == CHIP_RS740)) {
 711                 if (radeon_read_mc_reg(dev_priv, RS690_MC_STATUS) & RS690_MC_STATUS_IDLE)
 712                         return true;
 713                 else
 714                         return false;
 715         } else {
 716                 if (radeon_read_mc_reg(dev_priv, R520_MC_STATUS) & R520_MC_STATUS_IDLE)
 717                         return true;
 718                 else
 719                         return false;
 720         }
 721 }
 722
 723
 724 static void avivo_disable_mc_clients(struct drm_device *dev)
 725 {
 726         drm_radeon_private_t *dev_priv = dev->dev_private;
 727         uint32_t tmp;
 728         int timeout;
 729
 730         radeon_do_wait_for_idle(dev_priv);
 731
 732         RADEON_WRITE(AVIVO_D1VGA_CONTROL, RADEON_READ(AVIVO_D1VGA_CONTROL) & ~AVIVO_DVGA_CONTROL_MODE_ENABLE);
 733         RADEON_WRITE(AVIVO_D2VGA_CONTROL, RADEON_READ(AVIVO_D2VGA_CONTROL) & ~AVIVO_DVGA_CONTROL_MODE_ENABLE);
 734
 735         tmp = RADEON_READ(AVIVO_D1CRTC_CONTROL);
 736         RADEON_WRITE(AVIVO_D1CRTC_CONTROL, tmp & ~AVIVO_CRTC_EN);
 737
 738         tmp = RADEON_READ(AVIVO_D2CRTC_CONTROL);
 739         RADEON_WRITE(AVIVO_D2CRTC_CONTROL, tmp & ~AVIVO_CRTC_EN);
 740
 741         tmp = RADEON_READ(AVIVO_D2CRTC_CONTROL);
 742
 743         udelay(1000);
 744
 745         timeout = 0;
 746         while (!(avivo_get_mc_idle(dev))) {
 747                 if (++timeout > 100000) {
 748                         DRM_ERROR("Timeout waiting for memory controller to update settings\n");
 749                         DRM_ERROR("Bad things may or may not happen\n");
 750                 }
 751                 udelay(10);
 752         }
 753 }
 754
 755 static inline u32 radeon_busy_wait(struct drm_device *dev, uint32_t reg, uint32_t bits,
 756                                   unsigned int timeout)
 757 {
 758         drm_radeon_private_t *dev_priv = dev->dev_private;
 759         u32 status;
 760
 761         do {
 762                 udelay(10);
 763                 status = RADEON_READ(reg);
 764                 timeout--;
 765         } while(status != 0xffffffff && (status & bits) && (timeout > 0));
 766
 767         if (timeout == 0)
 768                 status = 0xffffffff;
 769
 770         return status;
 771 }
 772
 773 /* Wait for vertical sync on primary CRTC */
 774 static void radeon_wait_for_vsync(struct drm_device *dev)
 775 {
 776         drm_radeon_private_t *dev_priv = dev->dev_private;
 777         uint32_t       crtc_gen_cntl;
 778
 779         crtc_gen_cntl = RADEON_READ(RADEON_CRTC_GEN_CNTL);
 780         if ((crtc_gen_cntl & RADEON_CRTC_DISP_REQ_EN_B) ||
 781             !(crtc_gen_cntl & RADEON_CRTC_EN))
 782                 return;
 783
 784         /* Clear the CRTC_VBLANK_SAVE bit */
 785         RADEON_WRITE(RADEON_CRTC_STATUS, RADEON_CRTC_VBLANK_SAVE_CLEAR);
 786
 787         radeon_busy_wait(dev, RADEON_CRTC_STATUS, RADEON_CRTC_VBLANK_SAVE, 2000);
 788
 789 }
 790
 791 /* Wait for vertical sync on primary CRTC */
 792 static void radeon_wait_for_vsync2(struct drm_device *dev)
 793 {
 794         drm_radeon_private_t *dev_priv = dev->dev_private;
 795         uint32_t       crtc2_gen_cntl;
 796
 797         crtc2_gen_cntl = RADEON_READ(RADEON_CRTC2_GEN_CNTL);
 798         if ((crtc2_gen_cntl & RADEON_CRTC2_DISP_REQ_EN_B) ||
 799             !(crtc2_gen_cntl & RADEON_CRTC2_EN))
 800                 return;
 801
 802         /* Clear the CRTC_VBLANK_SAVE bit */
 803         RADEON_WRITE(RADEON_CRTC2_STATUS, RADEON_CRTC2_VBLANK_SAVE_CLEAR);
 804
 805         radeon_busy_wait(dev, RADEON_CRTC2_STATUS, RADEON_CRTC2_VBLANK_SAVE, 2000);
 806 }
 807
 808 static void legacy_disable_mc_clients(struct drm_device *dev)
 809 {
 810         drm_radeon_private_t *dev_priv = dev->dev_private;
 811         uint32_t old_mc_status, status_idle;
 812         uint32_t ov0_scale_cntl, crtc_ext_cntl, crtc_gen_cntl, crtc2_gen_cntl;
 813         uint32_t status;
 814
 815         radeon_do_wait_for_idle(dev_priv);
 816
 817         if (dev_priv->flags & RADEON_IS_IGP)
 818                 return;
 819
 820         old_mc_status = RADEON_READ(RADEON_MC_STATUS);
 821
 822         /* stop display and memory access */
 823         ov0_scale_cntl = RADEON_READ(RADEON_OV0_SCALE_CNTL);
 824         RADEON_WRITE(RADEON_OV0_SCALE_CNTL, ov0_scale_cntl & ~RADEON_SCALER_ENABLE);
 825         crtc_ext_cntl = RADEON_READ(RADEON_CRTC_EXT_CNTL);
 826         RADEON_WRITE(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl | RADEON_CRTC_DISPLAY_DIS);
 827         crtc_gen_cntl = RADEON_READ(RADEON_CRTC_GEN_CNTL);
 828
 829         radeon_wait_for_vsync(dev);
 830
 831         RADEON_WRITE(RADEON_CRTC_GEN_CNTL,
 832                      (crtc_gen_cntl & ~(RADEON_CRTC_CUR_EN | RADEON_CRTC_ICON_EN)) |
 833                      RADEON_CRTC_DISP_REQ_EN_B | RADEON_CRTC_EXT_DISP_EN);
 834
 835         if (!(dev_priv->flags & RADEON_SINGLE_CRTC)) {
 836                 crtc2_gen_cntl = RADEON_READ(RADEON_CRTC2_GEN_CNTL);
 837
 838                 radeon_wait_for_vsync2(dev);
 839                 RADEON_WRITE(RADEON_CRTC2_GEN_CNTL,
 840                              (crtc2_gen_cntl &
 841                               ~(RADEON_CRTC2_CUR_EN | RADEON_CRTC2_ICON_EN)) |
 842                              RADEON_CRTC2_DISP_REQ_EN_B);
 843         }
 844
 845         udelay(500);
 846
 847         if (radeon_is_r300(dev_priv))
 848                 status_idle = R300_MC_IDLE;
 849         else
 850                 status_idle = RADEON_MC_IDLE;
 851
 852         status = radeon_busy_wait(dev, RADEON_MC_STATUS, status_idle, 200000);
 853         if (status == 0xffffffff) {
 854                 DRM_ERROR("Timeout waiting for memory controller to update settings\n");
 855                 DRM_ERROR("Bad things may or may not happen\n");
 856         }
 857 }
 858
 859
 860 void radeon_init_memory_map(struct drm_device *dev)
 861 {
 862         drm_radeon_private_t *dev_priv = dev->dev_private;
 863         u32 mem_size, aper_size;
 864
 865         dev_priv->mc_fb_location = radeon_read_fb_location(dev_priv);
 866         radeon_read_agp_location(dev_priv, &dev_priv->mc_agp_loc_lo, &dev_priv->mc_agp_loc_hi);
 867
 868         if (dev_priv->chip_family >= CHIP_R600) {
 869                 mem_size = RADEON_READ(R600_CONFIG_MEMSIZE);
 870                 aper_size = RADEON_READ(R600_CONFIG_APER_SIZE);
 871         } else {
 872                 mem_size = RADEON_READ(RADEON_CONFIG_MEMSIZE);
 873                 aper_size = RADEON_READ(RADEON_CONFIG_APER_SIZE);
 874         }
 875
 876         /* M6s report illegal memory size */
 877         if (mem_size == 0)
 878                 mem_size = 8 * 1024 * 1024;
 879
 880         /* for RN50/M6/M7 - Novell bug 204882 */
 881         if (aper_size > mem_size)
 882                 mem_size = aper_size;
 883
 884         if ((dev_priv->chip_family != CHIP_RS600) &&
 885             (dev_priv->chip_family != CHIP_RS690) &&
 886             (dev_priv->chip_family != CHIP_RS740)) {
 887                 if (dev_priv->flags & RADEON_IS_IGP)
 888                         dev_priv->mc_fb_location = RADEON_READ(RADEON_NB_TOM);
 889                 else {
 890                         uint32_t aper0_base;
 891
 892                         if (dev_priv->chip_family >= CHIP_R600)
 893                                 aper0_base = RADEON_READ(R600_CONFIG_F0_BASE);
 894                         else
 895                                 aper0_base = RADEON_READ(RADEON_CONFIG_APER_0_BASE);
 896
 897
 898                         /* Some chips have an "issue" with the memory controller, the
 899                          * location must be aligned to the size. We just align it down,
 900                          * too bad if we walk over the top of system memory, we don't
 901                          * use DMA without a remapped anyway.
 902                          * Affected chips are rv280, all r3xx, and all r4xx, but not IGP
 903                          */
 904                         if (dev_priv->chip_family == CHIP_RV280 ||
 905                             dev_priv->chip_family == CHIP_R300 ||
 906                             dev_priv->chip_family == CHIP_R350 ||
 907                             dev_priv->chip_family == CHIP_RV350 ||
 908                             dev_priv->chip_family == CHIP_RV380 ||
 909                             dev_priv->chip_family == CHIP_R420 ||
 910                             dev_priv->chip_family == CHIP_R423 ||
 911                             dev_priv->chip_family == CHIP_RV410)
 912                                 aper0_base &= ~(mem_size - 1);
 913
 914                         if (dev_priv->chip_family >= CHIP_R600) {
 915                                 dev_priv->mc_fb_location = (aper0_base >> 24) |
 916                                         (((aper0_base + mem_size - 1) & 0xff000000U) >> 8);
 917                         } else {
 918                                 dev_priv->mc_fb_location = (aper0_base >> 16) |
 919                                         ((aper0_base + mem_size - 1) & 0xffff0000U);
 920                         }
 921                 }
 922         }
 923
 924         if (dev_priv->chip_family >= CHIP_R600)
 925                 dev_priv->fb_location = (dev_priv->mc_fb_location & 0xffff) << 24;
 926         else
 927                 dev_priv->fb_location = (dev_priv->mc_fb_location & 0xffff) << 16;
 928
 929         /* updating mc regs here */
 930         if (radeon_is_avivo(dev_priv))
 931                 avivo_disable_mc_clients(dev);
 932         else
 933                 legacy_disable_mc_clients(dev);
 934
 935         radeon_write_fb_location(dev_priv, dev_priv->mc_fb_location);
 936
 937         if (radeon_is_avivo(dev_priv)) {
 938                 if (dev_priv->chip_family >= CHIP_R600)
 939                         RADEON_WRITE(R600_HDP_NONSURFACE_BASE, (dev_priv->mc_fb_location << 16) & 0xff0000);
 940                 else
 941                         RADEON_WRITE(AVIVO_HDP_FB_LOCATION, dev_priv->mc_fb_location);
 942         }
 943
 944         if (dev_priv->chip_family >= CHIP_R600) {
 945                 dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffffff) << 24;
 946                 dev_priv->fb_size = ((radeon_read_fb_location(dev_priv) & 0xff000000u) + 0x1000000)
 947                         - dev_priv->fb_location;
 948         } else {
 949                 dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
 950                 dev_priv->fb_size =
 951                         ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
 952                         - dev_priv->fb_location;
 953         }
 954
 955         /* add an MTRR for the VRAM */
 956         dev_priv->aper_size = aper_size;
 957         dev_priv->vram_mtrr = mtrr_add(dev_priv->fb_aper_offset, dev_priv->aper_size, MTRR_TYPE_WRCOMB, 1);
 958
 959 }
 960
 961 /* init memory manager - start with all of VRAM and a 32MB GART aperture for now */
 962 int radeon_gem_mm_init(struct drm_device *dev)
 963 {
 964         drm_radeon_private_t *dev_priv = dev->dev_private;
 965         int ret;
 966         u32 pg_offset;
 967
 968         /* init TTM underneath */
 969         drm_bo_driver_init(dev);
 970
 971         /* use the uncached allocator */
 972         dev->bm.allocator_type = _DRM_BM_ALLOCATOR_UNCACHED;
 973
 974         /* size the mappable VRAM memory for now */
 975         radeon_vram_setup(dev);
 976
 977         radeon_init_memory_map(dev);
 978
 979 #define VRAM_RESERVE_TEXT (256*1024) /* need to reserve 256 for text mode for now */
 980         dev_priv->mm.vram_visible -= VRAM_RESERVE_TEXT;
 981         pg_offset = VRAM_RESERVE_TEXT >> PAGE_SHIFT;
 982         drm_bo_init_mm(dev, DRM_BO_MEM_VRAM, pg_offset, /*dev_priv->mm.vram_offset >> PAGE_SHIFT,*/
 983                        ((dev_priv->mm.vram_visible) >> PAGE_SHIFT) - 16,
 984                        0);
 985
 986
 987         if (dev_priv->chip_family > CHIP_R600) {
 988                 dev_priv->mm_enabled = true;
 989                 return 0;
 990         }
 991
 992         dev_priv->mm.gart_size = (32 * 1024 * 1024);
 993         dev_priv->mm.gart_start = 0;
 994         dev_priv->mm.gart_useable = dev_priv->mm.gart_size;
 995         ret = radeon_gart_init(dev);
 996         if (ret)
 997                 return -EINVAL;
 998
 999         drm_bo_init_mm(dev, DRM_BO_MEM_TT, 0,
1000                        dev_priv->mm.gart_size >> PAGE_SHIFT,
1001                        0);
1002
1003         /* need to allocate some objects in the GART */
1004         /* ring + ring read ptr */
1005         ret = radeon_alloc_gart_objects(dev);
1006         if (ret) {
1007                 radeon_gem_mm_fini(dev);
1008                 return -EINVAL;
1009         }
1010
1011         dev_priv->mm_enabled = true;
1012         return 0;
1013 }
1014
1015 void radeon_gem_mm_fini(struct drm_device *dev)
1016 {
1017         drm_radeon_private_t *dev_priv = dev->dev_private;
1018
1019         radeon_gem_dma_bufs_destroy(dev);
1020         radeon_gem_ib_destroy(dev);
1021
1022         mutex_lock(&dev->struct_mutex);
1023
1024         if (dev_priv->mm.ring_read.bo) {
1025                 drm_bo_kunmap(&dev_priv->mm.ring_read.kmap);
1026                 drm_bo_usage_deref_locked(&dev_priv->mm.ring_read.bo);
1027         }
1028
1029         if (dev_priv->mm.ring.bo) {
1030                 drm_bo_kunmap(&dev_priv->mm.ring.kmap);
1031                 drm_bo_usage_deref_locked(&dev_priv->mm.ring.bo);
1032         }
1033
1034         if (drm_bo_clean_mm(dev, DRM_BO_MEM_TT, 1)) {
1035                 DRM_DEBUG("delaying takedown of TTM memory\n");
1036         }
1037
1038         if (dev_priv->flags & RADEON_IS_PCIE) {
1039                 if (dev_priv->mm.pcie_table_backup) {
1040                         kfree(dev_priv->mm.pcie_table_backup);
1041                         dev_priv->mm.pcie_table_backup = NULL;
1042                 }
1043                 if (dev_priv->mm.pcie_table.bo) {
1044                         drm_bo_kunmap(&dev_priv->mm.pcie_table.kmap);
1045                         drm_bo_usage_deref_locked(&dev_priv->mm.pcie_table.bo);
1046                 }
1047         }
1048
1049         if (drm_bo_clean_mm(dev, DRM_BO_MEM_VRAM, 1)) {
1050                 DRM_DEBUG("delaying takedown of VRAM memory\n");
1051         }
1052
1053         if (dev_priv->vram_mtrr)
1054                 mtrr_del(dev_priv->vram_mtrr, dev_priv->fb_aper_offset, dev_priv->aper_size);
1055         mutex_unlock(&dev->struct_mutex);
1056
1057         drm_bo_driver_finish(dev);
1058         dev_priv->mm_enabled = false;
1059 }
1060
1061 int radeon_gem_object_pin(struct drm_gem_object *obj,
1062                           uint32_t alignment, uint32_t pin_domain)
1063 {
1064         struct drm_radeon_gem_object *obj_priv;
1065         int ret;
1066         uint32_t flags = DRM_BO_FLAG_NO_EVICT;
1067         uint32_t mask = DRM_BO_FLAG_NO_EVICT;
1068
1069         obj_priv = obj->driver_private;
1070
1071         if (pin_domain) {
1072                 mask |= DRM_BO_MASK_MEM;
1073                 if (pin_domain == RADEON_GEM_DOMAIN_GTT)
1074                         flags |= DRM_BO_FLAG_MEM_TT;
1075                 else if (pin_domain == RADEON_GEM_DOMAIN_VRAM)
1076                         flags |= DRM_BO_FLAG_MEM_VRAM;
1077                 else
1078                         return -EINVAL;
1079         }
1080         ret = drm_bo_do_validate(obj_priv->bo, flags, mask,
1081                                  DRM_BO_HINT_DONT_FENCE, 0);
1082
1083         return ret;
1084 }
1085
1086 int radeon_gem_object_unpin(struct drm_gem_object *obj)
1087 {
1088         struct drm_radeon_gem_object *obj_priv;
1089         int ret;
1090
1091         obj_priv = obj->driver_private;
1092
1093         ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
1094                                  DRM_BO_HINT_DONT_FENCE, 0);
1095
1096         return ret;
1097 }
1098
1099 #define RADEON_IB_MEMORY (1*1024*1024)
1100 #define RADEON_IB_SIZE (65536)
1101
1102 #define RADEON_NUM_IB (RADEON_IB_MEMORY / RADEON_IB_SIZE)
1103
1104 int radeon_gem_ib_get(struct drm_radeon_cs_parser *parser)
1105 {
1106         int i, index = -1;
1107         int ret;
1108         drm_radeon_private_t *dev_priv = parser->dev->dev_private;
1109
1110         for (i = 0; i < RADEON_NUM_IB; i++) {
1111                 if (!(dev_priv->ib_alloc_bitmap & (1 << i))){
1112                         index = i;
1113                         break;
1114                 }
1115         }
1116
1117         /* if all in use we need to wait */
1118         if (index == -1) {
1119                 for (i = 0; i < RADEON_NUM_IB; i++) {
1120                         if (dev_priv->ib_alloc_bitmap & (1 << i)) {
1121                                 mutex_lock(&dev_priv->ib_objs[i]->bo->mutex);
1122                                 ret = drm_bo_wait(dev_priv->ib_objs[i]->bo, 0, 1, 0, 0);
1123                                 mutex_unlock(&dev_priv->ib_objs[i]->bo->mutex);
1124                                 if (ret)
1125                                         continue;
1126                                 dev_priv->ib_alloc_bitmap &= ~(1 << i);
1127                                 index = i;
1128                                 break;
1129                         }
1130                 }
1131         }
1132
1133         if (index == -1) {
1134                 DRM_ERROR("Major case fail to allocate IB from freelist %llx\n", dev_priv->ib_alloc_bitmap);
1135                 return -EINVAL;
1136         }
1137
1138
1139         if (parser->chunks[parser->ib_index].length_dw > RADEON_IB_SIZE / sizeof(uint32_t))
1140                 return -EINVAL;
1141
1142         ret = drm_bo_do_validate(dev_priv->ib_objs[index]->bo, 0,
1143                                  DRM_BO_FLAG_NO_EVICT,
1144                                  0, 0);
1145         if (ret) {
1146                 DRM_ERROR("Failed to validate IB %d\n", index);
1147                 return -EINVAL;
1148         }
1149
1150         parser->ib = dev_priv->ib_objs[index]->kmap.virtual;
1151         parser->card_offset = dev_priv->gart_vm_start + dev_priv->ib_objs[index]->bo->offset;
1152         dev_priv->ib_alloc_bitmap |= (1 << i);
1153         return 0;
1154 }
1155
1156 static void radeon_gem_ib_free(struct drm_radeon_cs_parser *parser)
1157 {
1158         struct drm_device *dev = parser->dev;
1159         drm_radeon_private_t *dev_priv = dev->dev_private;
1160         struct drm_fence_object *fence;
1161         int ret;
1162         int i;
1163
1164         for (i = 0; i < RADEON_NUM_IB; i++) {
1165                 if (dev_priv->ib_objs[i]->kmap.virtual == parser->ib) {
1166                         /* emit a fence object */
1167                         ret = drm_fence_buffer_objects(dev, NULL, 0, NULL, &fence);
1168                         dev_priv->irq_emitted = 0;
1169                         if (ret) {
1170                                 drm_putback_buffer_objects(dev);
1171                         }
1172                         /* dereference the fence object */
1173                         if (fence)
1174                                 drm_fence_usage_deref_unlocked(&fence);
1175                 }
1176         }
1177
1178 }
1179
1180 static int radeon_gem_ib_destroy(struct drm_device *dev)
1181 {
1182         drm_radeon_private_t *dev_priv = dev->dev_private;
1183         int i;
1184
1185         if (dev_priv->ib_objs) {
1186                 for (i = 0; i < RADEON_NUM_IB; i++) {
1187                         if (dev_priv->ib_objs[i]) {
1188                                 drm_bo_kunmap(&dev_priv->ib_objs[i]->kmap);
1189                                 drm_bo_usage_deref_unlocked(&dev_priv->ib_objs[i]->bo);
1190                         }
1191                         drm_free(dev_priv->ib_objs[i], sizeof(struct radeon_mm_obj), DRM_MEM_DRIVER);
1192                 }
1193                 drm_free(dev_priv->ib_objs, RADEON_NUM_IB*sizeof(struct radeon_mm_obj *), DRM_MEM_DRIVER);
1194         }
1195         dev_priv->ib_objs = NULL;
1196         return 0;
1197 }
1198
1199 static int radeon_gem_find_reloc(struct drm_radeon_cs_parser *parser,
1200                                  uint32_t offset, uint32_t *handle,
1201                                  uint32_t *read_domains, uint32_t *write_domain)
1202 {
1203         struct drm_device *dev = parser->dev;
1204         drm_radeon_private_t *dev_priv = dev->dev_private;
1205         struct drm_radeon_kernel_chunk *reloc_chunk = &parser->chunks[parser->reloc_index];
1206
1207         if (!reloc_chunk->kdata)
1208                 return -EINVAL;
1209
1210         if (offset > reloc_chunk->length_dw){
1211                 DRM_ERROR("Offset larger than chunk %d %d\n", offset, reloc_chunk->length_dw);
1212                 return -EINVAL;
1213         }
1214
1215         *handle = reloc_chunk->kdata[offset];
1216         *read_domains = reloc_chunk->kdata[offset + 1];
1217         *write_domain = reloc_chunk->kdata[offset + 2];
1218         return 0;
1219 }
1220
1221 static int radeon_gem_relocate(struct drm_radeon_cs_parser *parser,
1222                                uint32_t *reloc, uint32_t *offset)
1223 {
1224         struct drm_device *dev = parser->dev;
1225         drm_radeon_private_t *dev_priv = dev->dev_private;
1226         /* relocate the handle */
1227         uint32_t read_domains, write_domain;
1228         struct drm_gem_object *obj;
1229         int flags = 0;
1230         int ret;
1231         struct drm_radeon_gem_object *obj_priv;
1232
1233         if (parser->reloc_index == -1) {
1234                 obj = drm_gem_object_lookup(dev, parser->file_priv, reloc[1]);
1235                 if (!obj)
1236                         return -EINVAL;
1237                 read_domains = reloc[2];
1238                 write_domain = reloc[3];
1239         } else {
1240                 uint32_t handle;
1241
1242                 /* have to lookup handle in other chunk */
1243                 ret = radeon_gem_find_reloc(parser, reloc[1], &handle, &read_domains, &write_domain);
1244                 if (ret < 0)
1245                         return ret;
1246
1247                 obj = drm_gem_object_lookup(dev, parser->file_priv, handle);
1248                 if (!obj)
1249                         return -EINVAL;
1250         }
1251
1252         obj_priv = obj->driver_private;
1253         radeon_gem_set_domain(obj, read_domains, write_domain, &flags, false);
1254
1255         obj_priv->bo->mem.flags &= ~DRM_BO_FLAG_CLEAN;
1256         obj_priv->bo->mem.proposed_flags &= ~DRM_BO_FLAG_CLEAN;
1257
1258         if (flags == DRM_BO_FLAG_MEM_VRAM)
1259                 *offset = obj_priv->bo->offset + dev_priv->fb_location;
1260         else if (flags == DRM_BO_FLAG_MEM_TT)
1261                 *offset = obj_priv->bo->offset + dev_priv->gart_vm_start;
1262
1263         /* BAD BAD BAD - LINKED LIST THE OBJS and UNREF ONCE IB is SUBMITTED */
1264         drm_gem_object_unreference(obj);
1265         return 0;
1266 }
1267
1268 /* allocate 1MB of 64k IBs the the kernel can keep mapped */
1269 static int radeon_gem_ib_init(struct drm_device *dev)
1270 {
1271         drm_radeon_private_t *dev_priv = dev->dev_private;
1272         int i;
1273         int ret;
1274
1275         dev_priv->ib_objs = drm_calloc(RADEON_NUM_IB, sizeof(struct radeon_mm_obj *), DRM_MEM_DRIVER);
1276         if (!dev_priv->ib_objs)
1277                 goto free_all;
1278
1279         for (i = 0; i < RADEON_NUM_IB; i++) {
1280                 dev_priv->ib_objs[i] = drm_calloc(1, sizeof(struct radeon_mm_obj), DRM_MEM_DRIVER);
1281                 if (!dev_priv->ib_objs[i])
1282                         goto free_all;
1283
1284                 ret = drm_buffer_object_create(dev, RADEON_IB_SIZE,
1285                                                drm_bo_type_kernel,
1286                                                DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
1287                                                DRM_BO_FLAG_MAPPABLE, 0,
1288                                                0, 0, &dev_priv->ib_objs[i]->bo);
1289                 if (ret)
1290                         goto free_all;
1291
1292                 ret = drm_bo_kmap(dev_priv->ib_objs[i]->bo, 0, RADEON_IB_SIZE >> PAGE_SHIFT,
1293                                   &dev_priv->ib_objs[i]->kmap);
1294
1295                 if (ret)
1296                         goto free_all;
1297         }
1298
1299         dev_priv->mm.gart_useable -= RADEON_IB_SIZE * RADEON_NUM_IB;
1300         dev_priv->ib_alloc_bitmap = 0;
1301
1302         dev_priv->cs.ib_get = radeon_gem_ib_get;
1303         dev_priv->cs.ib_free = radeon_gem_ib_free;
1304
1305         radeon_cs_init(dev);
1306         dev_priv->cs.relocate = radeon_gem_relocate;
1307         return 0;
1308
1309 free_all:
1310         radeon_gem_ib_destroy(dev);
1311         return -ENOMEM;
1312 }
1313
1314 #define RADEON_DMA_BUFFER_SIZE (64 * 1024)
1315 #define RADEON_DMA_BUFFER_COUNT (16)
1316
1317
1318 /**
1319  * Cleanup after an error on one of the addbufs() functions.
1320  *
1321  * \param dev DRM device.
1322  * \param entry buffer entry where the error occurred.
1323  *
1324  * Frees any pages and buffers associated with the given entry.
1325  */
1326 static void drm_cleanup_buf_error(struct drm_device * dev,
1327                                   struct drm_buf_entry * entry)
1328 {
1329         int i;
1330
1331         if (entry->seg_count) {
1332                 for (i = 0; i < entry->seg_count; i++) {
1333                         if (entry->seglist[i]) {
1334                                 drm_pci_free(dev, entry->seglist[i]);
1335                         }
1336                 }
1337                 drm_free(entry->seglist,
1338                          entry->seg_count *
1339                          sizeof(*entry->seglist), DRM_MEM_SEGS);
1340
1341                 entry->seg_count = 0;
1342         }
1343
1344         if (entry->buf_count) {
1345                 for (i = 0; i < entry->buf_count; i++) {
1346                         if (entry->buflist[i].dev_private) {
1347                                 drm_free(entry->buflist[i].dev_private,
1348                                          entry->buflist[i].dev_priv_size,
1349                                          DRM_MEM_BUFS);
1350                         }
1351                 }
1352                 drm_free(entry->buflist,
1353                          entry->buf_count *
1354                          sizeof(*entry->buflist), DRM_MEM_BUFS);
1355
1356                 entry->buf_count = 0;
1357         }
1358 }
1359
1360 static int radeon_gem_addbufs(struct drm_device *dev)
1361 {
1362         struct drm_radeon_private *dev_priv = dev->dev_private;
1363         struct drm_device_dma *dma = dev->dma;
1364         struct drm_buf_entry *entry;
1365         struct drm_buf *buf;
1366         unsigned long offset;
1367         unsigned long agp_offset;
1368         int count;
1369         int order;
1370         int size;
1371         int alignment;
1372         int page_order;
1373         int total;
1374         int byte_count;
1375         int i;
1376         struct drm_buf **temp_buflist;
1377
1378         if (!dma)
1379                 return -EINVAL;
1380
1381         count = RADEON_DMA_BUFFER_COUNT;
1382         order = drm_order(RADEON_DMA_BUFFER_SIZE);
1383         size = 1 << order;
1384
1385         alignment = PAGE_ALIGN(size);
1386         page_order = order - PAGE_SHIFT > 0 ? order - PAGE_SHIFT : 0;
1387         total = PAGE_SIZE << page_order;
1388
1389         byte_count = 0;
1390         agp_offset = dev_priv->mm.dma_bufs.bo->offset;
1391
1392         DRM_DEBUG("count:      %d\n", count);
1393         DRM_DEBUG("order:      %d\n", order);
1394         DRM_DEBUG("size:       %d\n", size);
1395         DRM_DEBUG("agp_offset: %lu\n", agp_offset);
1396         DRM_DEBUG("alignment:  %d\n", alignment);
1397         DRM_DEBUG("page_order: %d\n", page_order);
1398         DRM_DEBUG("total:      %d\n", total);
1399
1400         if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
1401                 return -EINVAL;
1402         if (dev->queue_count)
1403                 return -EBUSY;  /* Not while in use */
1404
1405         spin_lock(&dev->count_lock);
1406         if (dev->buf_use) {
1407                 spin_unlock(&dev->count_lock);
1408                 return -EBUSY;
1409         }
1410         atomic_inc(&dev->buf_alloc);
1411         spin_unlock(&dev->count_lock);
1412
1413         mutex_lock(&dev->struct_mutex);
1414         entry = &dma->bufs[order];
1415         if (entry->buf_count) {
1416                 mutex_unlock(&dev->struct_mutex);
1417                 atomic_dec(&dev->buf_alloc);
1418                 return -ENOMEM; /* May only call once for each order */
1419         }
1420
1421         if (count < 0 || count > 4096) {
1422                 mutex_unlock(&dev->struct_mutex);
1423                 atomic_dec(&dev->buf_alloc);
1424                 return -EINVAL;
1425         }
1426
1427         entry->buflist = drm_alloc(count * sizeof(*entry->buflist),
1428                                    DRM_MEM_BUFS);
1429         if (!entry->buflist) {
1430                 mutex_unlock(&dev->struct_mutex);
1431                 atomic_dec(&dev->buf_alloc);
1432                 return -ENOMEM;
1433         }
1434         memset(entry->buflist, 0, count * sizeof(*entry->buflist));
1435
1436         entry->buf_size = size;
1437         entry->page_order = page_order;
1438
1439         offset = 0;
1440
1441         while (entry->buf_count < count) {
1442                 buf = &entry->buflist[entry->buf_count];
1443                 buf->idx = dma->buf_count + entry->buf_count;
1444                 buf->total = alignment;
1445                 buf->order = order;
1446                 buf->used = 0;
1447
1448                 buf->offset = (dma->byte_count + offset);
1449                 buf->bus_address = dev_priv->gart_vm_start + agp_offset + offset;
1450                 buf->address = (void *)(agp_offset + offset);
1451                 buf->next = NULL;
1452                 buf->waiting = 0;
1453                 buf->pending = 0;
1454                 init_waitqueue_head(&buf->dma_wait);
1455                 buf->file_priv = NULL;
1456
1457                 buf->dev_priv_size = dev->driver->dev_priv_size;
1458                 buf->dev_private = drm_alloc(buf->dev_priv_size, DRM_MEM_BUFS);
1459                 if (!buf->dev_private) {
1460                         /* Set count correctly so we free the proper amount. */
1461                         entry->buf_count = count;
1462                         drm_cleanup_buf_error(dev, entry);
1463                         mutex_unlock(&dev->struct_mutex);
1464                         atomic_dec(&dev->buf_alloc);
1465                         return -ENOMEM;
1466                 }
1467
1468                 memset(buf->dev_private, 0, buf->dev_priv_size);
1469
1470                 DRM_DEBUG("buffer %d @ %p\n", entry->buf_count, buf->address);
1471
1472                 offset += alignment;
1473                 entry->buf_count++;
1474                 byte_count += PAGE_SIZE << page_order;
1475         }
1476
1477         DRM_DEBUG("byte_count: %d\n", byte_count);
1478
1479         temp_buflist = drm_realloc(dma->buflist,
1480                                    dma->buf_count * sizeof(*dma->buflist),
1481                                    (dma->buf_count + entry->buf_count)
1482                                    * sizeof(*dma->buflist), DRM_MEM_BUFS);
1483         if (!temp_buflist) {
1484                 /* Free the entry because it isn't valid */
1485                 drm_cleanup_buf_error(dev, entry);
1486                 mutex_unlock(&dev->struct_mutex);
1487                 atomic_dec(&dev->buf_alloc);
1488                 return -ENOMEM;
1489         }
1490         dma->buflist = temp_buflist;
1491
1492         for (i = 0; i < entry->buf_count; i++) {
1493                 dma->buflist[i + dma->buf_count] = &entry->buflist[i];
1494         }
1495
1496         dma->buf_count += entry->buf_count;
1497         dma->seg_count += entry->seg_count;
1498         dma->page_count += byte_count >> PAGE_SHIFT;
1499         dma->byte_count += byte_count;
1500
1501         DRM_DEBUG("dma->buf_count : %d\n", dma->buf_count);
1502         DRM_DEBUG("entry->buf_count : %d\n", entry->buf_count);
1503
1504         mutex_unlock(&dev->struct_mutex);
1505
1506         dma->flags = _DRM_DMA_USE_SG;
1507         atomic_dec(&dev->buf_alloc);
1508         return 0;
1509 }
1510
1511 static int radeon_gem_dma_bufs_init(struct drm_device *dev)
1512 {
1513         struct drm_radeon_private *dev_priv = dev->dev_private;
1514         int size = RADEON_DMA_BUFFER_SIZE * RADEON_DMA_BUFFER_COUNT;
1515         int ret;
1516
1517         ret = drm_dma_setup(dev);
1518         if (ret < 0)
1519                 return ret;
1520
1521         ret = drm_buffer_object_create(dev, size, drm_bo_type_kernel,
1522                                        DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_NO_EVICT |
1523                                        DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_MAPPABLE, 0,
1524                                        0, 0, &dev_priv->mm.dma_bufs.bo);
1525         if (ret) {
1526                 DRM_ERROR("Failed to create DMA bufs\n");
1527                 return -ENOMEM;
1528         }
1529
1530         ret = drm_bo_kmap(dev_priv->mm.dma_bufs.bo, 0, size >> PAGE_SHIFT,
1531                           &dev_priv->mm.dma_bufs.kmap);
1532         if (ret) {
1533                 DRM_ERROR("Failed to mmap DMA buffers\n");
1534                 return -ENOMEM;
1535         }
1536         dev_priv->mm.gart_useable -= size;
1537         DRM_DEBUG("\n");
1538         radeon_gem_addbufs(dev);
1539
1540         DRM_DEBUG("%lx %d\n", dev_priv->mm.dma_bufs.bo->map_list.hash.key, size);
1541         dev->agp_buffer_token = dev_priv->mm.dma_bufs.bo->map_list.hash.key << PAGE_SHIFT;
1542         dev_priv->mm.fake_agp_map.handle = dev_priv->mm.dma_bufs.kmap.virtual;
1543         dev_priv->mm.fake_agp_map.size = size;
1544
1545         dev->agp_buffer_map = &dev_priv->mm.fake_agp_map;
1546         dev_priv->gart_buffers_offset = dev_priv->mm.dma_bufs.bo->offset + dev_priv->gart_vm_start;
1547         return 0;
1548 }
1549
1550 static void radeon_gem_dma_bufs_destroy(struct drm_device *dev)
1551 {
1552
1553         struct drm_radeon_private *dev_priv = dev->dev_private;
1554         drm_dma_takedown(dev);
1555
1556         if (dev_priv->mm.dma_bufs.bo) {
1557                 drm_bo_kunmap(&dev_priv->mm.dma_bufs.kmap);
1558                 drm_bo_usage_deref_unlocked(&dev_priv->mm.dma_bufs.bo);
1559         }
1560 }
1561
1562
1563 static struct drm_gem_object *gem_object_get(struct drm_device *dev, uint32_t name)
1564 {
1565         struct drm_gem_object *obj;
1566
1567         spin_lock(&dev->object_name_lock);
1568         obj = idr_find(&dev->object_name_idr, name);
1569         if (obj)
1570                 drm_gem_object_reference(obj);
1571         spin_unlock(&dev->object_name_lock);
1572         return obj;
1573 }
1574
1575 void radeon_gem_update_offsets(struct drm_device *dev, struct drm_master *master)
1576 {
1577         drm_radeon_private_t *dev_priv = dev->dev_private;
1578         struct drm_radeon_master_private *master_priv = master->driver_priv;
1579         drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
1580         struct drm_gem_object *obj;
1581         struct drm_radeon_gem_object *obj_priv;
1582
1583         /* update front_pitch_offset and back_pitch_offset */
1584         obj = gem_object_get(dev, sarea_priv->front_handle);
1585         if (obj) {
1586                 obj_priv = obj->driver_private;
1587
1588                 dev_priv->front_offset = obj_priv->bo->offset;
1589                 dev_priv->front_pitch_offset = (((sarea_priv->front_pitch / 64) << 22) |
1590                                                 ((obj_priv->bo->offset
1591                                                   + dev_priv->fb_location) >> 10));
1592                 drm_gem_object_unreference(obj);
1593         }
1594
1595         obj = gem_object_get(dev, sarea_priv->back_handle);
1596         if (obj) {
1597                 obj_priv = obj->driver_private;
1598                 dev_priv->back_offset = obj_priv->bo->offset;
1599                 dev_priv->back_pitch_offset = (((sarea_priv->back_pitch / 64) << 22) |
1600                                                 ((obj_priv->bo->offset
1601                                                   + dev_priv->fb_location) >> 10));
1602                 drm_gem_object_unreference(obj);
1603         }
1604         dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1605
1606 }
1607
1608