linux-core/radeon_gem.c

   1 /*
   2  * Copyright 2008 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Dave Airlie
  23  */
  24 #include "drmP.h"
  25 #include "drm.h"
  26
  27 #include "radeon_drm.h"
  28 #include "radeon_drv.h"
  29
  30 static int radeon_gem_ib_init(struct drm_device *dev);
  31 static int radeon_gem_ib_destroy(struct drm_device *dev);
  32 static int radeon_gem_dma_bufs_init(struct drm_device *dev);
  33 static void radeon_gem_dma_bufs_destroy(struct drm_device *dev);
  34
  35 int radeon_gem_init_object(struct drm_gem_object *obj)
  36 {
  37         struct drm_radeon_gem_object *obj_priv;
  38
  39         obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
  40         if (!obj_priv) {
  41                 return -ENOMEM;
  42         }
  43
  44         obj->driver_private = obj_priv;
  45         obj_priv->obj = obj;
  46         return 0;
  47 }
  48
  49 void radeon_gem_free_object(struct drm_gem_object *obj)
  50 {
  51
  52         struct drm_radeon_gem_object *obj_priv = obj->driver_private;
  53
  54         /* tear down the buffer object - gem holds struct mutex */
  55         drm_bo_takedown_vm_locked(obj_priv->bo);
  56         drm_bo_usage_deref_locked(&obj_priv->bo);
  57         drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
  58 }
  59
  60 int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
  61                           struct drm_file *file_priv)
  62 {
  63         struct drm_radeon_private *dev_priv = dev->dev_private;
  64         struct drm_radeon_gem_info *args = data;
  65
  66         args->vram_start = dev_priv->mm.vram_offset;
  67         args->vram_size = dev_priv->mm.vram_size;
  68         args->vram_visible = dev_priv->mm.vram_visible;
  69
  70         args->gart_start = dev_priv->mm.gart_start;
  71         args->gart_size = dev_priv->mm.gart_size;
  72
  73         return 0;
  74 }
  75
  76 struct drm_gem_object *radeon_gem_object_alloc(struct drm_device *dev, int size, int alignment,
  77                                                int initial_domain)
  78 {
  79         struct drm_gem_object *obj;
  80         struct drm_radeon_gem_object *obj_priv;
  81         int ret;
  82         uint32_t flags;
  83
  84         obj = drm_gem_object_alloc(dev, size);
  85         if (!obj)
  86                 return NULL;;
  87
  88         obj_priv = obj->driver_private;
  89         flags = DRM_BO_FLAG_MAPPABLE;
  90         if (initial_domain == RADEON_GEM_DOMAIN_VRAM)
  91                 flags |= DRM_BO_FLAG_MEM_VRAM;
  92         else if (initial_domain == RADEON_GEM_DOMAIN_GTT)
  93                 flags |= DRM_BO_FLAG_MEM_TT;
  94         else
  95                 flags |= DRM_BO_FLAG_MEM_LOCAL | DRM_BO_FLAG_CACHED;
  96
  97         flags |= DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_EXE;
  98         /* create a TTM BO */
  99         ret = drm_buffer_object_create(dev,
 100                                        size, drm_bo_type_device,
 101                                        flags, 0, alignment,
 102                                        0, &obj_priv->bo);
 103         if (ret)
 104                 goto fail;
 105
 106         DRM_DEBUG("%p : size 0x%x, alignment %d, initial_domain %d\n", obj_priv->bo, size, alignment, initial_domain);
 107         return obj;
 108 fail:
 109
 110         return NULL;
 111 }
 112
 113 int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
 114                             struct drm_file *file_priv)
 115 {
 116         struct drm_radeon_gem_create *args = data;
 117         struct drm_radeon_gem_object *obj_priv;
 118         struct drm_gem_object *obj;
 119         int ret = 0;
 120         uint32_t flags;
 121         int handle;
 122
 123         /* create a gem object to contain this object in */
 124         args->size = roundup(args->size, PAGE_SIZE);
 125
 126         obj = radeon_gem_object_alloc(dev, args->size, args->alignment, args->initial_domain);
 127         if (!obj)
 128                 return -EINVAL;
 129
 130         obj_priv = obj->driver_private;
 131         DRM_DEBUG("obj is %p bo is %p, %d\n", obj, obj_priv->bo, obj_priv->bo->num_pages);
 132         ret = drm_gem_handle_create(file_priv, obj, &handle);
 133         mutex_lock(&dev->struct_mutex);
 134         drm_gem_object_handle_unreference(obj);
 135         mutex_unlock(&dev->struct_mutex);
 136
 137         if (ret)
 138                 goto fail;
 139
 140         args->handle = handle;
 141
 142         return 0;
 143 fail:
 144         drm_gem_object_unreference(obj);
 145
 146         return ret;
 147 }
 148
 149 int radeon_gem_set_domain(struct drm_gem_object *obj, uint32_t read_domains, uint32_t write_domain, uint32_t *flags_p, bool unfenced)
 150 {
 151         struct drm_device *dev = obj->dev;
 152         drm_radeon_private_t *dev_priv = dev->dev_private;
 153         struct drm_radeon_gem_object *obj_priv;
 154         uint32_t flags = 0;
 155         int ret;
 156
 157         obj_priv = obj->driver_private;
 158
 159         /* work out where to validate the buffer to */
 160         if (write_domain) { /* write domains always win */
 161                 if (write_domain == RADEON_GEM_DOMAIN_VRAM)
 162                         flags = DRM_BO_FLAG_MEM_VRAM;
 163                 else if (write_domain == RADEON_GEM_DOMAIN_GTT)
 164                         flags = DRM_BO_FLAG_MEM_TT; // need a can write gart check
 165                 else
 166                         return -EINVAL; // we can't write to system RAM
 167         } else {
 168                 /* okay for a read domain - prefer wherever the object is now or close enough */
 169                 if (read_domains == 0)
 170                         return -EINVAL;
 171
 172                 /* if its already a local memory and CPU is valid do nothing */
 173                 if (read_domains & RADEON_GEM_DOMAIN_CPU) {
 174                         if (obj_priv->bo->mem.mem_type == DRM_BO_MEM_LOCAL)
 175                                 return 0;
 176                         if (read_domains == RADEON_GEM_DOMAIN_CPU)
 177                                 return -EINVAL;
 178                 }
 179
 180                 /* simple case no choice in domains */
 181                 if (read_domains == RADEON_GEM_DOMAIN_VRAM)
 182                         flags = DRM_BO_FLAG_MEM_VRAM;
 183                 else if (read_domains == RADEON_GEM_DOMAIN_GTT)
 184                         flags = DRM_BO_FLAG_MEM_TT;
 185                 else if ((obj_priv->bo->mem.mem_type == DRM_BO_MEM_VRAM) && (read_domains & RADEON_GEM_DOMAIN_VRAM))
 186                         flags = DRM_BO_FLAG_MEM_VRAM;
 187                 else if ((obj_priv->bo->mem.mem_type == DRM_BO_MEM_TT) && (read_domains & RADEON_GEM_DOMAIN_GTT))
 188                         flags = DRM_BO_FLAG_MEM_TT;
 189                 else if ((obj_priv->bo->mem.mem_type == DRM_BO_MEM_LOCAL) && (read_domains & RADEON_GEM_DOMAIN_GTT))
 190                         flags = DRM_BO_FLAG_MEM_TT;
 191                 else if (read_domains & RADEON_GEM_DOMAIN_VRAM)
 192                         flags = DRM_BO_FLAG_MEM_VRAM;
 193                 else if (read_domains & RADEON_GEM_DOMAIN_GTT)
 194                         flags = DRM_BO_FLAG_MEM_TT;
 195         }
 196
 197         /* if this BO is pinned then we ain't moving it anywhere */
 198         if (obj_priv->bo->pinned_mem_type && unfenced)
 199                 return 0;
 200
 201         DRM_DEBUG("validating %p from %d into %x %d %d\n", obj_priv->bo, obj_priv->bo->mem.mem_type, flags, read_domains, write_domain);
 202         ret = drm_bo_do_validate(obj_priv->bo, flags, DRM_BO_MASK_MEM | DRM_BO_FLAG_CACHED,
 203                                  unfenced ? DRM_BO_HINT_DONT_FENCE : 0, 0);
 204         if (ret)
 205                 return ret;
 206
 207         if (flags_p)
 208                 *flags_p = flags;
 209         return 0;
 210
 211 }
 212
 213 int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 214                                 struct drm_file *file_priv)
 215 {
 216         /* transition the BO to a domain - just validate the BO into a certain domain */
 217         struct drm_radeon_gem_set_domain *args = data;
 218         struct drm_gem_object *obj;
 219         struct drm_radeon_gem_object *obj_priv;
 220         int ret;
 221
 222         /* for now if someone requests domain CPU - just make sure the buffer is finished with */
 223
 224         /* just do a BO wait for now */
 225         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 226         if (obj == NULL)
 227                 return -EINVAL;
 228
 229         obj_priv = obj->driver_private;
 230
 231         ret = radeon_gem_set_domain(obj, args->read_domains, args->write_domain, NULL, true);
 232
 233         mutex_lock(&dev->struct_mutex);
 234         drm_gem_object_unreference(obj);
 235         mutex_unlock(&dev->struct_mutex);
 236         return ret;
 237 }
 238
 239 int radeon_gem_pread_ioctl(struct drm_device *dev, void *data,
 240                           struct drm_file *file_priv)
 241 {
 242         return -ENOSYS;
 243 }
 244
 245 int radeon_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 246                           struct drm_file *file_priv)
 247 {
 248         struct drm_radeon_gem_pwrite *args = data;
 249         struct drm_gem_object *obj;
 250         struct drm_radeon_gem_object *obj_priv;
 251         int ret;
 252
 253         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 254         if (obj == NULL)
 255                 return -EINVAL;
 256
 257         obj_priv = obj->driver_private;
 258
 259         /* check where the buffer is first - if not in VRAM
 260            fallback to userspace copying for now */
 261         mutex_lock(&obj_priv->bo->mutex);
 262         if (obj_priv->bo->mem.mem_type != DRM_BO_MEM_VRAM) {
 263                 ret = -EINVAL;
 264                 goto out_unlock;
 265         }
 266
 267         DRM_ERROR("pwriting data->size %lld %llx\n", args->size, args->offset);
 268         ret = -EINVAL;
 269
 270 #if 0
 271         /* so need to grab an IB, copy the data into it in a loop
 272            and send them to VRAM using HDB */
 273         while ((buf = radeon_host_data_blit(dev, cpp, w, dst_pitch_off, &buf_pitch,
 274                                             x, &y, (unsigned int*)&h, &hpass)) != 0) {
 275                 radeon_host_data_blit_copy_pass(dev, cpp, buf, (uint8_t *)src,
 276                                                 hpass, buf_pitch, src_pitch);
 277                 src += hpass * src_pitch;
 278         }
 279 #endif
 280 out_unlock:
 281         mutex_unlock(&obj_priv->bo->mutex);
 282         mutex_lock(&dev->struct_mutex);
 283         drm_gem_object_unreference(obj);
 284         mutex_unlock(&dev->struct_mutex);
 285         return ret;
 286 }
 287
 288 int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
 289                           struct drm_file *file_priv)
 290 {
 291         struct drm_radeon_gem_mmap *args = data;
 292         struct drm_gem_object *obj;
 293         struct drm_radeon_gem_object *obj_priv;
 294         loff_t offset;
 295         unsigned long addr;
 296
 297         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 298         if (obj == NULL)
 299                 return -EINVAL;
 300
 301         offset = args->offset;
 302
 303         DRM_DEBUG("got here %p\n", obj);
 304         obj_priv = obj->driver_private;
 305
 306         DRM_DEBUG("got here %p %p %lld %ld\n", obj, obj_priv->bo, args->size, obj_priv->bo->num_pages);
 307         if (!obj_priv->bo) {
 308                 mutex_lock(&dev->struct_mutex);
 309                 drm_gem_object_unreference(obj);
 310                 mutex_unlock(&dev->struct_mutex);
 311                 return -EINVAL;
 312         }
 313
 314         down_write(&current->mm->mmap_sem);
 315         addr = do_mmap_pgoff(file_priv->filp, 0, args->size,
 316                              PROT_READ | PROT_WRITE, MAP_SHARED,
 317                              obj_priv->bo->map_list.hash.key);
 318         up_write(&current->mm->mmap_sem);
 319
 320         DRM_DEBUG("got here %p %d\n", obj, obj_priv->bo->mem.mem_type);
 321         mutex_lock(&dev->struct_mutex);
 322         drm_gem_object_unreference(obj);
 323         mutex_unlock(&dev->struct_mutex);
 324         if (IS_ERR((void *)addr))
 325                 return addr;
 326
 327         args->addr_ptr = (uint64_t) addr;
 328
 329         return 0;
 330
 331 }
 332
 333 int radeon_gem_pin_ioctl(struct drm_device *dev, void *data,
 334                          struct drm_file *file_priv)
 335 {
 336         struct drm_radeon_gem_pin *args = data;
 337         struct drm_gem_object *obj;
 338         struct drm_radeon_gem_object *obj_priv;
 339         int ret;
 340         int flags = DRM_BO_FLAG_NO_EVICT;
 341         int mask = DRM_BO_FLAG_NO_EVICT;
 342
 343         /* check for valid args */
 344         if (args->pin_domain) {
 345                 mask |= DRM_BO_MASK_MEM;
 346                 if (args->pin_domain == RADEON_GEM_DOMAIN_GTT)
 347                         flags |= DRM_BO_FLAG_MEM_TT;
 348                 else if (args->pin_domain == RADEON_GEM_DOMAIN_VRAM)
 349                         flags |= DRM_BO_FLAG_MEM_VRAM;
 350                 else
 351                         return -EINVAL;
 352         }
 353
 354         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 355         if (obj == NULL)
 356                 return -EINVAL;
 357
 358         obj_priv = obj->driver_private;
 359
 360         /* validate into a pin with no fence */
 361         DRM_DEBUG("got here %p %p %d\n", obj, obj_priv->bo, atomic_read(&obj_priv->bo->usage));
 362         if (!(obj_priv->bo->type != drm_bo_type_kernel && !DRM_SUSER(DRM_CURPROC))) {
 363                 ret = drm_bo_do_validate(obj_priv->bo, flags, mask,
 364                                          DRM_BO_HINT_DONT_FENCE, 0);
 365         } else
 366           ret = 0;
 367
 368         args->offset = obj_priv->bo->offset;
 369         DRM_DEBUG("got here %p %p %x\n", obj, obj_priv->bo, obj_priv->bo->offset);
 370
 371         mutex_lock(&dev->struct_mutex);
 372         drm_gem_object_unreference(obj);
 373         mutex_unlock(&dev->struct_mutex);
 374         return ret;
 375 }
 376
 377 int radeon_gem_unpin_ioctl(struct drm_device *dev, void *data,
 378                            struct drm_file *file_priv)
 379 {
 380         struct drm_radeon_gem_unpin *args = data;
 381         struct drm_gem_object *obj;
 382         struct drm_radeon_gem_object *obj_priv;
 383         int ret;
 384
 385         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 386         if (obj == NULL)
 387                 return -EINVAL;
 388
 389         obj_priv = obj->driver_private;
 390
 391         /* validate into a pin with no fence */
 392
 393         ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
 394                                  DRM_BO_HINT_DONT_FENCE, 0);
 395
 396         mutex_lock(&dev->struct_mutex);
 397         drm_gem_object_unreference(obj);
 398         mutex_unlock(&dev->struct_mutex);
 399         return ret;
 400 }
 401
 402 int radeon_gem_busy(struct drm_device *dev, void *data,
 403                      struct drm_file *file_priv)
 404 {
 405         return 0;
 406 }
 407
 408 int radeon_gem_execbuffer(struct drm_device *dev, void *data,
 409                           struct drm_file *file_priv)
 410 {
 411         return -ENOSYS;
 412
 413
 414 }
 415
 416 int radeon_gem_indirect_ioctl(struct drm_device *dev, void *data,
 417                               struct drm_file *file_priv)
 418 {
 419         struct drm_radeon_gem_indirect *args = data;
 420         struct drm_radeon_private *dev_priv = dev->dev_private;
 421         struct drm_gem_object *obj;
 422         struct drm_radeon_gem_object *obj_priv;
 423         uint32_t start, end;
 424         int ret;
 425         RING_LOCALS;
 426
 427         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 428         if (obj == NULL)
 429                 return -EINVAL;
 430
 431         obj_priv = obj->driver_private;
 432
 433         DRM_DEBUG("got here %p %d\n", obj, args->used);
 434         //RING_SPACE_TEST_WITH_RETURN(dev_priv);
 435         //VB_AGE_TEST_WITH_RETURN(dev_priv);
 436
 437         ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
 438                                  0 , 0);
 439         if (ret)
 440                 return ret;
 441
 442         /* Wait for the 3D stream to idle before the indirect buffer
 443          * containing 2D acceleration commands is processed.
 444          */
 445         BEGIN_RING(2);
 446
 447         RADEON_WAIT_UNTIL_3D_IDLE();
 448
 449         ADVANCE_RING();
 450
 451         start = 0;
 452         end = args->used;
 453
 454         if (start != end) {
 455                 int offset = (dev_priv->gart_vm_start +
 456                               + obj_priv->bo->offset + start);
 457                 int dwords = (end - start + 3) / sizeof(u32);
 458
 459                 /* Fire off the indirect buffer */
 460                 BEGIN_RING(3);
 461
 462                 OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
 463                 OUT_RING(offset);
 464                 OUT_RING(dwords);
 465
 466                 ADVANCE_RING();
 467         }
 468
 469         COMMIT_RING();
 470
 471         /* we need to fence the buffer */
 472         ret = drm_fence_buffer_objects(dev, NULL, 0, NULL, &obj_priv->fence);
 473         if (ret) {
 474
 475                 drm_putback_buffer_objects(dev);
 476                 ret = 0;
 477                 goto fail;
 478         }
 479
 480         /* dereference he fence object */
 481         drm_fence_usage_deref_unlocked(&obj_priv->fence);
 482
 483         mutex_lock(&dev->struct_mutex);
 484         drm_gem_object_unreference(obj);
 485         mutex_unlock(&dev->struct_mutex);
 486         ret = 0;
 487  fail:
 488         return ret;
 489 }
 490
 491 /*
 492  * Depending on card genertation, chipset bugs, etc... the amount of vram
 493  * accessible to the CPU can vary. This function is our best shot at figuring
 494  * it out. Returns a value in KB.
 495  */
 496 static uint32_t radeon_get_accessible_vram(struct drm_device *dev)
 497 {
 498         drm_radeon_private_t *dev_priv = dev->dev_private;
 499         uint32_t aper_size;
 500         u8 byte;
 501
 502         if (dev_priv->chip_family >= CHIP_R600)
 503                 aper_size = RADEON_READ(R600_CONFIG_APER_SIZE) / 1024;
 504         else
 505                 aper_size = RADEON_READ(RADEON_CONFIG_APER_SIZE) / 1024;
 506
 507         /* Set HDP_APER_CNTL only on cards that are known not to be broken,
 508          * that is has the 2nd generation multifunction PCI interface
 509          */
 510         if (dev_priv->chip_family == CHIP_RV280 ||
 511             dev_priv->chip_family == CHIP_RV350 ||
 512             dev_priv->chip_family == CHIP_RV380 ||
 513             dev_priv->chip_family == CHIP_R420 ||
 514             dev_priv->chip_family == CHIP_RV410 ||
 515             radeon_is_avivo(dev_priv)) {
 516                 uint32_t temp = RADEON_READ(RADEON_HOST_PATH_CNTL);
 517                 temp |= RADEON_HDP_APER_CNTL;
 518                 RADEON_WRITE(RADEON_HOST_PATH_CNTL, temp);
 519                 return aper_size * 2;
 520         }
 521
 522         /* Older cards have all sorts of funny issues to deal with. First
 523          * check if it's a multifunction card by reading the PCI config
 524          * header type... Limit those to one aperture size
 525          */
 526         pci_read_config_byte(dev->pdev, 0xe, &byte);
 527         if (byte & 0x80)
 528                 return aper_size;
 529
 530         /* Single function older card. We read HDP_APER_CNTL to see how the BIOS
 531          * have set it up. We don't write this as it's broken on some ASICs but
 532          * we expect the BIOS to have done the right thing (might be too optimistic...)
 533          */
 534         if (RADEON_READ(RADEON_HOST_PATH_CNTL) & RADEON_HDP_APER_CNTL)
 535                 return aper_size * 2;
 536
 537         return aper_size;
 538 }
 539
 540 /* code from the DDX - do memory sizing */
 541 void radeon_vram_setup(struct drm_device *dev)
 542 {
 543         drm_radeon_private_t *dev_priv = dev->dev_private;
 544         uint32_t vram;
 545         uint32_t accessible,  bar_size;
 546
 547         if (!radeon_is_avivo(dev_priv) && (dev_priv->flags & RADEON_IS_IGP)) {
 548                 uint32_t tom = RADEON_READ(RADEON_NB_TOM);
 549
 550                 vram = (((tom >> 16) - (tom & 0xffff) + 1) << 6);
 551                 RADEON_WRITE(RADEON_CONFIG_MEMSIZE, vram * 1024);
 552         } else {
 553                 if (dev_priv->chip_family >= CHIP_R600)
 554                         vram = RADEON_READ(R600_CONFIG_MEMSIZE) / 1024;
 555                 else {
 556                         vram = RADEON_READ(RADEON_CONFIG_MEMSIZE) / 1024;
 557
 558                         /* Some production boards of m6 will return 0 if it's 8 MB */
 559                         if (vram == 0) {
 560                                 vram = 8192;
 561                                 RADEON_WRITE(RADEON_CONFIG_MEMSIZE, 0x800000);
 562                         }
 563                 }
 564         }
 565
 566         accessible = radeon_get_accessible_vram(dev);
 567
 568         bar_size = drm_get_resource_len(dev, 0) / 1024;
 569         if (bar_size == 0)
 570                 bar_size = 0x20000;
 571         if (accessible > bar_size)
 572                 accessible = bar_size;
 573
 574         DRM_INFO("Detected VRAM RAM=%dK, accessible=%uK, BAR=%uK\n",
 575                  vram, accessible, bar_size);
 576
 577         dev_priv->mm.vram_offset = dev_priv->fb_aper_offset;
 578         dev_priv->mm.vram_size = vram * 1024;
 579         dev_priv->mm.vram_visible = accessible * 1024;
 580
 581
 582 }
 583
 584 static int radeon_gart_init(struct drm_device *dev)
 585 {
 586         drm_radeon_private_t *dev_priv = dev->dev_private;
 587         int ret;
 588         u32 base = 0;
 589
 590         /* setup a 32MB GART */
 591         dev_priv->gart_size = dev_priv->mm.gart_size;
 592
 593         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
 594
 595 #if __OS_HAS_AGP
 596         /* setup VRAM vs GART here */
 597         if (dev_priv->flags & RADEON_IS_AGP) {
 598                 base = dev->agp->base;
 599                 if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
 600                     base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
 601                         DRM_INFO("Can't use agp base @0x%08xlx, won't fit\n",
 602                                  dev->agp->base);
 603                         base = 0;
 604                 }
 605         }
 606 #endif
 607
 608         if (base == 0) {
 609                 base = dev_priv->fb_location + dev_priv->fb_size;
 610                 if (base < dev_priv->fb_location ||
 611                     ((base + dev_priv->gart_size) & 0xfffffffful) < base)
 612                         base = dev_priv->fb_location
 613                                 - dev_priv->gart_size;
 614         }
 615         /* start on the card */
 616         dev_priv->gart_vm_start = base & 0xffc00000u;
 617         if (dev_priv->gart_vm_start != base)
 618                 DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
 619                          base, dev_priv->gart_vm_start);
 620
 621         /* if on PCIE we need to allocate an fb object for the PCIE GART table */
 622         if (dev_priv->flags & RADEON_IS_PCIE) {
 623                 ret = drm_buffer_object_create(dev, RADEON_PCIGART_TABLE_SIZE,
 624                                                drm_bo_type_kernel,
 625                                                DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_VRAM | DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
 626                                                0, 1, 0, &dev_priv->mm.pcie_table.bo);
 627                 if (ret)
 628                         return -EINVAL;
 629
 630                 dev_priv->mm.pcie_table_backup = kzalloc(RADEON_PCIGART_TABLE_SIZE, GFP_KERNEL);
 631                 if (!dev_priv->mm.pcie_table_backup)
 632                         return -EINVAL;
 633
 634                 ret = drm_bo_kmap(dev_priv->mm.pcie_table.bo, 0, RADEON_PCIGART_TABLE_SIZE >> PAGE_SHIFT,
 635                                   &dev_priv->mm.pcie_table.kmap);
 636                 if (ret)
 637                         return -EINVAL;
 638
 639                 dev_priv->pcigart_offset_set = 2;
 640                 dev_priv->gart_info.bus_addr =  dev_priv->fb_location + dev_priv->mm.pcie_table.bo->offset;
 641                 dev_priv->gart_info.addr = dev_priv->mm.pcie_table.kmap.virtual;
 642                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
 643                 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_FB;
 644                 memset(dev_priv->gart_info.addr, 0, RADEON_PCIGART_TABLE_SIZE);
 645         } else if (!(dev_priv->flags & RADEON_IS_AGP)) {
 646                 /* allocate PCI GART table */
 647                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
 648                 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
 649                 if (dev_priv->flags & RADEON_IS_IGPGART)
 650                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
 651                 else
 652                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
 653
 654                 ret = drm_ati_alloc_pcigart_table(dev, &dev_priv->gart_info);
 655                 if (ret) {
 656                         DRM_ERROR("cannot allocate PCI GART page!\n");
 657                         return -EINVAL;
 658                 }
 659
 660                 dev_priv->gart_info.addr = dev_priv->gart_info.table_handle->vaddr;
 661                 dev_priv->gart_info.bus_addr = dev_priv->gart_info.table_handle->busaddr;
 662         }
 663
 664         /* gart values setup - start the GART */
 665         if (dev_priv->flags & RADEON_IS_AGP) {
 666                 radeon_set_pcigart(dev_priv, 0);
 667         } else {
 668                 radeon_set_pcigart(dev_priv, 1);
 669         }
 670
 671         return 0;
 672 }
 673
 674 int radeon_alloc_gart_objects(struct drm_device *dev)
 675 {
 676         drm_radeon_private_t *dev_priv = dev->dev_private;
 677         int ret;
 678
 679         ret = drm_buffer_object_create(dev, RADEON_DEFAULT_RING_SIZE,
 680                                        drm_bo_type_kernel,
 681                                        DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
 682                                        DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
 683                                        0, 1, 0, &dev_priv->mm.ring.bo);
 684         if (ret) {
 685                 if (dev_priv->flags & RADEON_IS_AGP)
 686                         DRM_ERROR("failed to allocate ring - most likely an AGP driver bug\n");
 687                 else
 688                         DRM_ERROR("failed to allocate ring\n");
 689                 return -EINVAL;
 690         }
 691
 692         ret = drm_bo_kmap(dev_priv->mm.ring.bo, 0, RADEON_DEFAULT_RING_SIZE >> PAGE_SHIFT,
 693                           &dev_priv->mm.ring.kmap);
 694         if (ret) {
 695                 DRM_ERROR("failed to map ring\n");
 696                 return -EINVAL;
 697         }
 698
 699         ret = drm_buffer_object_create(dev, PAGE_SIZE,
 700                                        drm_bo_type_kernel,
 701                                        DRM_BO_FLAG_WRITE |DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
 702                                        DRM_BO_FLAG_MAPPABLE | DRM_BO_FLAG_NO_EVICT,
 703                                        0, 1, 0, &dev_priv->mm.ring_read.bo);
 704         if (ret) {
 705                 DRM_ERROR("failed to allocate ring read\n");
 706                 return -EINVAL;
 707         }
 708
 709         ret = drm_bo_kmap(dev_priv->mm.ring_read.bo, 0,
 710                           PAGE_SIZE >> PAGE_SHIFT,
 711                           &dev_priv->mm.ring_read.kmap);
 712         if (ret) {
 713                 DRM_ERROR("failed to map ring read\n");
 714                 return -EINVAL;
 715         }
 716
 717         DRM_DEBUG("Ring ptr %p mapped at %d %p, read ptr %p maped at %d %p\n",
 718                   dev_priv->mm.ring.bo, dev_priv->mm.ring.bo->offset, dev_priv->mm.ring.kmap.virtual,
 719                   dev_priv->mm.ring_read.bo, dev_priv->mm.ring_read.bo->offset, dev_priv->mm.ring_read.kmap.virtual);
 720
 721         /* init the indirect buffers */
 722         radeon_gem_ib_init(dev);
 723         radeon_gem_dma_bufs_init(dev);
 724         return 0;
 725
 726 }
 727
 728 static bool avivo_get_mc_idle(struct drm_device *dev)
 729 {
 730         drm_radeon_private_t *dev_priv = dev->dev_private;
 731
 732         if (dev_priv->chip_family >= CHIP_R600) {
 733                 /* no idea where this is on r600 yet */
 734                 return true;
 735         } else if (dev_priv->chip_family == CHIP_RV515) {
 736                 if (radeon_read_mc_reg(dev_priv, RV515_MC_STATUS) & RV515_MC_STATUS_IDLE)
 737                         return true;
 738                 else
 739                         return false;
 740         } else if (dev_priv->chip_family == CHIP_RS600) {
 741                 if (radeon_read_mc_reg(dev_priv, RS600_MC_STATUS) & RS600_MC_STATUS_IDLE)
 742                         return true;
 743                 else
 744                         return false;
 745         } else if ((dev_priv->chip_family == CHIP_RS690) ||
 746                    (dev_priv->chip_family == CHIP_RS740)) {
 747                 if (radeon_read_mc_reg(dev_priv, RS690_MC_STATUS) & RS690_MC_STATUS_IDLE)
 748                         return true;
 749                 else
 750                         return false;
 751         } else {
 752                 if (radeon_read_mc_reg(dev_priv, R520_MC_STATUS) & R520_MC_STATUS_IDLE)
 753                         return true;
 754                 else
 755                         return false;
 756         }
 757 }
 758
 759
 760 static void avivo_disable_mc_clients(struct drm_device *dev)
 761 {
 762         drm_radeon_private_t *dev_priv = dev->dev_private;
 763         uint32_t tmp;
 764         int timeout;
 765
 766         radeon_do_wait_for_idle(dev_priv);
 767
 768         RADEON_WRITE(AVIVO_D1VGA_CONTROL, RADEON_READ(AVIVO_D1VGA_CONTROL) & ~AVIVO_DVGA_CONTROL_MODE_ENABLE);
 769         RADEON_WRITE(AVIVO_D2VGA_CONTROL, RADEON_READ(AVIVO_D2VGA_CONTROL) & ~AVIVO_DVGA_CONTROL_MODE_ENABLE);
 770
 771         tmp = RADEON_READ(AVIVO_D1CRTC_CONTROL);
 772         RADEON_WRITE(AVIVO_D1CRTC_CONTROL, tmp & ~AVIVO_CRTC_EN);
 773
 774         tmp = RADEON_READ(AVIVO_D2CRTC_CONTROL);
 775         RADEON_WRITE(AVIVO_D2CRTC_CONTROL, tmp & ~AVIVO_CRTC_EN);
 776
 777         tmp = RADEON_READ(AVIVO_D2CRTC_CONTROL);
 778
 779         udelay(1000);
 780
 781         timeout = 0;
 782         while (!(avivo_get_mc_idle(dev))) {
 783                 if (++timeout > 100000) {
 784                         DRM_ERROR("Timeout waiting for memory controller to update settings\n");
 785                         DRM_ERROR("Bad things may or may not happen\n");
 786                 }
 787                 udelay(10);
 788         }
 789 }
 790
 791 static inline u32 radeon_busy_wait(struct drm_device *dev, uint32_t reg, uint32_t bits,
 792                                   unsigned int timeout)
 793 {
 794         drm_radeon_private_t *dev_priv = dev->dev_private;
 795         u32 status;
 796
 797         do {
 798                 udelay(10);
 799                 status = RADEON_READ(reg);
 800                 timeout--;
 801         } while(status != 0xffffffff && (status & bits) && (timeout > 0));
 802
 803         if (timeout == 0)
 804                 status = 0xffffffff;
 805
 806         return status;
 807 }
 808
 809 /* Wait for vertical sync on primary CRTC */
 810 static void radeon_wait_for_vsync(struct drm_device *dev)
 811 {
 812         drm_radeon_private_t *dev_priv = dev->dev_private;
 813         uint32_t       crtc_gen_cntl;
 814         int ret;
 815
 816         crtc_gen_cntl = RADEON_READ(RADEON_CRTC_GEN_CNTL);
 817         if ((crtc_gen_cntl & RADEON_CRTC_DISP_REQ_EN_B) ||
 818             !(crtc_gen_cntl & RADEON_CRTC_EN))
 819                 return;
 820
 821         /* Clear the CRTC_VBLANK_SAVE bit */
 822         RADEON_WRITE(RADEON_CRTC_STATUS, RADEON_CRTC_VBLANK_SAVE_CLEAR);
 823
 824         radeon_busy_wait(dev, RADEON_CRTC_STATUS, RADEON_CRTC_VBLANK_SAVE, 2000);
 825
 826 }
 827
 828 /* Wait for vertical sync on primary CRTC */
 829 static void radeon_wait_for_vsync2(struct drm_device *dev)
 830 {
 831         drm_radeon_private_t *dev_priv = dev->dev_private;
 832         uint32_t       crtc2_gen_cntl;
 833         struct timeval timeout;
 834
 835         crtc2_gen_cntl = RADEON_READ(RADEON_CRTC2_GEN_CNTL);
 836         if ((crtc2_gen_cntl & RADEON_CRTC2_DISP_REQ_EN_B) ||
 837             !(crtc2_gen_cntl & RADEON_CRTC2_EN))
 838                 return;
 839
 840         /* Clear the CRTC_VBLANK_SAVE bit */
 841         RADEON_WRITE(RADEON_CRTC2_STATUS, RADEON_CRTC2_VBLANK_SAVE_CLEAR);
 842
 843         radeon_busy_wait(dev, RADEON_CRTC2_STATUS, RADEON_CRTC2_VBLANK_SAVE, 2000);
 844 }
 845
 846 static void legacy_disable_mc_clients(struct drm_device *dev)
 847 {
 848         drm_radeon_private_t *dev_priv = dev->dev_private;
 849         uint32_t old_mc_status, status_idle;
 850         uint32_t ov0_scale_cntl, crtc_ext_cntl, crtc_gen_cntl, crtc2_gen_cntl;
 851         uint32_t status;
 852
 853         radeon_do_wait_for_idle(dev_priv);
 854
 855         if (dev_priv->flags & RADEON_IS_IGP)
 856                 return;
 857
 858         old_mc_status = RADEON_READ(RADEON_MC_STATUS);
 859
 860         /* stop display and memory access */
 861         ov0_scale_cntl = RADEON_READ(RADEON_OV0_SCALE_CNTL);
 862         RADEON_WRITE(RADEON_OV0_SCALE_CNTL, ov0_scale_cntl & ~RADEON_SCALER_ENABLE);
 863         crtc_ext_cntl = RADEON_READ(RADEON_CRTC_EXT_CNTL);
 864         RADEON_WRITE(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl | RADEON_CRTC_DISPLAY_DIS);
 865         crtc_gen_cntl = RADEON_READ(RADEON_CRTC_GEN_CNTL);
 866
 867         radeon_wait_for_vsync(dev);
 868
 869         RADEON_WRITE(RADEON_CRTC_GEN_CNTL,
 870                      (crtc_gen_cntl & ~(RADEON_CRTC_CUR_EN | RADEON_CRTC_ICON_EN)) |
 871                      RADEON_CRTC_DISP_REQ_EN_B | RADEON_CRTC_EXT_DISP_EN);
 872
 873         if (!(dev_priv->flags & RADEON_SINGLE_CRTC)) {
 874                 crtc2_gen_cntl = RADEON_READ(RADEON_CRTC2_GEN_CNTL);
 875
 876                 radeon_wait_for_vsync2(dev);
 877                 RADEON_WRITE(RADEON_CRTC2_GEN_CNTL,
 878                              (crtc2_gen_cntl &
 879                               ~(RADEON_CRTC2_CUR_EN | RADEON_CRTC2_ICON_EN)) |
 880                              RADEON_CRTC2_DISP_REQ_EN_B);
 881         }
 882
 883         udelay(500);
 884
 885         if (radeon_is_r300(dev_priv))
 886                 status_idle = R300_MC_IDLE;
 887         else
 888                 status_idle = RADEON_MC_IDLE;
 889
 890         status = radeon_busy_wait(dev, RADEON_MC_STATUS, status_idle, 200000);
 891         if (status == 0xffffffff) {
 892                 DRM_ERROR("Timeout waiting for memory controller to update settings\n");
 893                 DRM_ERROR("Bad things may or may not happen\n");
 894         }
 895 }
 896
 897
 898 void radeon_init_memory_map(struct drm_device *dev)
 899 {
 900         drm_radeon_private_t *dev_priv = dev->dev_private;
 901         u32 mem_size, aper_size;
 902         u32 tmp;
 903
 904         dev_priv->mc_fb_location = radeon_read_fb_location(dev_priv);
 905         radeon_read_agp_location(dev_priv, &dev_priv->mc_agp_loc_lo, &dev_priv->mc_agp_loc_hi);
 906
 907         if (dev_priv->chip_family >= CHIP_R600) {
 908                 mem_size = RADEON_READ(R600_CONFIG_MEMSIZE);
 909                 aper_size = RADEON_READ(R600_CONFIG_APER_SIZE);
 910         } else {
 911                 mem_size = RADEON_READ(RADEON_CONFIG_MEMSIZE);
 912                 aper_size = RADEON_READ(RADEON_CONFIG_APER_SIZE);
 913         }
 914
 915         /* M6s report illegal memory size */
 916         if (mem_size == 0)
 917                 mem_size = 8 * 1024 * 1024;
 918
 919         /* for RN50/M6/M7 - Novell bug 204882 */
 920         if (aper_size > mem_size)
 921                 mem_size = aper_size;
 922
 923         if ((dev_priv->chip_family != CHIP_RS600) &&
 924             (dev_priv->chip_family != CHIP_RS690) &&
 925             (dev_priv->chip_family != CHIP_RS740)) {
 926                 if (dev_priv->flags & RADEON_IS_IGP)
 927                         dev_priv->mc_fb_location = RADEON_READ(RADEON_NB_TOM);
 928                 else {
 929                         uint32_t aper0_base;
 930
 931                         if (dev_priv->chip_family >= CHIP_R600)
 932                                 aper0_base = RADEON_READ(R600_CONFIG_F0_BASE);
 933                         else
 934                                 aper0_base = RADEON_READ(RADEON_CONFIG_APER_0_BASE);
 935
 936
 937                         /* Some chips have an "issue" with the memory controller, the
 938                          * location must be aligned to the size. We just align it down,
 939                          * too bad if we walk over the top of system memory, we don't
 940                          * use DMA without a remapped anyway.
 941                          * Affected chips are rv280, all r3xx, and all r4xx, but not IGP
 942                          */
 943                         if (dev_priv->chip_family == CHIP_RV280 ||
 944                             dev_priv->chip_family == CHIP_R300 ||
 945                             dev_priv->chip_family == CHIP_R350 ||
 946                             dev_priv->chip_family == CHIP_RV350 ||
 947                             dev_priv->chip_family == CHIP_RV380 ||
 948                             dev_priv->chip_family == CHIP_R420 ||
 949                             dev_priv->chip_family == CHIP_RV410)
 950                                 aper0_base &= ~(mem_size - 1);
 951
 952                         if (dev_priv->chip_family >= CHIP_R600) {
 953                                 dev_priv->mc_fb_location = (aper0_base >> 24) |
 954                                         (((aper0_base + mem_size - 1) & 0xff000000U) >> 8);
 955                         } else {
 956                                 dev_priv->mc_fb_location = (aper0_base >> 16) |
 957                                         ((aper0_base + mem_size - 1) & 0xffff0000U);
 958                         }
 959                 }
 960         }
 961
 962         if (dev_priv->chip_family >= CHIP_R600)
 963                 dev_priv->fb_location = (dev_priv->mc_fb_location & 0xffff) << 24;
 964         else
 965                 dev_priv->fb_location = (dev_priv->mc_fb_location & 0xffff) << 16;
 966
 967         /* updating mc regs here */
 968         if (radeon_is_avivo(dev_priv))
 969                 avivo_disable_mc_clients(dev);
 970         else
 971                 legacy_disable_mc_clients(dev);
 972
 973         radeon_write_fb_location(dev_priv, dev_priv->mc_fb_location);
 974
 975         if (radeon_is_avivo(dev_priv)) {
 976                 if (dev_priv->chip_family >= CHIP_R600)
 977                         RADEON_WRITE(R600_HDP_NONSURFACE_BASE, (dev_priv->mc_fb_location << 16) & 0xff0000);
 978                 else
 979                         RADEON_WRITE(AVIVO_HDP_FB_LOCATION, dev_priv->mc_fb_location);
 980         }
 981
 982         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
 983         dev_priv->fb_size =
 984                 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
 985                 - dev_priv->fb_location;
 986
 987 }
 988
 989 /* init memory manager - start with all of VRAM and a 32MB GART aperture for now */
 990 int radeon_gem_mm_init(struct drm_device *dev)
 991 {
 992         drm_radeon_private_t *dev_priv = dev->dev_private;
 993         int ret;
 994         u32 pg_offset;
 995
 996         /* init TTM underneath */
 997         drm_bo_driver_init(dev);
 998
 999         /* size the mappable VRAM memory for now */
1000         radeon_vram_setup(dev);
1001
1002         radeon_init_memory_map(dev);
1003
1004 #define VRAM_RESERVE_TEXT (256*1024) /* need to reserve 256 for text mode for now */
1005         dev_priv->mm.vram_visible -= VRAM_RESERVE_TEXT;
1006         pg_offset = VRAM_RESERVE_TEXT >> PAGE_SHIFT;
1007         drm_bo_init_mm(dev, DRM_BO_MEM_VRAM, pg_offset, /*dev_priv->mm.vram_offset >> PAGE_SHIFT,*/
1008                        ((dev_priv->mm.vram_visible) >> PAGE_SHIFT) - 16,
1009                        0);
1010
1011
1012         dev_priv->mm.gart_size = (32 * 1024 * 1024);
1013         dev_priv->mm.gart_start = 0;
1014         ret = radeon_gart_init(dev);
1015         if (ret)
1016                 return -EINVAL;
1017
1018         drm_bo_init_mm(dev, DRM_BO_MEM_TT, 0,
1019                        dev_priv->mm.gart_size >> PAGE_SHIFT,
1020                        0);
1021
1022         /* need to allocate some objects in the GART */
1023         /* ring + ring read ptr */
1024         ret = radeon_alloc_gart_objects(dev);
1025         if (ret) {
1026                 radeon_gem_mm_fini(dev);
1027                 return -EINVAL;
1028         }
1029
1030         dev_priv->mm_enabled = true;
1031         return 0;
1032 }
1033
1034 void radeon_gem_mm_fini(struct drm_device *dev)
1035 {
1036         drm_radeon_private_t *dev_priv = dev->dev_private;
1037
1038         radeon_gem_dma_bufs_destroy(dev);
1039         radeon_gem_ib_destroy(dev);
1040
1041         mutex_lock(&dev->struct_mutex);
1042
1043         if (dev_priv->mm.ring_read.bo) {
1044                 drm_bo_kunmap(&dev_priv->mm.ring_read.kmap);
1045                 drm_bo_usage_deref_locked(&dev_priv->mm.ring_read.bo);
1046         }
1047
1048         if (dev_priv->mm.ring.bo) {
1049                 drm_bo_kunmap(&dev_priv->mm.ring.kmap);
1050                 drm_bo_usage_deref_locked(&dev_priv->mm.ring.bo);
1051         }
1052
1053         if (drm_bo_clean_mm(dev, DRM_BO_MEM_TT, 1)) {
1054                 DRM_DEBUG("delaying takedown of TTM memory\n");
1055         }
1056
1057         if (dev_priv->flags & RADEON_IS_PCIE) {
1058                 if (dev_priv->mm.pcie_table_backup) {
1059                         kfree(dev_priv->mm.pcie_table_backup);
1060                         dev_priv->mm.pcie_table_backup = NULL;
1061                 }
1062                 if (dev_priv->mm.pcie_table.bo) {
1063                         drm_bo_kunmap(&dev_priv->mm.pcie_table.kmap);
1064                         drm_bo_usage_deref_locked(&dev_priv->mm.pcie_table.bo);
1065                 }
1066         }
1067
1068         if (drm_bo_clean_mm(dev, DRM_BO_MEM_VRAM, 1)) {
1069                 DRM_DEBUG("delaying takedown of VRAM memory\n");
1070         }
1071
1072         mutex_unlock(&dev->struct_mutex);
1073
1074         drm_bo_driver_finish(dev);
1075         dev_priv->mm_enabled = false;
1076 }
1077
1078 int radeon_gem_object_pin(struct drm_gem_object *obj,
1079                           uint32_t alignment, uint32_t pin_domain)
1080 {
1081         struct drm_radeon_gem_object *obj_priv;
1082         int ret;
1083         uint32_t flags = DRM_BO_FLAG_NO_EVICT;
1084         uint32_t mask = DRM_BO_FLAG_NO_EVICT;
1085
1086         obj_priv = obj->driver_private;
1087
1088         if (pin_domain) {
1089                 mask |= DRM_BO_MASK_MEM;
1090                 if (pin_domain == RADEON_GEM_DOMAIN_GTT)
1091                         flags |= DRM_BO_FLAG_MEM_TT;
1092                 else if (pin_domain == RADEON_GEM_DOMAIN_VRAM)
1093                         flags |= DRM_BO_FLAG_MEM_VRAM;
1094                 else
1095                         return -EINVAL;
1096         }
1097         ret = drm_bo_do_validate(obj_priv->bo, flags, mask,
1098                                  DRM_BO_HINT_DONT_FENCE, 0);
1099
1100         return ret;
1101 }
1102
1103 int radeon_gem_object_unpin(struct drm_gem_object *obj)
1104 {
1105         struct drm_radeon_gem_object *obj_priv;
1106         int ret;
1107
1108         obj_priv = obj->driver_private;
1109
1110         ret = drm_bo_do_validate(obj_priv->bo, 0, DRM_BO_FLAG_NO_EVICT,
1111                                  DRM_BO_HINT_DONT_FENCE, 0);
1112
1113         return ret;
1114 }
1115
1116 #define RADEON_IB_MEMORY (1*1024*1024)
1117 #define RADEON_IB_SIZE (65536)
1118
1119 #define RADEON_NUM_IB (RADEON_IB_MEMORY / RADEON_IB_SIZE)
1120
1121 int radeon_gem_ib_get(struct drm_device *dev, void **ib, uint32_t dwords, uint32_t *card_offset)
1122 {
1123         int i, index = -1;
1124         int ret;
1125         drm_radeon_private_t *dev_priv = dev->dev_private;
1126
1127         for (i = 0; i < RADEON_NUM_IB; i++) {
1128                 if (!(dev_priv->ib_alloc_bitmap & (1 << i))){
1129                         index = i;
1130                         break;
1131                 }
1132         }
1133
1134         /* if all in use we need to wait */
1135         if (index == -1) {
1136                 for (i = 0; i < RADEON_NUM_IB; i++) {
1137                         if (dev_priv->ib_alloc_bitmap & (1 << i)) {
1138                                 mutex_lock(&dev_priv->ib_objs[i]->bo->mutex);
1139                                 ret = drm_bo_wait(dev_priv->ib_objs[i]->bo, 0, 1, 0, 0);
1140                                 mutex_unlock(&dev_priv->ib_objs[i]->bo->mutex);
1141                                 if (ret)
1142                                         continue;
1143                                 dev_priv->ib_alloc_bitmap &= ~(1 << i);
1144                                 index = i;
1145                                 break;
1146                         }
1147                 }
1148         }
1149
1150         if (index == -1) {
1151                 DRM_ERROR("Major case fail to allocate IB from freelist %x\n", dev_priv->ib_alloc_bitmap);
1152                 return -EINVAL;
1153         }
1154
1155
1156         if (dwords > RADEON_IB_SIZE / sizeof(uint32_t))
1157                 return -EINVAL;
1158
1159         ret = drm_bo_do_validate(dev_priv->ib_objs[index]->bo, 0,
1160                                  DRM_BO_FLAG_NO_EVICT,
1161                                  0, 0);
1162         if (ret) {
1163                 DRM_ERROR("Failed to validate IB %d\n", index);
1164                 return -EINVAL;
1165         }
1166
1167         *card_offset = dev_priv->gart_vm_start + dev_priv->ib_objs[index]->bo->offset;
1168         *ib = dev_priv->ib_objs[index]->kmap.virtual;
1169         dev_priv->ib_alloc_bitmap |= (1 << i);
1170         return 0;
1171 }
1172
1173 static void radeon_gem_ib_free(struct drm_device *dev, void *ib, uint32_t dwords)
1174 {
1175         drm_radeon_private_t *dev_priv = dev->dev_private;
1176         struct drm_fence_object *fence;
1177         int ret;
1178         int i;
1179
1180         for (i = 0; i < RADEON_NUM_IB; i++) {
1181
1182                 if (dev_priv->ib_objs[i]->kmap.virtual == ib) {
1183                         /* emit a fence object */
1184                         ret = drm_fence_buffer_objects(dev, NULL, 0, NULL, &fence);
1185                         if (ret) {
1186
1187                                 drm_putback_buffer_objects(dev);
1188                         }
1189                         /* dereference the fence object */
1190                         if (fence)
1191                                 drm_fence_usage_deref_unlocked(&fence);
1192                 }
1193         }
1194
1195 }
1196
1197 static int radeon_gem_ib_destroy(struct drm_device *dev)
1198 {
1199         drm_radeon_private_t *dev_priv = dev->dev_private;
1200         int i;
1201
1202         if (dev_priv->ib_objs) {
1203                 for (i = 0; i < RADEON_NUM_IB; i++) {
1204                         if (dev_priv->ib_objs[i]) {
1205                                 drm_bo_kunmap(&dev_priv->ib_objs[i]->kmap);
1206                                 drm_bo_usage_deref_unlocked(&dev_priv->ib_objs[i]->bo);
1207                         }
1208                         drm_free(dev_priv->ib_objs[i], sizeof(struct radeon_mm_obj), DRM_MEM_DRIVER);
1209                 }
1210                 drm_free(dev_priv->ib_objs, RADEON_NUM_IB*sizeof(struct radeon_mm_obj *), DRM_MEM_DRIVER);
1211         }
1212         dev_priv->ib_objs = NULL;
1213         return 0;
1214 }
1215
1216 static int radeon_gem_relocate(struct drm_device *dev, struct drm_file *file_priv,
1217                                 uint32_t *reloc, uint32_t *offset)
1218 {
1219         drm_radeon_private_t *dev_priv = dev->dev_private;
1220         /* relocate the handle */
1221         uint32_t read_domains = reloc[2];
1222         uint32_t write_domain = reloc[3];
1223         struct drm_gem_object *obj;
1224         int flags = 0;
1225         int ret;
1226         struct drm_radeon_gem_object *obj_priv;
1227
1228         obj = drm_gem_object_lookup(dev, file_priv, reloc[1]);
1229         if (!obj)
1230                 return -EINVAL;
1231
1232         obj_priv = obj->driver_private;
1233         radeon_gem_set_domain(obj, read_domains, write_domain, &flags, false);
1234
1235         obj_priv->bo->mem.flags &= ~DRM_BO_FLAG_CLEAN;
1236         if (flags == DRM_BO_FLAG_MEM_VRAM)
1237                 *offset = obj_priv->bo->offset + dev_priv->fb_location;
1238         else if (flags == DRM_BO_FLAG_MEM_TT)
1239                 *offset = obj_priv->bo->offset + dev_priv->gart_vm_start;
1240
1241         /* BAD BAD BAD - LINKED LIST THE OBJS and UNREF ONCE IB is SUBMITTED */
1242         drm_gem_object_unreference(obj);
1243         return 0;
1244 }
1245
1246 /* allocate 1MB of 64k IBs the the kernel can keep mapped */
1247 static int radeon_gem_ib_init(struct drm_device *dev)
1248 {
1249         drm_radeon_private_t *dev_priv = dev->dev_private;
1250         int i;
1251         int ret;
1252
1253         dev_priv->ib_objs = drm_calloc(RADEON_NUM_IB, sizeof(struct radeon_mm_obj *), DRM_MEM_DRIVER);
1254         if (!dev_priv->ib_objs)
1255                 goto free_all;
1256
1257         for (i = 0; i < RADEON_NUM_IB; i++) {
1258                 dev_priv->ib_objs[i] = drm_calloc(1, sizeof(struct radeon_mm_obj), DRM_MEM_DRIVER);
1259                 if (!dev_priv->ib_objs[i])
1260                         goto free_all;
1261
1262                 ret = drm_buffer_object_create(dev, RADEON_IB_SIZE,
1263                                                drm_bo_type_kernel,
1264                                                DRM_BO_FLAG_READ | DRM_BO_FLAG_MEM_TT |
1265                                                DRM_BO_FLAG_MAPPABLE, 0,
1266                                                0, 0, &dev_priv->ib_objs[i]->bo);
1267                 if (ret)
1268                         goto free_all;
1269
1270                 ret = drm_bo_kmap(dev_priv->ib_objs[i]->bo, 0, RADEON_IB_SIZE >> PAGE_SHIFT,
1271                                   &dev_priv->ib_objs[i]->kmap);
1272
1273                 if (ret)
1274                         goto free_all;
1275         }
1276
1277         dev_priv->ib_alloc_bitmap = 0;
1278
1279         dev_priv->cs.ib_get = radeon_gem_ib_get;
1280         dev_priv->cs.ib_free = radeon_gem_ib_free;
1281
1282         radeon_cs_init(dev);
1283         dev_priv->cs.relocate = radeon_gem_relocate;
1284         return 0;
1285
1286 free_all:
1287         radeon_gem_ib_destroy(dev);
1288         return -ENOMEM;
1289 }
1290
1291 #define RADEON_DMA_BUFFER_SIZE (64 * 1024)
1292 #define RADEON_DMA_BUFFER_COUNT (16)
1293
1294
1295 /**
1296  * Cleanup after an error on one of the addbufs() functions.
1297  *
1298  * \param dev DRM device.
1299  * \param entry buffer entry where the error occurred.
1300  *
1301  * Frees any pages and buffers associated with the given entry.
1302  */
1303 static void drm_cleanup_buf_error(struct drm_device * dev,
1304                                   struct drm_buf_entry * entry)
1305 {
1306         int i;
1307
1308         if (entry->seg_count) {
1309                 for (i = 0; i < entry->seg_count; i++) {
1310                         if (entry->seglist[i]) {
1311                                 drm_pci_free(dev, entry->seglist[i]);
1312                         }
1313                 }
1314                 drm_free(entry->seglist,
1315                          entry->seg_count *
1316                          sizeof(*entry->seglist), DRM_MEM_SEGS);
1317
1318                 entry->seg_count = 0;
1319         }
1320
1321         if (entry->buf_count) {
1322                 for (i = 0; i < entry->buf_count; i++) {
1323                         if (entry->buflist[i].dev_private) {
1324                                 drm_free(entry->buflist[i].dev_private,
1325                                          entry->buflist[i].dev_priv_size,
1326                                          DRM_MEM_BUFS);
1327                         }
1328                 }
1329                 drm_free(entry->buflist,
1330                          entry->buf_count *
1331                          sizeof(*entry->buflist), DRM_MEM_BUFS);
1332
1333                 entry->buf_count = 0;
1334         }
1335 }
1336
1337 static int radeon_gem_addbufs(struct drm_device *dev)
1338 {
1339         struct drm_radeon_private *dev_priv = dev->dev_private;
1340         struct drm_device_dma *dma = dev->dma;
1341         struct drm_buf_entry *entry;
1342         struct drm_buf *buf;
1343         unsigned long offset;
1344         unsigned long agp_offset;
1345         int count;
1346         int order;
1347         int size;
1348         int alignment;
1349         int page_order;
1350         int total;
1351         int byte_count;
1352         int i;
1353         struct drm_buf **temp_buflist;
1354
1355         if (!dma)
1356                 return -EINVAL;
1357
1358         count = RADEON_DMA_BUFFER_COUNT;
1359         order = drm_order(RADEON_DMA_BUFFER_SIZE);
1360         size = 1 << order;
1361
1362         alignment = PAGE_ALIGN(size);
1363         page_order = order - PAGE_SHIFT > 0 ? order - PAGE_SHIFT : 0;
1364         total = PAGE_SIZE << page_order;
1365
1366         byte_count = 0;
1367         agp_offset = dev_priv->mm.dma_bufs.bo->offset;
1368
1369         DRM_DEBUG("count:      %d\n", count);
1370         DRM_DEBUG("order:      %d\n", order);
1371         DRM_DEBUG("size:       %d\n", size);
1372         DRM_DEBUG("agp_offset: %lu\n", agp_offset);
1373         DRM_DEBUG("alignment:  %d\n", alignment);
1374         DRM_DEBUG("page_order: %d\n", page_order);
1375         DRM_DEBUG("total:      %d\n", total);
1376
1377         if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
1378                 return -EINVAL;
1379         if (dev->queue_count)
1380                 return -EBUSY;  /* Not while in use */
1381
1382         spin_lock(&dev->count_lock);
1383         if (dev->buf_use) {
1384                 spin_unlock(&dev->count_lock);
1385                 return -EBUSY;
1386         }
1387         atomic_inc(&dev->buf_alloc);
1388         spin_unlock(&dev->count_lock);
1389
1390         mutex_lock(&dev->struct_mutex);
1391         entry = &dma->bufs[order];
1392         if (entry->buf_count) {
1393                 mutex_unlock(&dev->struct_mutex);
1394                 atomic_dec(&dev->buf_alloc);
1395                 return -ENOMEM; /* May only call once for each order */
1396         }
1397
1398         if (count < 0 || count > 4096) {
1399                 mutex_unlock(&dev->struct_mutex);
1400                 atomic_dec(&dev->buf_alloc);
1401                 return -EINVAL;
1402         }
1403
1404         entry->buflist = drm_alloc(count * sizeof(*entry->buflist),
1405                                    DRM_MEM_BUFS);
1406         if (!entry->buflist) {
1407                 mutex_unlock(&dev->struct_mutex);
1408                 atomic_dec(&dev->buf_alloc);
1409                 return -ENOMEM;
1410         }
1411         memset(entry->buflist, 0, count * sizeof(*entry->buflist));
1412
1413         entry->buf_size = size;
1414         entry->page_order = page_order;
1415
1416         offset = 0;
1417
1418         while (entry->buf_count < count) {
1419                 buf = &entry->buflist[entry->buf_count];
1420                 buf->idx = dma->buf_count + entry->buf_count;
1421                 buf->total = alignment;
1422                 buf->order = order;
1423                 buf->used = 0;
1424
1425                 buf->offset = (dma->byte_count + offset);
1426                 buf->bus_address = dev_priv->gart_vm_start + agp_offset + offset;
1427                 buf->address = (void *)(agp_offset + offset);
1428                 buf->next = NULL;
1429                 buf->waiting = 0;
1430                 buf->pending = 0;
1431                 init_waitqueue_head(&buf->dma_wait);
1432                 buf->file_priv = NULL;
1433
1434                 buf->dev_priv_size = dev->driver->dev_priv_size;
1435                 buf->dev_private = drm_alloc(buf->dev_priv_size, DRM_MEM_BUFS);
1436                 if (!buf->dev_private) {
1437                         /* Set count correctly so we free the proper amount. */
1438                         entry->buf_count = count;
1439                         drm_cleanup_buf_error(dev, entry);
1440                         mutex_unlock(&dev->struct_mutex);
1441                         atomic_dec(&dev->buf_alloc);
1442                         return -ENOMEM;
1443                 }
1444
1445                 memset(buf->dev_private, 0, buf->dev_priv_size);
1446
1447                 DRM_DEBUG("buffer %d @ %p\n", entry->buf_count, buf->address);
1448
1449                 offset += alignment;
1450                 entry->buf_count++;
1451                 byte_count += PAGE_SIZE << page_order;
1452         }
1453
1454         DRM_DEBUG("byte_count: %d\n", byte_count);
1455
1456         temp_buflist = drm_realloc(dma->buflist,
1457                                    dma->buf_count * sizeof(*dma->buflist),
1458                                    (dma->buf_count + entry->buf_count)
1459                                    * sizeof(*dma->buflist), DRM_MEM_BUFS);
1460         if (!temp_buflist) {
1461                 /* Free the entry because it isn't valid */
1462                 drm_cleanup_buf_error(dev, entry);
1463                 mutex_unlock(&dev->struct_mutex);
1464                 atomic_dec(&dev->buf_alloc);
1465                 return -ENOMEM;
1466         }
1467         dma->buflist = temp_buflist;
1468
1469         for (i = 0; i < entry->buf_count; i++) {
1470                 dma->buflist[i + dma->buf_count] = &entry->buflist[i];
1471         }
1472
1473         dma->buf_count += entry->buf_count;
1474         dma->seg_count += entry->seg_count;
1475         dma->page_count += byte_count >> PAGE_SHIFT;
1476         dma->byte_count += byte_count;
1477
1478         DRM_DEBUG("dma->buf_count : %d\n", dma->buf_count);
1479         DRM_DEBUG("entry->buf_count : %d\n", entry->buf_count);
1480
1481         mutex_unlock(&dev->struct_mutex);
1482
1483         dma->flags = _DRM_DMA_USE_SG;
1484         atomic_dec(&dev->buf_alloc);
1485         return 0;
1486 }
1487
1488 static int radeon_gem_dma_bufs_init(struct drm_device *dev)
1489 {
1490         struct drm_radeon_private *dev_priv = dev->dev_private;
1491         int size = RADEON_DMA_BUFFER_SIZE * RADEON_DMA_BUFFER_COUNT;
1492         int ret;
1493
1494         ret = drm_dma_setup(dev);
1495         if (ret < 0)
1496                 return ret;
1497
1498         ret = drm_buffer_object_create(dev, size, drm_bo_type_device,
1499                                        DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_NO_EVICT |
1500                                        DRM_BO_FLAG_MEM_TT | DRM_BO_FLAG_MAPPABLE, 0,
1501                                        0, 0, &dev_priv->mm.dma_bufs.bo);
1502         if (ret) {
1503                 DRM_ERROR("Failed to create DMA bufs\n");
1504                 return -ENOMEM;
1505         }
1506
1507         ret = drm_bo_kmap(dev_priv->mm.dma_bufs.bo, 0, size >> PAGE_SHIFT,
1508                           &dev_priv->mm.dma_bufs.kmap);
1509         if (ret) {
1510                 DRM_ERROR("Failed to mmap DMA buffers\n");
1511                 return -ENOMEM;
1512         }
1513         DRM_DEBUG("\n");
1514         radeon_gem_addbufs(dev);
1515
1516         DRM_DEBUG("%x %d\n", dev_priv->mm.dma_bufs.bo->map_list.hash.key, size);
1517         dev->agp_buffer_token = dev_priv->mm.dma_bufs.bo->map_list.hash.key << PAGE_SHIFT;
1518         dev_priv->mm.fake_agp_map.handle = dev_priv->mm.dma_bufs.kmap.virtual;
1519         dev_priv->mm.fake_agp_map.size = size;
1520
1521         dev->agp_buffer_map = &dev_priv->mm.fake_agp_map;
1522         dev_priv->gart_buffers_offset = dev_priv->mm.dma_bufs.bo->offset + dev_priv->gart_vm_start;
1523         return 0;
1524 }
1525
1526 static void radeon_gem_dma_bufs_destroy(struct drm_device *dev)
1527 {
1528
1529         struct drm_radeon_private *dev_priv = dev->dev_private;
1530         drm_dma_takedown(dev);
1531
1532         if (dev_priv->mm.dma_bufs.bo) {
1533                 drm_bo_kunmap(&dev_priv->mm.dma_bufs.kmap);
1534                 drm_bo_usage_deref_unlocked(&dev_priv->mm.dma_bufs.bo);
1535         }
1536 }
1537
1538
1539 static struct drm_gem_object *gem_object_get(struct drm_device *dev, uint32_t name)
1540 {
1541         struct drm_gem_object *obj;
1542
1543         spin_lock(&dev->object_name_lock);
1544         obj = idr_find(&dev->object_name_idr, name);
1545         if (obj)
1546                 drm_gem_object_reference(obj);
1547         spin_unlock(&dev->object_name_lock);
1548         return obj;
1549 }
1550
1551 void radeon_gem_update_offsets(struct drm_device *dev, struct drm_master *master)
1552 {
1553         drm_radeon_private_t *dev_priv = dev->dev_private;
1554         struct drm_radeon_master_private *master_priv = master->driver_priv;
1555         drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
1556         struct drm_gem_object *obj;
1557         struct drm_radeon_gem_object *obj_priv;
1558
1559         /* update front_pitch_offset and back_pitch_offset */
1560         obj = gem_object_get(dev, sarea_priv->front_handle);
1561         if (obj) {
1562                 obj_priv = obj->driver_private;
1563
1564                 dev_priv->front_offset = obj_priv->bo->offset;
1565                 dev_priv->front_pitch_offset = (((sarea_priv->front_pitch / 64) << 22) |
1566                                                 ((obj_priv->bo->offset
1567                                                   + dev_priv->fb_location) >> 10));
1568                 drm_gem_object_unreference(obj);
1569         }
1570
1571         obj = gem_object_get(dev, sarea_priv->back_handle);
1572         if (obj) {
1573                 obj_priv = obj->driver_private;
1574                 dev_priv->back_offset = obj_priv->bo->offset;
1575                 dev_priv->back_pitch_offset = (((sarea_priv->back_pitch / 64) << 22) |
1576                                                 ((obj_priv->bo->offset
1577                                                   + dev_priv->fb_location) >> 10));
1578                 drm_gem_object_unreference(obj);
1579         }
1580         dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1581
1582 }
1583
1584