drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37
  38 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  39 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  42 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  43                                              int write);
  44 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  45                                                      uint64_t offset,
  46                                                      uint64_t size);
  47 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  48 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  49 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  50                                            unsigned alignment);
  51 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  52 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  53                                 struct drm_i915_gem_pwrite *args,
  54                                 struct drm_file *file_priv);
  55 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  56
  57 static LIST_HEAD(shrink_list);
  58 static DEFINE_SPINLOCK(shrink_list_lock);
  59
  60 static inline bool
  61 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
  62 {
  63         return obj_priv->gtt_space &&
  64                 !obj_priv->active &&
  65                 obj_priv->pin_count == 0;
  66 }
  67
  68 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  69                      unsigned long end)
  70 {
  71         drm_i915_private_t *dev_priv = dev->dev_private;
  72
  73         if (start >= end ||
  74             (start & (PAGE_SIZE - 1)) != 0 ||
  75             (end & (PAGE_SIZE - 1)) != 0) {
  76                 return -EINVAL;
  77         }
  78
  79         drm_mm_init(&dev_priv->mm.gtt_space, start,
  80                     end - start);
  81
  82         dev->gtt_total = (uint32_t) (end - start);
  83
  84         return 0;
  85 }
  86
  87 int
  88 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  89                     struct drm_file *file_priv)
  90 {
  91         struct drm_i915_gem_init *args = data;
  92         int ret;
  93
  94         mutex_lock(&dev->struct_mutex);
  95         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  96         mutex_unlock(&dev->struct_mutex);
  97
  98         return ret;
  99 }
 100
 101 int
 102 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 103                             struct drm_file *file_priv)
 104 {
 105         struct drm_i915_gem_get_aperture *args = data;
 106
 107         if (!(dev->driver->driver_features & DRIVER_GEM))
 108                 return -ENODEV;
 109
 110         args->aper_size = dev->gtt_total;
 111         args->aper_available_size = (args->aper_size -
 112                                      atomic_read(&dev->pin_memory));
 113
 114         return 0;
 115 }
 116
 117
 118 /**
 119  * Creates a new mm object and returns a handle to it.
 120  */
 121 int
 122 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 123                       struct drm_file *file_priv)
 124 {
 125         struct drm_i915_gem_create *args = data;
 126         struct drm_gem_object *obj;
 127         int ret;
 128         u32 handle;
 129
 130         args->size = roundup(args->size, PAGE_SIZE);
 131
 132         /* Allocate the new object */
 133         obj = i915_gem_alloc_object(dev, args->size);
 134         if (obj == NULL)
 135                 return -ENOMEM;
 136
 137         ret = drm_gem_handle_create(file_priv, obj, &handle);
 138         drm_gem_object_unreference_unlocked(obj);
 139         if (ret)
 140                 return ret;
 141
 142         args->handle = handle;
 143
 144         return 0;
 145 }
 146
 147 static inline int
 148 fast_shmem_read(struct page **pages,
 149                 loff_t page_base, int page_offset,
 150                 char __user *data,
 151                 int length)
 152 {
 153         char __iomem *vaddr;
 154         int unwritten;
 155
 156         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 157         if (vaddr == NULL)
 158                 return -ENOMEM;
 159         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 160         kunmap_atomic(vaddr, KM_USER0);
 161
 162         if (unwritten)
 163                 return -EFAULT;
 164
 165         return 0;
 166 }
 167
 168 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 169 {
 170         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 171         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 172
 173         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 174                 obj_priv->tiling_mode != I915_TILING_NONE;
 175 }
 176
 177 static inline void
 178 slow_shmem_copy(struct page *dst_page,
 179                 int dst_offset,
 180                 struct page *src_page,
 181                 int src_offset,
 182                 int length)
 183 {
 184         char *dst_vaddr, *src_vaddr;
 185
 186         dst_vaddr = kmap(dst_page);
 187         src_vaddr = kmap(src_page);
 188
 189         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 190
 191         kunmap(src_page);
 192         kunmap(dst_page);
 193 }
 194
 195 static inline void
 196 slow_shmem_bit17_copy(struct page *gpu_page,
 197                       int gpu_offset,
 198                       struct page *cpu_page,
 199                       int cpu_offset,
 200                       int length,
 201                       int is_read)
 202 {
 203         char *gpu_vaddr, *cpu_vaddr;
 204
 205         /* Use the unswizzled path if this page isn't affected. */
 206         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 207                 if (is_read)
 208                         return slow_shmem_copy(cpu_page, cpu_offset,
 209                                                gpu_page, gpu_offset, length);
 210                 else
 211                         return slow_shmem_copy(gpu_page, gpu_offset,
 212                                                cpu_page, cpu_offset, length);
 213         }
 214
 215         gpu_vaddr = kmap(gpu_page);
 216         cpu_vaddr = kmap(cpu_page);
 217
 218         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 219          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 220          */
 221         while (length > 0) {
 222                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 223                 int this_length = min(cacheline_end - gpu_offset, length);
 224                 int swizzled_gpu_offset = gpu_offset ^ 64;
 225
 226                 if (is_read) {
 227                         memcpy(cpu_vaddr + cpu_offset,
 228                                gpu_vaddr + swizzled_gpu_offset,
 229                                this_length);
 230                 } else {
 231                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 232                                cpu_vaddr + cpu_offset,
 233                                this_length);
 234                 }
 235                 cpu_offset += this_length;
 236                 gpu_offset += this_length;
 237                 length -= this_length;
 238         }
 239
 240         kunmap(cpu_page);
 241         kunmap(gpu_page);
 242 }
 243
 244 /**
 245  * This is the fast shmem pread path, which attempts to copy_from_user directly
 246  * from the backing pages of the object to the user's address space.  On a
 247  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 248  */
 249 static int
 250 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 251                           struct drm_i915_gem_pread *args,
 252                           struct drm_file *file_priv)
 253 {
 254         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 255         ssize_t remain;
 256         loff_t offset, page_base;
 257         char __user *user_data;
 258         int page_offset, page_length;
 259         int ret;
 260
 261         user_data = (char __user *) (uintptr_t) args->data_ptr;
 262         remain = args->size;
 263
 264         mutex_lock(&dev->struct_mutex);
 265
 266         ret = i915_gem_object_get_pages(obj, 0);
 267         if (ret != 0)
 268                 goto fail_unlock;
 269
 270         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 271                                                         args->size);
 272         if (ret != 0)
 273                 goto fail_put_pages;
 274
 275         obj_priv = to_intel_bo(obj);
 276         offset = args->offset;
 277
 278         while (remain > 0) {
 279                 /* Operation in this page
 280                  *
 281                  * page_base = page offset within aperture
 282                  * page_offset = offset within page
 283                  * page_length = bytes to copy for this page
 284                  */
 285                 page_base = (offset & ~(PAGE_SIZE-1));
 286                 page_offset = offset & (PAGE_SIZE-1);
 287                 page_length = remain;
 288                 if ((page_offset + remain) > PAGE_SIZE)
 289                         page_length = PAGE_SIZE - page_offset;
 290
 291                 ret = fast_shmem_read(obj_priv->pages,
 292                                       page_base, page_offset,
 293                                       user_data, page_length);
 294                 if (ret)
 295                         goto fail_put_pages;
 296
 297                 remain -= page_length;
 298                 user_data += page_length;
 299                 offset += page_length;
 300         }
 301
 302 fail_put_pages:
 303         i915_gem_object_put_pages(obj);
 304 fail_unlock:
 305         mutex_unlock(&dev->struct_mutex);
 306
 307         return ret;
 308 }
 309
 310 static int
 311 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 312 {
 313         int ret;
 314
 315         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 316
 317         /* If we've insufficient memory to map in the pages, attempt
 318          * to make some space by throwing out some old buffers.
 319          */
 320         if (ret == -ENOMEM) {
 321                 struct drm_device *dev = obj->dev;
 322
 323                 ret = i915_gem_evict_something(dev, obj->size,
 324                                                i915_gem_get_gtt_alignment(obj));
 325                 if (ret)
 326                         return ret;
 327
 328                 ret = i915_gem_object_get_pages(obj, 0);
 329         }
 330
 331         return ret;
 332 }
 333
 334 /**
 335  * This is the fallback shmem pread path, which allocates temporary storage
 336  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 337  * can copy out of the object's backing pages while holding the struct mutex
 338  * and not take page faults.
 339  */
 340 static int
 341 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 342                           struct drm_i915_gem_pread *args,
 343                           struct drm_file *file_priv)
 344 {
 345         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 346         struct mm_struct *mm = current->mm;
 347         struct page **user_pages;
 348         ssize_t remain;
 349         loff_t offset, pinned_pages, i;
 350         loff_t first_data_page, last_data_page, num_pages;
 351         int shmem_page_index, shmem_page_offset;
 352         int data_page_index,  data_page_offset;
 353         int page_length;
 354         int ret;
 355         uint64_t data_ptr = args->data_ptr;
 356         int do_bit17_swizzling;
 357
 358         remain = args->size;
 359
 360         /* Pin the user pages containing the data.  We can't fault while
 361          * holding the struct mutex, yet we want to hold it while
 362          * dereferencing the user data.
 363          */
 364         first_data_page = data_ptr / PAGE_SIZE;
 365         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 366         num_pages = last_data_page - first_data_page + 1;
 367
 368         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 369         if (user_pages == NULL)
 370                 return -ENOMEM;
 371
 372         down_read(&mm->mmap_sem);
 373         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 374                                       num_pages, 1, 0, user_pages, NULL);
 375         up_read(&mm->mmap_sem);
 376         if (pinned_pages < num_pages) {
 377                 ret = -EFAULT;
 378                 goto fail_put_user_pages;
 379         }
 380
 381         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 382
 383         mutex_lock(&dev->struct_mutex);
 384
 385         ret = i915_gem_object_get_pages_or_evict(obj);
 386         if (ret)
 387                 goto fail_unlock;
 388
 389         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 390                                                         args->size);
 391         if (ret != 0)
 392                 goto fail_put_pages;
 393
 394         obj_priv = to_intel_bo(obj);
 395         offset = args->offset;
 396
 397         while (remain > 0) {
 398                 /* Operation in this page
 399                  *
 400                  * shmem_page_index = page number within shmem file
 401                  * shmem_page_offset = offset within page in shmem file
 402                  * data_page_index = page number in get_user_pages return
 403                  * data_page_offset = offset with data_page_index page.
 404                  * page_length = bytes to copy for this page
 405                  */
 406                 shmem_page_index = offset / PAGE_SIZE;
 407                 shmem_page_offset = offset & ~PAGE_MASK;
 408                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 409                 data_page_offset = data_ptr & ~PAGE_MASK;
 410
 411                 page_length = remain;
 412                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 413                         page_length = PAGE_SIZE - shmem_page_offset;
 414                 if ((data_page_offset + page_length) > PAGE_SIZE)
 415                         page_length = PAGE_SIZE - data_page_offset;
 416
 417                 if (do_bit17_swizzling) {
 418                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 419                                               shmem_page_offset,
 420                                               user_pages[data_page_index],
 421                                               data_page_offset,
 422                                               page_length,
 423                                               1);
 424                 } else {
 425                         slow_shmem_copy(user_pages[data_page_index],
 426                                         data_page_offset,
 427                                         obj_priv->pages[shmem_page_index],
 428                                         shmem_page_offset,
 429                                         page_length);
 430                 }
 431
 432                 remain -= page_length;
 433                 data_ptr += page_length;
 434                 offset += page_length;
 435         }
 436
 437 fail_put_pages:
 438         i915_gem_object_put_pages(obj);
 439 fail_unlock:
 440         mutex_unlock(&dev->struct_mutex);
 441 fail_put_user_pages:
 442         for (i = 0; i < pinned_pages; i++) {
 443                 SetPageDirty(user_pages[i]);
 444                 page_cache_release(user_pages[i]);
 445         }
 446         drm_free_large(user_pages);
 447
 448         return ret;
 449 }
 450
 451 /**
 452  * Reads data from the object referenced by handle.
 453  *
 454  * On error, the contents of *data are undefined.
 455  */
 456 int
 457 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 458                      struct drm_file *file_priv)
 459 {
 460         struct drm_i915_gem_pread *args = data;
 461         struct drm_gem_object *obj;
 462         struct drm_i915_gem_object *obj_priv;
 463         int ret;
 464
 465         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 466         if (obj == NULL)
 467                 return -ENOENT;
 468         obj_priv = to_intel_bo(obj);
 469
 470         /* Bounds check source.
 471          *
 472          * XXX: This could use review for overflow issues...
 473          */
 474         if (args->offset > obj->size || args->size > obj->size ||
 475             args->offset + args->size > obj->size) {
 476                 drm_gem_object_unreference_unlocked(obj);
 477                 return -EINVAL;
 478         }
 479
 480         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 481                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 482         } else {
 483                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 484                 if (ret != 0)
 485                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 486                                                         file_priv);
 487         }
 488
 489         drm_gem_object_unreference_unlocked(obj);
 490
 491         return ret;
 492 }
 493
 494 /* This is the fast write path which cannot handle
 495  * page faults in the source data
 496  */
 497
 498 static inline int
 499 fast_user_write(struct io_mapping *mapping,
 500                 loff_t page_base, int page_offset,
 501                 char __user *user_data,
 502                 int length)
 503 {
 504         char *vaddr_atomic;
 505         unsigned long unwritten;
 506
 507         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 508         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 509                                                       user_data, length);
 510         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 511         if (unwritten)
 512                 return -EFAULT;
 513         return 0;
 514 }
 515
 516 /* Here's the write path which can sleep for
 517  * page faults
 518  */
 519
 520 static inline void
 521 slow_kernel_write(struct io_mapping *mapping,
 522                   loff_t gtt_base, int gtt_offset,
 523                   struct page *user_page, int user_offset,
 524                   int length)
 525 {
 526         char __iomem *dst_vaddr;
 527         char *src_vaddr;
 528
 529         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 530         src_vaddr = kmap(user_page);
 531
 532         memcpy_toio(dst_vaddr + gtt_offset,
 533                     src_vaddr + user_offset,
 534                     length);
 535
 536         kunmap(user_page);
 537         io_mapping_unmap(dst_vaddr);
 538 }
 539
 540 static inline int
 541 fast_shmem_write(struct page **pages,
 542                  loff_t page_base, int page_offset,
 543                  char __user *data,
 544                  int length)
 545 {
 546         char __iomem *vaddr;
 547         unsigned long unwritten;
 548
 549         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 550         if (vaddr == NULL)
 551                 return -ENOMEM;
 552         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 553         kunmap_atomic(vaddr, KM_USER0);
 554
 555         if (unwritten)
 556                 return -EFAULT;
 557         return 0;
 558 }
 559
 560 /**
 561  * This is the fast pwrite path, where we copy the data directly from the
 562  * user into the GTT, uncached.
 563  */
 564 static int
 565 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 566                          struct drm_i915_gem_pwrite *args,
 567                          struct drm_file *file_priv)
 568 {
 569         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 570         drm_i915_private_t *dev_priv = dev->dev_private;
 571         ssize_t remain;
 572         loff_t offset, page_base;
 573         char __user *user_data;
 574         int page_offset, page_length;
 575         int ret;
 576
 577         user_data = (char __user *) (uintptr_t) args->data_ptr;
 578         remain = args->size;
 579         if (!access_ok(VERIFY_READ, user_data, remain))
 580                 return -EFAULT;
 581
 582
 583         mutex_lock(&dev->struct_mutex);
 584         ret = i915_gem_object_pin(obj, 0);
 585         if (ret) {
 586                 mutex_unlock(&dev->struct_mutex);
 587                 return ret;
 588         }
 589         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 590         if (ret)
 591                 goto fail;
 592
 593         obj_priv = to_intel_bo(obj);
 594         offset = obj_priv->gtt_offset + args->offset;
 595
 596         while (remain > 0) {
 597                 /* Operation in this page
 598                  *
 599                  * page_base = page offset within aperture
 600                  * page_offset = offset within page
 601                  * page_length = bytes to copy for this page
 602                  */
 603                 page_base = (offset & ~(PAGE_SIZE-1));
 604                 page_offset = offset & (PAGE_SIZE-1);
 605                 page_length = remain;
 606                 if ((page_offset + remain) > PAGE_SIZE)
 607                         page_length = PAGE_SIZE - page_offset;
 608
 609                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 610                                        page_offset, user_data, page_length);
 611
 612                 /* If we get a fault while copying data, then (presumably) our
 613                  * source page isn't available.  Return the error and we'll
 614                  * retry in the slow path.
 615                  */
 616                 if (ret)
 617                         goto fail;
 618
 619                 remain -= page_length;
 620                 user_data += page_length;
 621                 offset += page_length;
 622         }
 623
 624 fail:
 625         i915_gem_object_unpin(obj);
 626         mutex_unlock(&dev->struct_mutex);
 627
 628         return ret;
 629 }
 630
 631 /**
 632  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 633  * the memory and maps it using kmap_atomic for copying.
 634  *
 635  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 636  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 637  */
 638 static int
 639 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 640                          struct drm_i915_gem_pwrite *args,
 641                          struct drm_file *file_priv)
 642 {
 643         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 644         drm_i915_private_t *dev_priv = dev->dev_private;
 645         ssize_t remain;
 646         loff_t gtt_page_base, offset;
 647         loff_t first_data_page, last_data_page, num_pages;
 648         loff_t pinned_pages, i;
 649         struct page **user_pages;
 650         struct mm_struct *mm = current->mm;
 651         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 652         int ret;
 653         uint64_t data_ptr = args->data_ptr;
 654
 655         remain = args->size;
 656
 657         /* Pin the user pages containing the data.  We can't fault while
 658          * holding the struct mutex, and all of the pwrite implementations
 659          * want to hold it while dereferencing the user data.
 660          */
 661         first_data_page = data_ptr / PAGE_SIZE;
 662         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 663         num_pages = last_data_page - first_data_page + 1;
 664
 665         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 666         if (user_pages == NULL)
 667                 return -ENOMEM;
 668
 669         down_read(&mm->mmap_sem);
 670         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 671                                       num_pages, 0, 0, user_pages, NULL);
 672         up_read(&mm->mmap_sem);
 673         if (pinned_pages < num_pages) {
 674                 ret = -EFAULT;
 675                 goto out_unpin_pages;
 676         }
 677
 678         mutex_lock(&dev->struct_mutex);
 679         ret = i915_gem_object_pin(obj, 0);
 680         if (ret)
 681                 goto out_unlock;
 682
 683         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 684         if (ret)
 685                 goto out_unpin_object;
 686
 687         obj_priv = to_intel_bo(obj);
 688         offset = obj_priv->gtt_offset + args->offset;
 689
 690         while (remain > 0) {
 691                 /* Operation in this page
 692                  *
 693                  * gtt_page_base = page offset within aperture
 694                  * gtt_page_offset = offset within page in aperture
 695                  * data_page_index = page number in get_user_pages return
 696                  * data_page_offset = offset with data_page_index page.
 697                  * page_length = bytes to copy for this page
 698                  */
 699                 gtt_page_base = offset & PAGE_MASK;
 700                 gtt_page_offset = offset & ~PAGE_MASK;
 701                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 702                 data_page_offset = data_ptr & ~PAGE_MASK;
 703
 704                 page_length = remain;
 705                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 706                         page_length = PAGE_SIZE - gtt_page_offset;
 707                 if ((data_page_offset + page_length) > PAGE_SIZE)
 708                         page_length = PAGE_SIZE - data_page_offset;
 709
 710                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 711                                   gtt_page_base, gtt_page_offset,
 712                                   user_pages[data_page_index],
 713                                   data_page_offset,
 714                                   page_length);
 715
 716                 remain -= page_length;
 717                 offset += page_length;
 718                 data_ptr += page_length;
 719         }
 720
 721 out_unpin_object:
 722         i915_gem_object_unpin(obj);
 723 out_unlock:
 724         mutex_unlock(&dev->struct_mutex);
 725 out_unpin_pages:
 726         for (i = 0; i < pinned_pages; i++)
 727                 page_cache_release(user_pages[i]);
 728         drm_free_large(user_pages);
 729
 730         return ret;
 731 }
 732
 733 /**
 734  * This is the fast shmem pwrite path, which attempts to directly
 735  * copy_from_user into the kmapped pages backing the object.
 736  */
 737 static int
 738 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 739                            struct drm_i915_gem_pwrite *args,
 740                            struct drm_file *file_priv)
 741 {
 742         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 743         ssize_t remain;
 744         loff_t offset, page_base;
 745         char __user *user_data;
 746         int page_offset, page_length;
 747         int ret;
 748
 749         user_data = (char __user *) (uintptr_t) args->data_ptr;
 750         remain = args->size;
 751
 752         mutex_lock(&dev->struct_mutex);
 753
 754         ret = i915_gem_object_get_pages(obj, 0);
 755         if (ret != 0)
 756                 goto fail_unlock;
 757
 758         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 759         if (ret != 0)
 760                 goto fail_put_pages;
 761
 762         obj_priv = to_intel_bo(obj);
 763         offset = args->offset;
 764         obj_priv->dirty = 1;
 765
 766         while (remain > 0) {
 767                 /* Operation in this page
 768                  *
 769                  * page_base = page offset within aperture
 770                  * page_offset = offset within page
 771                  * page_length = bytes to copy for this page
 772                  */
 773                 page_base = (offset & ~(PAGE_SIZE-1));
 774                 page_offset = offset & (PAGE_SIZE-1);
 775                 page_length = remain;
 776                 if ((page_offset + remain) > PAGE_SIZE)
 777                         page_length = PAGE_SIZE - page_offset;
 778
 779                 ret = fast_shmem_write(obj_priv->pages,
 780                                        page_base, page_offset,
 781                                        user_data, page_length);
 782                 if (ret)
 783                         goto fail_put_pages;
 784
 785                 remain -= page_length;
 786                 user_data += page_length;
 787                 offset += page_length;
 788         }
 789
 790 fail_put_pages:
 791         i915_gem_object_put_pages(obj);
 792 fail_unlock:
 793         mutex_unlock(&dev->struct_mutex);
 794
 795         return ret;
 796 }
 797
 798 /**
 799  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 800  * the memory and maps it using kmap_atomic for copying.
 801  *
 802  * This avoids taking mmap_sem for faulting on the user's address while the
 803  * struct_mutex is held.
 804  */
 805 static int
 806 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 807                            struct drm_i915_gem_pwrite *args,
 808                            struct drm_file *file_priv)
 809 {
 810         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 811         struct mm_struct *mm = current->mm;
 812         struct page **user_pages;
 813         ssize_t remain;
 814         loff_t offset, pinned_pages, i;
 815         loff_t first_data_page, last_data_page, num_pages;
 816         int shmem_page_index, shmem_page_offset;
 817         int data_page_index,  data_page_offset;
 818         int page_length;
 819         int ret;
 820         uint64_t data_ptr = args->data_ptr;
 821         int do_bit17_swizzling;
 822
 823         remain = args->size;
 824
 825         /* Pin the user pages containing the data.  We can't fault while
 826          * holding the struct mutex, and all of the pwrite implementations
 827          * want to hold it while dereferencing the user data.
 828          */
 829         first_data_page = data_ptr / PAGE_SIZE;
 830         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 831         num_pages = last_data_page - first_data_page + 1;
 832
 833         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 834         if (user_pages == NULL)
 835                 return -ENOMEM;
 836
 837         down_read(&mm->mmap_sem);
 838         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 839                                       num_pages, 0, 0, user_pages, NULL);
 840         up_read(&mm->mmap_sem);
 841         if (pinned_pages < num_pages) {
 842                 ret = -EFAULT;
 843                 goto fail_put_user_pages;
 844         }
 845
 846         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 847
 848         mutex_lock(&dev->struct_mutex);
 849
 850         ret = i915_gem_object_get_pages_or_evict(obj);
 851         if (ret)
 852                 goto fail_unlock;
 853
 854         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 855         if (ret != 0)
 856                 goto fail_put_pages;
 857
 858         obj_priv = to_intel_bo(obj);
 859         offset = args->offset;
 860         obj_priv->dirty = 1;
 861
 862         while (remain > 0) {
 863                 /* Operation in this page
 864                  *
 865                  * shmem_page_index = page number within shmem file
 866                  * shmem_page_offset = offset within page in shmem file
 867                  * data_page_index = page number in get_user_pages return
 868                  * data_page_offset = offset with data_page_index page.
 869                  * page_length = bytes to copy for this page
 870                  */
 871                 shmem_page_index = offset / PAGE_SIZE;
 872                 shmem_page_offset = offset & ~PAGE_MASK;
 873                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 874                 data_page_offset = data_ptr & ~PAGE_MASK;
 875
 876                 page_length = remain;
 877                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 878                         page_length = PAGE_SIZE - shmem_page_offset;
 879                 if ((data_page_offset + page_length) > PAGE_SIZE)
 880                         page_length = PAGE_SIZE - data_page_offset;
 881
 882                 if (do_bit17_swizzling) {
 883                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 884                                               shmem_page_offset,
 885                                               user_pages[data_page_index],
 886                                               data_page_offset,
 887                                               page_length,
 888                                               0);
 889                 } else {
 890                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 891                                         shmem_page_offset,
 892                                         user_pages[data_page_index],
 893                                         data_page_offset,
 894                                         page_length);
 895                 }
 896
 897                 remain -= page_length;
 898                 data_ptr += page_length;
 899                 offset += page_length;
 900         }
 901
 902 fail_put_pages:
 903         i915_gem_object_put_pages(obj);
 904 fail_unlock:
 905         mutex_unlock(&dev->struct_mutex);
 906 fail_put_user_pages:
 907         for (i = 0; i < pinned_pages; i++)
 908                 page_cache_release(user_pages[i]);
 909         drm_free_large(user_pages);
 910
 911         return ret;
 912 }
 913
 914 /**
 915  * Writes data to the object referenced by handle.
 916  *
 917  * On error, the contents of the buffer that were to be modified are undefined.
 918  */
 919 int
 920 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 921                       struct drm_file *file_priv)
 922 {
 923         struct drm_i915_gem_pwrite *args = data;
 924         struct drm_gem_object *obj;
 925         struct drm_i915_gem_object *obj_priv;
 926         int ret = 0;
 927
 928         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 929         if (obj == NULL)
 930                 return -ENOENT;
 931         obj_priv = to_intel_bo(obj);
 932
 933         /* Bounds check destination.
 934          *
 935          * XXX: This could use review for overflow issues...
 936          */
 937         if (args->offset > obj->size || args->size > obj->size ||
 938             args->offset + args->size > obj->size) {
 939                 drm_gem_object_unreference_unlocked(obj);
 940                 return -EINVAL;
 941         }
 942
 943         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 944          * it would end up going through the fenced access, and we'll get
 945          * different detiling behavior between reading and writing.
 946          * pread/pwrite currently are reading and writing from the CPU
 947          * perspective, requiring manual detiling by the client.
 948          */
 949         if (obj_priv->phys_obj)
 950                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 951         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 952                  dev->gtt_total != 0 &&
 953                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
 954                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 955                 if (ret == -EFAULT) {
 956                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 957                                                        file_priv);
 958                 }
 959         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 960                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 961         } else {
 962                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 963                 if (ret == -EFAULT) {
 964                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 965                                                          file_priv);
 966                 }
 967         }
 968
 969 #if WATCH_PWRITE
 970         if (ret)
 971                 DRM_INFO("pwrite failed %d\n", ret);
 972 #endif
 973
 974         drm_gem_object_unreference_unlocked(obj);
 975
 976         return ret;
 977 }
 978
 979 /**
 980  * Called when user space prepares to use an object with the CPU, either
 981  * through the mmap ioctl's mapping or a GTT mapping.
 982  */
 983 int
 984 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 985                           struct drm_file *file_priv)
 986 {
 987         struct drm_i915_private *dev_priv = dev->dev_private;
 988         struct drm_i915_gem_set_domain *args = data;
 989         struct drm_gem_object *obj;
 990         struct drm_i915_gem_object *obj_priv;
 991         uint32_t read_domains = args->read_domains;
 992         uint32_t write_domain = args->write_domain;
 993         int ret;
 994
 995         if (!(dev->driver->driver_features & DRIVER_GEM))
 996                 return -ENODEV;
 997
 998         /* Only handle setting domains to types used by the CPU. */
 999         if (write_domain & I915_GEM_GPU_DOMAINS)
1000                 return -EINVAL;
1001
1002         if (read_domains & I915_GEM_GPU_DOMAINS)
1003                 return -EINVAL;
1004
1005         /* Having something in the write domain implies it's in the read
1006          * domain, and only that read domain.  Enforce that in the request.
1007          */
1008         if (write_domain != 0 && read_domains != write_domain)
1009                 return -EINVAL;
1010
1011         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1012         if (obj == NULL)
1013                 return -ENOENT;
1014         obj_priv = to_intel_bo(obj);
1015
1016         mutex_lock(&dev->struct_mutex);
1017
1018         intel_mark_busy(dev, obj);
1019
1020 #if WATCH_BUF
1021         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1022                  obj, obj->size, read_domains, write_domain);
1023 #endif
1024         if (read_domains & I915_GEM_DOMAIN_GTT) {
1025                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1026
1027                 /* Update the LRU on the fence for the CPU access that's
1028                  * about to occur.
1029                  */
1030                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1031                         struct drm_i915_fence_reg *reg =
1032                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1033                         list_move_tail(&reg->lru_list,
1034                                        &dev_priv->mm.fence_list);
1035                 }
1036
1037                 /* Silently promote "you're not bound, there was nothing to do"
1038                  * to success, since the client was just asking us to
1039                  * make sure everything was done.
1040                  */
1041                 if (ret == -EINVAL)
1042                         ret = 0;
1043         } else {
1044                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1045         }
1046
1047
1048         /* Maintain LRU order of "inactive" objects */
1049         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1050                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1051
1052         drm_gem_object_unreference(obj);
1053         mutex_unlock(&dev->struct_mutex);
1054         return ret;
1055 }
1056
1057 /**
1058  * Called when user space has done writes to this buffer
1059  */
1060 int
1061 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1062                       struct drm_file *file_priv)
1063 {
1064         struct drm_i915_gem_sw_finish *args = data;
1065         struct drm_gem_object *obj;
1066         struct drm_i915_gem_object *obj_priv;
1067         int ret = 0;
1068
1069         if (!(dev->driver->driver_features & DRIVER_GEM))
1070                 return -ENODEV;
1071
1072         mutex_lock(&dev->struct_mutex);
1073         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1074         if (obj == NULL) {
1075                 mutex_unlock(&dev->struct_mutex);
1076                 return -ENOENT;
1077         }
1078
1079 #if WATCH_BUF
1080         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1081                  __func__, args->handle, obj, obj->size);
1082 #endif
1083         obj_priv = to_intel_bo(obj);
1084
1085         /* Pinned buffers may be scanout, so flush the cache */
1086         if (obj_priv->pin_count)
1087                 i915_gem_object_flush_cpu_write_domain(obj);
1088
1089         drm_gem_object_unreference(obj);
1090         mutex_unlock(&dev->struct_mutex);
1091         return ret;
1092 }
1093
1094 /**
1095  * Maps the contents of an object, returning the address it is mapped
1096  * into.
1097  *
1098  * While the mapping holds a reference on the contents of the object, it doesn't
1099  * imply a ref on the object itself.
1100  */
1101 int
1102 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1103                    struct drm_file *file_priv)
1104 {
1105         struct drm_i915_gem_mmap *args = data;
1106         struct drm_gem_object *obj;
1107         loff_t offset;
1108         unsigned long addr;
1109
1110         if (!(dev->driver->driver_features & DRIVER_GEM))
1111                 return -ENODEV;
1112
1113         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1114         if (obj == NULL)
1115                 return -ENOENT;
1116
1117         offset = args->offset;
1118
1119         down_write(&current->mm->mmap_sem);
1120         addr = do_mmap(obj->filp, 0, args->size,
1121                        PROT_READ | PROT_WRITE, MAP_SHARED,
1122                        args->offset);
1123         up_write(&current->mm->mmap_sem);
1124         drm_gem_object_unreference_unlocked(obj);
1125         if (IS_ERR((void *)addr))
1126                 return addr;
1127
1128         args->addr_ptr = (uint64_t) addr;
1129
1130         return 0;
1131 }
1132
1133 /**
1134  * i915_gem_fault - fault a page into the GTT
1135  * vma: VMA in question
1136  * vmf: fault info
1137  *
1138  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1139  * from userspace.  The fault handler takes care of binding the object to
1140  * the GTT (if needed), allocating and programming a fence register (again,
1141  * only if needed based on whether the old reg is still valid or the object
1142  * is tiled) and inserting a new PTE into the faulting process.
1143  *
1144  * Note that the faulting process may involve evicting existing objects
1145  * from the GTT and/or fence registers to make room.  So performance may
1146  * suffer if the GTT working set is large or there are few fence registers
1147  * left.
1148  */
1149 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1150 {
1151         struct drm_gem_object *obj = vma->vm_private_data;
1152         struct drm_device *dev = obj->dev;
1153         drm_i915_private_t *dev_priv = dev->dev_private;
1154         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1155         pgoff_t page_offset;
1156         unsigned long pfn;
1157         int ret = 0;
1158         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1159
1160         /* We don't use vmf->pgoff since that has the fake offset */
1161         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1162                 PAGE_SHIFT;
1163
1164         /* Now bind it into the GTT if needed */
1165         mutex_lock(&dev->struct_mutex);
1166         if (!obj_priv->gtt_space) {
1167                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1168                 if (ret)
1169                         goto unlock;
1170
1171                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1172                 if (ret)
1173                         goto unlock;
1174         }
1175
1176         /* Need a new fence register? */
1177         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1178                 ret = i915_gem_object_get_fence_reg(obj);
1179                 if (ret)
1180                         goto unlock;
1181         }
1182
1183         if (i915_gem_object_is_inactive(obj_priv))
1184                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1185
1186         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1187                 page_offset;
1188
1189         /* Finally, remap it using the new GTT offset */
1190         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1191 unlock:
1192         mutex_unlock(&dev->struct_mutex);
1193
1194         switch (ret) {
1195         case 0:
1196         case -ERESTARTSYS:
1197                 return VM_FAULT_NOPAGE;
1198         case -ENOMEM:
1199         case -EAGAIN:
1200                 return VM_FAULT_OOM;
1201         default:
1202                 return VM_FAULT_SIGBUS;
1203         }
1204 }
1205
1206 /**
1207  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1208  * @obj: obj in question
1209  *
1210  * GEM memory mapping works by handing back to userspace a fake mmap offset
1211  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1212  * up the object based on the offset and sets up the various memory mapping
1213  * structures.
1214  *
1215  * This routine allocates and attaches a fake offset for @obj.
1216  */
1217 static int
1218 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1219 {
1220         struct drm_device *dev = obj->dev;
1221         struct drm_gem_mm *mm = dev->mm_private;
1222         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1223         struct drm_map_list *list;
1224         struct drm_local_map *map;
1225         int ret = 0;
1226
1227         /* Set the object up for mmap'ing */
1228         list = &obj->map_list;
1229         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1230         if (!list->map)
1231                 return -ENOMEM;
1232
1233         map = list->map;
1234         map->type = _DRM_GEM;
1235         map->size = obj->size;
1236         map->handle = obj;
1237
1238         /* Get a DRM GEM mmap offset allocated... */
1239         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1240                                                     obj->size / PAGE_SIZE, 0, 0);
1241         if (!list->file_offset_node) {
1242                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1243                 ret = -ENOMEM;
1244                 goto out_free_list;
1245         }
1246
1247         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1248                                                   obj->size / PAGE_SIZE, 0);
1249         if (!list->file_offset_node) {
1250                 ret = -ENOMEM;
1251                 goto out_free_list;
1252         }
1253
1254         list->hash.key = list->file_offset_node->start;
1255         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1256                 DRM_ERROR("failed to add to map hash\n");
1257                 ret = -ENOMEM;
1258                 goto out_free_mm;
1259         }
1260
1261         /* By now we should be all set, any drm_mmap request on the offset
1262          * below will get to our mmap & fault handler */
1263         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1264
1265         return 0;
1266
1267 out_free_mm:
1268         drm_mm_put_block(list->file_offset_node);
1269 out_free_list:
1270         kfree(list->map);
1271
1272         return ret;
1273 }
1274
1275 /**
1276  * i915_gem_release_mmap - remove physical page mappings
1277  * @obj: obj in question
1278  *
1279  * Preserve the reservation of the mmapping with the DRM core code, but
1280  * relinquish ownership of the pages back to the system.
1281  *
1282  * It is vital that we remove the page mapping if we have mapped a tiled
1283  * object through the GTT and then lose the fence register due to
1284  * resource pressure. Similarly if the object has been moved out of the
1285  * aperture, than pages mapped into userspace must be revoked. Removing the
1286  * mapping will then trigger a page fault on the next user access, allowing
1287  * fixup by i915_gem_fault().
1288  */
1289 void
1290 i915_gem_release_mmap(struct drm_gem_object *obj)
1291 {
1292         struct drm_device *dev = obj->dev;
1293         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1294
1295         if (dev->dev_mapping)
1296                 unmap_mapping_range(dev->dev_mapping,
1297                                     obj_priv->mmap_offset, obj->size, 1);
1298 }
1299
1300 static void
1301 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1302 {
1303         struct drm_device *dev = obj->dev;
1304         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1305         struct drm_gem_mm *mm = dev->mm_private;
1306         struct drm_map_list *list;
1307
1308         list = &obj->map_list;
1309         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1310
1311         if (list->file_offset_node) {
1312                 drm_mm_put_block(list->file_offset_node);
1313                 list->file_offset_node = NULL;
1314         }
1315
1316         if (list->map) {
1317                 kfree(list->map);
1318                 list->map = NULL;
1319         }
1320
1321         obj_priv->mmap_offset = 0;
1322 }
1323
1324 /**
1325  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1326  * @obj: object to check
1327  *
1328  * Return the required GTT alignment for an object, taking into account
1329  * potential fence register mapping if needed.
1330  */
1331 static uint32_t
1332 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1333 {
1334         struct drm_device *dev = obj->dev;
1335         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1336         int start, i;
1337
1338         /*
1339          * Minimum alignment is 4k (GTT page size), but might be greater
1340          * if a fence register is needed for the object.
1341          */
1342         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1343                 return 4096;
1344
1345         /*
1346          * Previous chips need to be aligned to the size of the smallest
1347          * fence register that can contain the object.
1348          */
1349         if (IS_I9XX(dev))
1350                 start = 1024*1024;
1351         else
1352                 start = 512*1024;
1353
1354         for (i = start; i < obj->size; i <<= 1)
1355                 ;
1356
1357         return i;
1358 }
1359
1360 /**
1361  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1362  * @dev: DRM device
1363  * @data: GTT mapping ioctl data
1364  * @file_priv: GEM object info
1365  *
1366  * Simply returns the fake offset to userspace so it can mmap it.
1367  * The mmap call will end up in drm_gem_mmap(), which will set things
1368  * up so we can get faults in the handler above.
1369  *
1370  * The fault handler will take care of binding the object into the GTT
1371  * (since it may have been evicted to make room for something), allocating
1372  * a fence register, and mapping the appropriate aperture address into
1373  * userspace.
1374  */
1375 int
1376 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1377                         struct drm_file *file_priv)
1378 {
1379         struct drm_i915_gem_mmap_gtt *args = data;
1380         struct drm_gem_object *obj;
1381         struct drm_i915_gem_object *obj_priv;
1382         int ret;
1383
1384         if (!(dev->driver->driver_features & DRIVER_GEM))
1385                 return -ENODEV;
1386
1387         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1388         if (obj == NULL)
1389                 return -ENOENT;
1390
1391         mutex_lock(&dev->struct_mutex);
1392
1393         obj_priv = to_intel_bo(obj);
1394
1395         if (obj_priv->madv != I915_MADV_WILLNEED) {
1396                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1397                 drm_gem_object_unreference(obj);
1398                 mutex_unlock(&dev->struct_mutex);
1399                 return -EINVAL;
1400         }
1401
1402
1403         if (!obj_priv->mmap_offset) {
1404                 ret = i915_gem_create_mmap_offset(obj);
1405                 if (ret) {
1406                         drm_gem_object_unreference(obj);
1407                         mutex_unlock(&dev->struct_mutex);
1408                         return ret;
1409                 }
1410         }
1411
1412         args->offset = obj_priv->mmap_offset;
1413
1414         /*
1415          * Pull it into the GTT so that we have a page list (makes the
1416          * initial fault faster and any subsequent flushing possible).
1417          */
1418         if (!obj_priv->agp_mem) {
1419                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1420                 if (ret) {
1421                         drm_gem_object_unreference(obj);
1422                         mutex_unlock(&dev->struct_mutex);
1423                         return ret;
1424                 }
1425         }
1426
1427         drm_gem_object_unreference(obj);
1428         mutex_unlock(&dev->struct_mutex);
1429
1430         return 0;
1431 }
1432
1433 void
1434 i915_gem_object_put_pages(struct drm_gem_object *obj)
1435 {
1436         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1437         int page_count = obj->size / PAGE_SIZE;
1438         int i;
1439
1440         BUG_ON(obj_priv->pages_refcount == 0);
1441         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1442
1443         if (--obj_priv->pages_refcount != 0)
1444                 return;
1445
1446         if (obj_priv->tiling_mode != I915_TILING_NONE)
1447                 i915_gem_object_save_bit_17_swizzle(obj);
1448
1449         if (obj_priv->madv == I915_MADV_DONTNEED)
1450                 obj_priv->dirty = 0;
1451
1452         for (i = 0; i < page_count; i++) {
1453                 if (obj_priv->dirty)
1454                         set_page_dirty(obj_priv->pages[i]);
1455
1456                 if (obj_priv->madv == I915_MADV_WILLNEED)
1457                         mark_page_accessed(obj_priv->pages[i]);
1458
1459                 page_cache_release(obj_priv->pages[i]);
1460         }
1461         obj_priv->dirty = 0;
1462
1463         drm_free_large(obj_priv->pages);
1464         obj_priv->pages = NULL;
1465 }
1466
1467 static void
1468 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1469                                struct intel_ring_buffer *ring)
1470 {
1471         struct drm_device *dev = obj->dev;
1472         drm_i915_private_t *dev_priv = dev->dev_private;
1473         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1474         BUG_ON(ring == NULL);
1475         obj_priv->ring = ring;
1476
1477         /* Add a reference if we're newly entering the active list. */
1478         if (!obj_priv->active) {
1479                 drm_gem_object_reference(obj);
1480                 obj_priv->active = 1;
1481         }
1482         /* Move from whatever list we were on to the tail of execution. */
1483         spin_lock(&dev_priv->mm.active_list_lock);
1484         list_move_tail(&obj_priv->list, &ring->active_list);
1485         spin_unlock(&dev_priv->mm.active_list_lock);
1486         obj_priv->last_rendering_seqno = seqno;
1487 }
1488
1489 static void
1490 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1491 {
1492         struct drm_device *dev = obj->dev;
1493         drm_i915_private_t *dev_priv = dev->dev_private;
1494         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1495
1496         BUG_ON(!obj_priv->active);
1497         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1498         obj_priv->last_rendering_seqno = 0;
1499 }
1500
1501 /* Immediately discard the backing storage */
1502 static void
1503 i915_gem_object_truncate(struct drm_gem_object *obj)
1504 {
1505         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1506         struct inode *inode;
1507
1508         /* Our goal here is to return as much of the memory as
1509          * is possible back to the system as we are called from OOM.
1510          * To do this we must instruct the shmfs to drop all of its
1511          * backing pages, *now*. Here we mirror the actions taken
1512          * when by shmem_delete_inode() to release the backing store.
1513          */
1514         inode = obj->filp->f_path.dentry->d_inode;
1515         truncate_inode_pages(inode->i_mapping, 0);
1516         if (inode->i_op->truncate_range)
1517                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1518
1519         obj_priv->madv = __I915_MADV_PURGED;
1520 }
1521
1522 static inline int
1523 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1524 {
1525         return obj_priv->madv == I915_MADV_DONTNEED;
1526 }
1527
1528 static void
1529 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1530 {
1531         struct drm_device *dev = obj->dev;
1532         drm_i915_private_t *dev_priv = dev->dev_private;
1533         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1534
1535         i915_verify_inactive(dev, __FILE__, __LINE__);
1536         if (obj_priv->pin_count != 0)
1537                 list_del_init(&obj_priv->list);
1538         else
1539                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1540
1541         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1542
1543         obj_priv->last_rendering_seqno = 0;
1544         obj_priv->ring = NULL;
1545         if (obj_priv->active) {
1546                 obj_priv->active = 0;
1547                 drm_gem_object_unreference(obj);
1548         }
1549         i915_verify_inactive(dev, __FILE__, __LINE__);
1550 }
1551
1552 static void
1553 i915_gem_process_flushing_list(struct drm_device *dev,
1554                                uint32_t flush_domains, uint32_t seqno,
1555                                struct intel_ring_buffer *ring)
1556 {
1557         drm_i915_private_t *dev_priv = dev->dev_private;
1558         struct drm_i915_gem_object *obj_priv, *next;
1559
1560         list_for_each_entry_safe(obj_priv, next,
1561                                  &dev_priv->mm.gpu_write_list,
1562                                  gpu_write_list) {
1563                 struct drm_gem_object *obj = &obj_priv->base;
1564
1565                 if ((obj->write_domain & flush_domains) ==
1566                     obj->write_domain &&
1567                     obj_priv->ring->ring_flag == ring->ring_flag) {
1568                         uint32_t old_write_domain = obj->write_domain;
1569
1570                         obj->write_domain = 0;
1571                         list_del_init(&obj_priv->gpu_write_list);
1572                         i915_gem_object_move_to_active(obj, seqno, ring);
1573
1574                         /* update the fence lru list */
1575                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1576                                 struct drm_i915_fence_reg *reg =
1577                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1578                                 list_move_tail(&reg->lru_list,
1579                                                 &dev_priv->mm.fence_list);
1580                         }
1581
1582                         trace_i915_gem_object_change_domain(obj,
1583                                                             obj->read_domains,
1584                                                             old_write_domain);
1585                 }
1586         }
1587 }
1588
1589 uint32_t
1590 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1591                  uint32_t flush_domains, struct intel_ring_buffer *ring)
1592 {
1593         drm_i915_private_t *dev_priv = dev->dev_private;
1594         struct drm_i915_file_private *i915_file_priv = NULL;
1595         struct drm_i915_gem_request *request;
1596         uint32_t seqno;
1597         int was_empty;
1598
1599         if (file_priv != NULL)
1600                 i915_file_priv = file_priv->driver_priv;
1601
1602         request = kzalloc(sizeof(*request), GFP_KERNEL);
1603         if (request == NULL)
1604                 return 0;
1605
1606         seqno = ring->add_request(dev, ring, file_priv, flush_domains);
1607
1608         request->seqno = seqno;
1609         request->ring = ring;
1610         request->emitted_jiffies = jiffies;
1611         was_empty = list_empty(&ring->request_list);
1612         list_add_tail(&request->list, &ring->request_list);
1613
1614         if (i915_file_priv) {
1615                 list_add_tail(&request->client_list,
1616                               &i915_file_priv->mm.request_list);
1617         } else {
1618                 INIT_LIST_HEAD(&request->client_list);
1619         }
1620
1621         /* Associate any objects on the flushing list matching the write
1622          * domain we're flushing with our flush.
1623          */
1624         if (flush_domains != 0)
1625                 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
1626
1627         if (!dev_priv->mm.suspended) {
1628                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1629                 if (was_empty)
1630                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1631         }
1632         return seqno;
1633 }
1634
1635 /**
1636  * Command execution barrier
1637  *
1638  * Ensures that all commands in the ring are finished
1639  * before signalling the CPU
1640  */
1641 static uint32_t
1642 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1643 {
1644         uint32_t flush_domains = 0;
1645
1646         /* The sampler always gets flushed on i965 (sigh) */
1647         if (IS_I965G(dev))
1648                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1649
1650         ring->flush(dev, ring,
1651                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1652         return flush_domains;
1653 }
1654
1655 /**
1656  * Moves buffers associated only with the given active seqno from the active
1657  * to inactive list, potentially freeing them.
1658  */
1659 static void
1660 i915_gem_retire_request(struct drm_device *dev,
1661                         struct drm_i915_gem_request *request)
1662 {
1663         drm_i915_private_t *dev_priv = dev->dev_private;
1664
1665         trace_i915_gem_request_retire(dev, request->seqno);
1666
1667         /* Move any buffers on the active list that are no longer referenced
1668          * by the ringbuffer to the flushing/inactive lists as appropriate.
1669          */
1670         spin_lock(&dev_priv->mm.active_list_lock);
1671         while (!list_empty(&request->ring->active_list)) {
1672                 struct drm_gem_object *obj;
1673                 struct drm_i915_gem_object *obj_priv;
1674
1675                 obj_priv = list_first_entry(&request->ring->active_list,
1676                                             struct drm_i915_gem_object,
1677                                             list);
1678                 obj = &obj_priv->base;
1679
1680                 /* If the seqno being retired doesn't match the oldest in the
1681                  * list, then the oldest in the list must still be newer than
1682                  * this seqno.
1683                  */
1684                 if (obj_priv->last_rendering_seqno != request->seqno)
1685                         goto out;
1686
1687 #if WATCH_LRU
1688                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1689                          __func__, request->seqno, obj);
1690 #endif
1691
1692                 if (obj->write_domain != 0)
1693                         i915_gem_object_move_to_flushing(obj);
1694                 else {
1695                         /* Take a reference on the object so it won't be
1696                          * freed while the spinlock is held.  The list
1697                          * protection for this spinlock is safe when breaking
1698                          * the lock like this since the next thing we do
1699                          * is just get the head of the list again.
1700                          */
1701                         drm_gem_object_reference(obj);
1702                         i915_gem_object_move_to_inactive(obj);
1703                         spin_unlock(&dev_priv->mm.active_list_lock);
1704                         drm_gem_object_unreference(obj);
1705                         spin_lock(&dev_priv->mm.active_list_lock);
1706                 }
1707         }
1708 out:
1709         spin_unlock(&dev_priv->mm.active_list_lock);
1710 }
1711
1712 /**
1713  * Returns true if seq1 is later than seq2.
1714  */
1715 bool
1716 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1717 {
1718         return (int32_t)(seq1 - seq2) >= 0;
1719 }
1720
1721 uint32_t
1722 i915_get_gem_seqno(struct drm_device *dev,
1723                    struct intel_ring_buffer *ring)
1724 {
1725         return ring->get_gem_seqno(dev, ring);
1726 }
1727
1728 /**
1729  * This function clears the request list as sequence numbers are passed.
1730  */
1731 static void
1732 i915_gem_retire_requests_ring(struct drm_device *dev,
1733                               struct intel_ring_buffer *ring)
1734 {
1735         drm_i915_private_t *dev_priv = dev->dev_private;
1736         uint32_t seqno;
1737
1738         if (!ring->status_page.page_addr
1739                         || list_empty(&ring->request_list))
1740                 return;
1741
1742         seqno = i915_get_gem_seqno(dev, ring);
1743
1744         while (!list_empty(&ring->request_list)) {
1745                 struct drm_i915_gem_request *request;
1746                 uint32_t retiring_seqno;
1747
1748                 request = list_first_entry(&ring->request_list,
1749                                            struct drm_i915_gem_request,
1750                                            list);
1751                 retiring_seqno = request->seqno;
1752
1753                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1754                     atomic_read(&dev_priv->mm.wedged)) {
1755                         i915_gem_retire_request(dev, request);
1756
1757                         list_del(&request->list);
1758                         list_del(&request->client_list);
1759                         kfree(request);
1760                 } else
1761                         break;
1762         }
1763
1764         if (unlikely (dev_priv->trace_irq_seqno &&
1765                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1766
1767                 ring->user_irq_put(dev, ring);
1768                 dev_priv->trace_irq_seqno = 0;
1769         }
1770 }
1771
1772 void
1773 i915_gem_retire_requests(struct drm_device *dev)
1774 {
1775         drm_i915_private_t *dev_priv = dev->dev_private;
1776
1777         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1778             struct drm_i915_gem_object *obj_priv, *tmp;
1779
1780             /* We must be careful that during unbind() we do not
1781              * accidentally infinitely recurse into retire requests.
1782              * Currently:
1783              *   retire -> free -> unbind -> wait -> retire_ring
1784              */
1785             list_for_each_entry_safe(obj_priv, tmp,
1786                                      &dev_priv->mm.deferred_free_list,
1787                                      list)
1788                     i915_gem_free_object_tail(&obj_priv->base);
1789         }
1790
1791         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1792         if (HAS_BSD(dev))
1793                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1794 }
1795
1796 void
1797 i915_gem_retire_work_handler(struct work_struct *work)
1798 {
1799         drm_i915_private_t *dev_priv;
1800         struct drm_device *dev;
1801
1802         dev_priv = container_of(work, drm_i915_private_t,
1803                                 mm.retire_work.work);
1804         dev = dev_priv->dev;
1805
1806         mutex_lock(&dev->struct_mutex);
1807         i915_gem_retire_requests(dev);
1808
1809         if (!dev_priv->mm.suspended &&
1810                 (!list_empty(&dev_priv->render_ring.request_list) ||
1811                         (HAS_BSD(dev) &&
1812                          !list_empty(&dev_priv->bsd_ring.request_list))))
1813                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1814         mutex_unlock(&dev->struct_mutex);
1815 }
1816
1817 int
1818 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1819                 int interruptible, struct intel_ring_buffer *ring)
1820 {
1821         drm_i915_private_t *dev_priv = dev->dev_private;
1822         u32 ier;
1823         int ret = 0;
1824
1825         BUG_ON(seqno == 0);
1826
1827         if (atomic_read(&dev_priv->mm.wedged))
1828                 return -EIO;
1829
1830         if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
1831                 if (HAS_PCH_SPLIT(dev))
1832                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1833                 else
1834                         ier = I915_READ(IER);
1835                 if (!ier) {
1836                         DRM_ERROR("something (likely vbetool) disabled "
1837                                   "interrupts, re-enabling\n");
1838                         i915_driver_irq_preinstall(dev);
1839                         i915_driver_irq_postinstall(dev);
1840                 }
1841
1842                 trace_i915_gem_request_wait_begin(dev, seqno);
1843
1844                 ring->waiting_gem_seqno = seqno;
1845                 ring->user_irq_get(dev, ring);
1846                 if (interruptible)
1847                         ret = wait_event_interruptible(ring->irq_queue,
1848                                 i915_seqno_passed(
1849                                         ring->get_gem_seqno(dev, ring), seqno)
1850                                 || atomic_read(&dev_priv->mm.wedged));
1851                 else
1852                         wait_event(ring->irq_queue,
1853                                 i915_seqno_passed(
1854                                         ring->get_gem_seqno(dev, ring), seqno)
1855                                 || atomic_read(&dev_priv->mm.wedged));
1856
1857                 ring->user_irq_put(dev, ring);
1858                 ring->waiting_gem_seqno = 0;
1859
1860                 trace_i915_gem_request_wait_end(dev, seqno);
1861         }
1862         if (atomic_read(&dev_priv->mm.wedged))
1863                 ret = -EIO;
1864
1865         if (ret && ret != -ERESTARTSYS)
1866                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1867                           __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
1868
1869         /* Directly dispatch request retiring.  While we have the work queue
1870          * to handle this, the waiter on a request often wants an associated
1871          * buffer to have made it to the inactive list, and we would need
1872          * a separate wait queue to handle that.
1873          */
1874         if (ret == 0)
1875                 i915_gem_retire_requests_ring(dev, ring);
1876
1877         return ret;
1878 }
1879
1880 /**
1881  * Waits for a sequence number to be signaled, and cleans up the
1882  * request and object lists appropriately for that event.
1883  */
1884 static int
1885 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1886                 struct intel_ring_buffer *ring)
1887 {
1888         return i915_do_wait_request(dev, seqno, 1, ring);
1889 }
1890
1891 static void
1892 i915_gem_flush(struct drm_device *dev,
1893                uint32_t invalidate_domains,
1894                uint32_t flush_domains)
1895 {
1896         drm_i915_private_t *dev_priv = dev->dev_private;
1897         if (flush_domains & I915_GEM_DOMAIN_CPU)
1898                 drm_agp_chipset_flush(dev);
1899         dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1900                         invalidate_domains,
1901                         flush_domains);
1902
1903         if (HAS_BSD(dev))
1904                 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1905                                 invalidate_domains,
1906                                 flush_domains);
1907 }
1908
1909 /**
1910  * Ensures that all rendering to the object has completed and the object is
1911  * safe to unbind from the GTT or access from the CPU.
1912  */
1913 static int
1914 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1915 {
1916         struct drm_device *dev = obj->dev;
1917         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1918         int ret;
1919
1920         /* This function only exists to support waiting for existing rendering,
1921          * not for emitting required flushes.
1922          */
1923         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1924
1925         /* If there is rendering queued on the buffer being evicted, wait for
1926          * it.
1927          */
1928         if (obj_priv->active) {
1929 #if WATCH_BUF
1930                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1931                           __func__, obj, obj_priv->last_rendering_seqno);
1932 #endif
1933                 ret = i915_wait_request(dev,
1934                                 obj_priv->last_rendering_seqno, obj_priv->ring);
1935                 if (ret != 0)
1936                         return ret;
1937         }
1938
1939         return 0;
1940 }
1941
1942 /**
1943  * Unbinds an object from the GTT aperture.
1944  */
1945 int
1946 i915_gem_object_unbind(struct drm_gem_object *obj)
1947 {
1948         struct drm_device *dev = obj->dev;
1949         drm_i915_private_t *dev_priv = dev->dev_private;
1950         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1951         int ret = 0;
1952
1953 #if WATCH_BUF
1954         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1955         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1956 #endif
1957         if (obj_priv->gtt_space == NULL)
1958                 return 0;
1959
1960         if (obj_priv->pin_count != 0) {
1961                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1962                 return -EINVAL;
1963         }
1964
1965         /* blow away mappings if mapped through GTT */
1966         i915_gem_release_mmap(obj);
1967
1968         /* Move the object to the CPU domain to ensure that
1969          * any possible CPU writes while it's not in the GTT
1970          * are flushed when we go to remap it. This will
1971          * also ensure that all pending GPU writes are finished
1972          * before we unbind.
1973          */
1974         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1975         if (ret == -ERESTARTSYS)
1976                 return ret;
1977         /* Continue on if we fail due to EIO, the GPU is hung so we
1978          * should be safe and we need to cleanup or else we might
1979          * cause memory corruption through use-after-free.
1980          */
1981
1982         /* release the fence reg _after_ flushing */
1983         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1984                 i915_gem_clear_fence_reg(obj);
1985
1986         if (obj_priv->agp_mem != NULL) {
1987                 drm_unbind_agp(obj_priv->agp_mem);
1988                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1989                 obj_priv->agp_mem = NULL;
1990         }
1991
1992         i915_gem_object_put_pages(obj);
1993         BUG_ON(obj_priv->pages_refcount);
1994
1995         if (obj_priv->gtt_space) {
1996                 atomic_dec(&dev->gtt_count);
1997                 atomic_sub(obj->size, &dev->gtt_memory);
1998
1999                 drm_mm_put_block(obj_priv->gtt_space);
2000                 obj_priv->gtt_space = NULL;
2001         }
2002
2003         /* Remove ourselves from the LRU list if present. */
2004         spin_lock(&dev_priv->mm.active_list_lock);
2005         if (!list_empty(&obj_priv->list))
2006                 list_del_init(&obj_priv->list);
2007         spin_unlock(&dev_priv->mm.active_list_lock);
2008
2009         if (i915_gem_object_is_purgeable(obj_priv))
2010                 i915_gem_object_truncate(obj);
2011
2012         trace_i915_gem_object_unbind(obj);
2013
2014         return ret;
2015 }
2016
2017 int
2018 i915_gpu_idle(struct drm_device *dev)
2019 {
2020         drm_i915_private_t *dev_priv = dev->dev_private;
2021         bool lists_empty;
2022         uint32_t seqno1, seqno2;
2023         int ret;
2024
2025         spin_lock(&dev_priv->mm.active_list_lock);
2026         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2027                        list_empty(&dev_priv->render_ring.active_list) &&
2028                        (!HAS_BSD(dev) ||
2029                         list_empty(&dev_priv->bsd_ring.active_list)));
2030         spin_unlock(&dev_priv->mm.active_list_lock);
2031
2032         if (lists_empty)
2033                 return 0;
2034
2035         /* Flush everything onto the inactive list. */
2036         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2037         seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2038                         &dev_priv->render_ring);
2039         if (seqno1 == 0)
2040                 return -ENOMEM;
2041         ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2042
2043         if (HAS_BSD(dev)) {
2044                 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2045                                 &dev_priv->bsd_ring);
2046                 if (seqno2 == 0)
2047                         return -ENOMEM;
2048
2049                 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2050                 if (ret)
2051                         return ret;
2052         }
2053
2054
2055         return ret;
2056 }
2057
2058 int
2059 i915_gem_object_get_pages(struct drm_gem_object *obj,
2060                           gfp_t gfpmask)
2061 {
2062         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2063         int page_count, i;
2064         struct address_space *mapping;
2065         struct inode *inode;
2066         struct page *page;
2067
2068         BUG_ON(obj_priv->pages_refcount
2069                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2070
2071         if (obj_priv->pages_refcount++ != 0)
2072                 return 0;
2073
2074         /* Get the list of pages out of our struct file.  They'll be pinned
2075          * at this point until we release them.
2076          */
2077         page_count = obj->size / PAGE_SIZE;
2078         BUG_ON(obj_priv->pages != NULL);
2079         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2080         if (obj_priv->pages == NULL) {
2081                 obj_priv->pages_refcount--;
2082                 return -ENOMEM;
2083         }
2084
2085         inode = obj->filp->f_path.dentry->d_inode;
2086         mapping = inode->i_mapping;
2087         for (i = 0; i < page_count; i++) {
2088                 page = read_cache_page_gfp(mapping, i,
2089                                            GFP_HIGHUSER |
2090                                            __GFP_COLD |
2091                                            __GFP_RECLAIMABLE |
2092                                            gfpmask);
2093                 if (IS_ERR(page))
2094                         goto err_pages;
2095
2096                 obj_priv->pages[i] = page;
2097         }
2098
2099         if (obj_priv->tiling_mode != I915_TILING_NONE)
2100                 i915_gem_object_do_bit_17_swizzle(obj);
2101
2102         return 0;
2103
2104 err_pages:
2105         while (i--)
2106                 page_cache_release(obj_priv->pages[i]);
2107
2108         drm_free_large(obj_priv->pages);
2109         obj_priv->pages = NULL;
2110         obj_priv->pages_refcount--;
2111         return PTR_ERR(page);
2112 }
2113
2114 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2115 {
2116         struct drm_gem_object *obj = reg->obj;
2117         struct drm_device *dev = obj->dev;
2118         drm_i915_private_t *dev_priv = dev->dev_private;
2119         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2120         int regnum = obj_priv->fence_reg;
2121         uint64_t val;
2122
2123         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2124                     0xfffff000) << 32;
2125         val |= obj_priv->gtt_offset & 0xfffff000;
2126         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2127                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2128
2129         if (obj_priv->tiling_mode == I915_TILING_Y)
2130                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2131         val |= I965_FENCE_REG_VALID;
2132
2133         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2134 }
2135
2136 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2137 {
2138         struct drm_gem_object *obj = reg->obj;
2139         struct drm_device *dev = obj->dev;
2140         drm_i915_private_t *dev_priv = dev->dev_private;
2141         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2142         int regnum = obj_priv->fence_reg;
2143         uint64_t val;
2144
2145         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2146                     0xfffff000) << 32;
2147         val |= obj_priv->gtt_offset & 0xfffff000;
2148         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2149         if (obj_priv->tiling_mode == I915_TILING_Y)
2150                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2151         val |= I965_FENCE_REG_VALID;
2152
2153         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2154 }
2155
2156 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2157 {
2158         struct drm_gem_object *obj = reg->obj;
2159         struct drm_device *dev = obj->dev;
2160         drm_i915_private_t *dev_priv = dev->dev_private;
2161         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2162         int regnum = obj_priv->fence_reg;
2163         int tile_width;
2164         uint32_t fence_reg, val;
2165         uint32_t pitch_val;
2166
2167         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2168             (obj_priv->gtt_offset & (obj->size - 1))) {
2169                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2170                      __func__, obj_priv->gtt_offset, obj->size);
2171                 return;
2172         }
2173
2174         if (obj_priv->tiling_mode == I915_TILING_Y &&
2175             HAS_128_BYTE_Y_TILING(dev))
2176                 tile_width = 128;
2177         else
2178                 tile_width = 512;
2179
2180         /* Note: pitch better be a power of two tile widths */
2181         pitch_val = obj_priv->stride / tile_width;
2182         pitch_val = ffs(pitch_val) - 1;
2183
2184         if (obj_priv->tiling_mode == I915_TILING_Y &&
2185             HAS_128_BYTE_Y_TILING(dev))
2186                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2187         else
2188                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2189
2190         val = obj_priv->gtt_offset;
2191         if (obj_priv->tiling_mode == I915_TILING_Y)
2192                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2193         val |= I915_FENCE_SIZE_BITS(obj->size);
2194         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2195         val |= I830_FENCE_REG_VALID;
2196
2197         if (regnum < 8)
2198                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2199         else
2200                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2201         I915_WRITE(fence_reg, val);
2202 }
2203
2204 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2205 {
2206         struct drm_gem_object *obj = reg->obj;
2207         struct drm_device *dev = obj->dev;
2208         drm_i915_private_t *dev_priv = dev->dev_private;
2209         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2210         int regnum = obj_priv->fence_reg;
2211         uint32_t val;
2212         uint32_t pitch_val;
2213         uint32_t fence_size_bits;
2214
2215         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2216             (obj_priv->gtt_offset & (obj->size - 1))) {
2217                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2218                      __func__, obj_priv->gtt_offset);
2219                 return;
2220         }
2221
2222         pitch_val = obj_priv->stride / 128;
2223         pitch_val = ffs(pitch_val) - 1;
2224         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2225
2226         val = obj_priv->gtt_offset;
2227         if (obj_priv->tiling_mode == I915_TILING_Y)
2228                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2229         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2230         WARN_ON(fence_size_bits & ~0x00000f00);
2231         val |= fence_size_bits;
2232         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2233         val |= I830_FENCE_REG_VALID;
2234
2235         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2236 }
2237
2238 static int i915_find_fence_reg(struct drm_device *dev)
2239 {
2240         struct drm_i915_fence_reg *reg = NULL;
2241         struct drm_i915_gem_object *obj_priv = NULL;
2242         struct drm_i915_private *dev_priv = dev->dev_private;
2243         struct drm_gem_object *obj = NULL;
2244         int i, avail, ret;
2245
2246         /* First try to find a free reg */
2247         avail = 0;
2248         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2249                 reg = &dev_priv->fence_regs[i];
2250                 if (!reg->obj)
2251                         return i;
2252
2253                 obj_priv = to_intel_bo(reg->obj);
2254                 if (!obj_priv->pin_count)
2255                     avail++;
2256         }
2257
2258         if (avail == 0)
2259                 return -ENOSPC;
2260
2261         /* None available, try to steal one or wait for a user to finish */
2262         i = I915_FENCE_REG_NONE;
2263         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2264                             lru_list) {
2265                 obj = reg->obj;
2266                 obj_priv = to_intel_bo(obj);
2267
2268                 if (obj_priv->pin_count)
2269                         continue;
2270
2271                 /* found one! */
2272                 i = obj_priv->fence_reg;
2273                 break;
2274         }
2275
2276         BUG_ON(i == I915_FENCE_REG_NONE);
2277
2278         /* We only have a reference on obj from the active list. put_fence_reg
2279          * might drop that one, causing a use-after-free in it. So hold a
2280          * private reference to obj like the other callers of put_fence_reg
2281          * (set_tiling ioctl) do. */
2282         drm_gem_object_reference(obj);
2283         ret = i915_gem_object_put_fence_reg(obj);
2284         drm_gem_object_unreference(obj);
2285         if (ret != 0)
2286                 return ret;
2287
2288         return i;
2289 }
2290
2291 /**
2292  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2293  * @obj: object to map through a fence reg
2294  *
2295  * When mapping objects through the GTT, userspace wants to be able to write
2296  * to them without having to worry about swizzling if the object is tiled.
2297  *
2298  * This function walks the fence regs looking for a free one for @obj,
2299  * stealing one if it can't find any.
2300  *
2301  * It then sets up the reg based on the object's properties: address, pitch
2302  * and tiling format.
2303  */
2304 int
2305 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2306 {
2307         struct drm_device *dev = obj->dev;
2308         struct drm_i915_private *dev_priv = dev->dev_private;
2309         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2310         struct drm_i915_fence_reg *reg = NULL;
2311         int ret;
2312
2313         /* Just update our place in the LRU if our fence is getting used. */
2314         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2315                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2316                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2317                 return 0;
2318         }
2319
2320         switch (obj_priv->tiling_mode) {
2321         case I915_TILING_NONE:
2322                 WARN(1, "allocating a fence for non-tiled object?\n");
2323                 break;
2324         case I915_TILING_X:
2325                 if (!obj_priv->stride)
2326                         return -EINVAL;
2327                 WARN((obj_priv->stride & (512 - 1)),
2328                      "object 0x%08x is X tiled but has non-512B pitch\n",
2329                      obj_priv->gtt_offset);
2330                 break;
2331         case I915_TILING_Y:
2332                 if (!obj_priv->stride)
2333                         return -EINVAL;
2334                 WARN((obj_priv->stride & (128 - 1)),
2335                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2336                      obj_priv->gtt_offset);
2337                 break;
2338         }
2339
2340         ret = i915_find_fence_reg(dev);
2341         if (ret < 0)
2342                 return ret;
2343
2344         obj_priv->fence_reg = ret;
2345         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2346         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2347
2348         reg->obj = obj;
2349
2350         if (IS_GEN6(dev))
2351                 sandybridge_write_fence_reg(reg);
2352         else if (IS_I965G(dev))
2353                 i965_write_fence_reg(reg);
2354         else if (IS_I9XX(dev))
2355                 i915_write_fence_reg(reg);
2356         else
2357                 i830_write_fence_reg(reg);
2358
2359         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2360                         obj_priv->tiling_mode);
2361
2362         return 0;
2363 }
2364
2365 /**
2366  * i915_gem_clear_fence_reg - clear out fence register info
2367  * @obj: object to clear
2368  *
2369  * Zeroes out the fence register itself and clears out the associated
2370  * data structures in dev_priv and obj_priv.
2371  */
2372 static void
2373 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2374 {
2375         struct drm_device *dev = obj->dev;
2376         drm_i915_private_t *dev_priv = dev->dev_private;
2377         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2378         struct drm_i915_fence_reg *reg =
2379                 &dev_priv->fence_regs[obj_priv->fence_reg];
2380
2381         if (IS_GEN6(dev)) {
2382                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2383                              (obj_priv->fence_reg * 8), 0);
2384         } else if (IS_I965G(dev)) {
2385                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2386         } else {
2387                 uint32_t fence_reg;
2388
2389                 if (obj_priv->fence_reg < 8)
2390                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2391                 else
2392                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2393                                                        8) * 4;
2394
2395                 I915_WRITE(fence_reg, 0);
2396         }
2397
2398         reg->obj = NULL;
2399         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2400         list_del_init(&reg->lru_list);
2401 }
2402
2403 /**
2404  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2405  * to the buffer to finish, and then resets the fence register.
2406  * @obj: tiled object holding a fence register.
2407  *
2408  * Zeroes out the fence register itself and clears out the associated
2409  * data structures in dev_priv and obj_priv.
2410  */
2411 int
2412 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2413 {
2414         struct drm_device *dev = obj->dev;
2415         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2416
2417         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2418                 return 0;
2419
2420         /* If we've changed tiling, GTT-mappings of the object
2421          * need to re-fault to ensure that the correct fence register
2422          * setup is in place.
2423          */
2424         i915_gem_release_mmap(obj);
2425
2426         /* On the i915, GPU access to tiled buffers is via a fence,
2427          * therefore we must wait for any outstanding access to complete
2428          * before clearing the fence.
2429          */
2430         if (!IS_I965G(dev)) {
2431                 int ret;
2432
2433                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2434                 if (ret != 0)
2435                         return ret;
2436
2437                 ret = i915_gem_object_wait_rendering(obj);
2438                 if (ret != 0)
2439                         return ret;
2440         }
2441
2442         i915_gem_object_flush_gtt_write_domain(obj);
2443         i915_gem_clear_fence_reg (obj);
2444
2445         return 0;
2446 }
2447
2448 /**
2449  * Finds free space in the GTT aperture and binds the object there.
2450  */
2451 static int
2452 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2453 {
2454         struct drm_device *dev = obj->dev;
2455         drm_i915_private_t *dev_priv = dev->dev_private;
2456         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2457         struct drm_mm_node *free_space;
2458         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2459         int ret;
2460
2461         if (obj_priv->madv != I915_MADV_WILLNEED) {
2462                 DRM_ERROR("Attempting to bind a purgeable object\n");
2463                 return -EINVAL;
2464         }
2465
2466         if (alignment == 0)
2467                 alignment = i915_gem_get_gtt_alignment(obj);
2468         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2469                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2470                 return -EINVAL;
2471         }
2472
2473         /* If the object is bigger than the entire aperture, reject it early
2474          * before evicting everything in a vain attempt to find space.
2475          */
2476         if (obj->size > dev->gtt_total) {
2477                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2478                 return -E2BIG;
2479         }
2480
2481  search_free:
2482         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2483                                         obj->size, alignment, 0);
2484         if (free_space != NULL) {
2485                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2486                                                        alignment);
2487                 if (obj_priv->gtt_space != NULL)
2488                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2489         }
2490         if (obj_priv->gtt_space == NULL) {
2491                 /* If the gtt is empty and we're still having trouble
2492                  * fitting our object in, we're out of memory.
2493                  */
2494 #if WATCH_LRU
2495                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2496 #endif
2497                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2498                 if (ret)
2499                         return ret;
2500
2501                 goto search_free;
2502         }
2503
2504 #if WATCH_BUF
2505         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2506                  obj->size, obj_priv->gtt_offset);
2507 #endif
2508         ret = i915_gem_object_get_pages(obj, gfpmask);
2509         if (ret) {
2510                 drm_mm_put_block(obj_priv->gtt_space);
2511                 obj_priv->gtt_space = NULL;
2512
2513                 if (ret == -ENOMEM) {
2514                         /* first try to clear up some space from the GTT */
2515                         ret = i915_gem_evict_something(dev, obj->size,
2516                                                        alignment);
2517                         if (ret) {
2518                                 /* now try to shrink everyone else */
2519                                 if (gfpmask) {
2520                                         gfpmask = 0;
2521                                         goto search_free;
2522                                 }
2523
2524                                 return ret;
2525                         }
2526
2527                         goto search_free;
2528                 }
2529
2530                 return ret;
2531         }
2532
2533         /* Create an AGP memory structure pointing at our pages, and bind it
2534          * into the GTT.
2535          */
2536         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2537                                                obj_priv->pages,
2538                                                obj->size >> PAGE_SHIFT,
2539                                                obj_priv->gtt_offset,
2540                                                obj_priv->agp_type);
2541         if (obj_priv->agp_mem == NULL) {
2542                 i915_gem_object_put_pages(obj);
2543                 drm_mm_put_block(obj_priv->gtt_space);
2544                 obj_priv->gtt_space = NULL;
2545
2546                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2547                 if (ret)
2548                         return ret;
2549
2550                 goto search_free;
2551         }
2552         atomic_inc(&dev->gtt_count);
2553         atomic_add(obj->size, &dev->gtt_memory);
2554
2555         /* keep track of bounds object by adding it to the inactive list */
2556         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2557
2558         /* Assert that the object is not currently in any GPU domain. As it
2559          * wasn't in the GTT, there shouldn't be any way it could have been in
2560          * a GPU cache
2561          */
2562         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2563         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2564
2565         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2566
2567         return 0;
2568 }
2569
2570 void
2571 i915_gem_clflush_object(struct drm_gem_object *obj)
2572 {
2573         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2574
2575         /* If we don't have a page list set up, then we're not pinned
2576          * to GPU, and we can ignore the cache flush because it'll happen
2577          * again at bind time.
2578          */
2579         if (obj_priv->pages == NULL)
2580                 return;
2581
2582         trace_i915_gem_object_clflush(obj);
2583
2584         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2585 }
2586
2587 /** Flushes any GPU write domain for the object if it's dirty. */
2588 static int
2589 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2590 {
2591         struct drm_device *dev = obj->dev;
2592         uint32_t old_write_domain;
2593         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2594
2595         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2596                 return 0;
2597
2598         /* Queue the GPU write cache flushing we need. */
2599         old_write_domain = obj->write_domain;
2600         i915_gem_flush(dev, 0, obj->write_domain);
2601         if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
2602                 return -ENOMEM;
2603
2604         trace_i915_gem_object_change_domain(obj,
2605                                             obj->read_domains,
2606                                             old_write_domain);
2607         return 0;
2608 }
2609
2610 /** Flushes the GTT write domain for the object if it's dirty. */
2611 static void
2612 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2613 {
2614         uint32_t old_write_domain;
2615
2616         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2617                 return;
2618
2619         /* No actual flushing is required for the GTT write domain.   Writes
2620          * to it immediately go to main memory as far as we know, so there's
2621          * no chipset flush.  It also doesn't land in render cache.
2622          */
2623         old_write_domain = obj->write_domain;
2624         obj->write_domain = 0;
2625
2626         trace_i915_gem_object_change_domain(obj,
2627                                             obj->read_domains,
2628                                             old_write_domain);
2629 }
2630
2631 /** Flushes the CPU write domain for the object if it's dirty. */
2632 static void
2633 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2634 {
2635         struct drm_device *dev = obj->dev;
2636         uint32_t old_write_domain;
2637
2638         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2639                 return;
2640
2641         i915_gem_clflush_object(obj);
2642         drm_agp_chipset_flush(dev);
2643         old_write_domain = obj->write_domain;
2644         obj->write_domain = 0;
2645
2646         trace_i915_gem_object_change_domain(obj,
2647                                             obj->read_domains,
2648                                             old_write_domain);
2649 }
2650
2651 int
2652 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2653 {
2654         int ret = 0;
2655
2656         switch (obj->write_domain) {
2657         case I915_GEM_DOMAIN_GTT:
2658                 i915_gem_object_flush_gtt_write_domain(obj);
2659                 break;
2660         case I915_GEM_DOMAIN_CPU:
2661                 i915_gem_object_flush_cpu_write_domain(obj);
2662                 break;
2663         default:
2664                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2665                 break;
2666         }
2667
2668         return ret;
2669 }
2670
2671 /**
2672  * Moves a single object to the GTT read, and possibly write domain.
2673  *
2674  * This function returns when the move is complete, including waiting on
2675  * flushes to occur.
2676  */
2677 int
2678 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2679 {
2680         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2681         uint32_t old_write_domain, old_read_domains;
2682         int ret;
2683
2684         /* Not valid to be called on unbound objects. */
2685         if (obj_priv->gtt_space == NULL)
2686                 return -EINVAL;
2687
2688         ret = i915_gem_object_flush_gpu_write_domain(obj);
2689         if (ret != 0)
2690                 return ret;
2691
2692         /* Wait on any GPU rendering and flushing to occur. */
2693         ret = i915_gem_object_wait_rendering(obj);
2694         if (ret != 0)
2695                 return ret;
2696
2697         old_write_domain = obj->write_domain;
2698         old_read_domains = obj->read_domains;
2699
2700         /* If we're writing through the GTT domain, then CPU and GPU caches
2701          * will need to be invalidated at next use.
2702          */
2703         if (write)
2704                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2705
2706         i915_gem_object_flush_cpu_write_domain(obj);
2707
2708         /* It should now be out of any other write domains, and we can update
2709          * the domain values for our changes.
2710          */
2711         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2712         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2713         if (write) {
2714                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2715                 obj_priv->dirty = 1;
2716         }
2717
2718         trace_i915_gem_object_change_domain(obj,
2719                                             old_read_domains,
2720                                             old_write_domain);
2721
2722         return 0;
2723 }
2724
2725 /*
2726  * Prepare buffer for display plane. Use uninterruptible for possible flush
2727  * wait, as in modesetting process we're not supposed to be interrupted.
2728  */
2729 int
2730 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2731 {
2732         struct drm_device *dev = obj->dev;
2733         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2734         uint32_t old_write_domain, old_read_domains;
2735         int ret;
2736
2737         /* Not valid to be called on unbound objects. */
2738         if (obj_priv->gtt_space == NULL)
2739                 return -EINVAL;
2740
2741         ret = i915_gem_object_flush_gpu_write_domain(obj);
2742         if (ret)
2743                 return ret;
2744
2745         /* Wait on any GPU rendering and flushing to occur. */
2746         if (obj_priv->active) {
2747 #if WATCH_BUF
2748                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2749                           __func__, obj, obj_priv->last_rendering_seqno);
2750 #endif
2751                 ret = i915_do_wait_request(dev,
2752                                 obj_priv->last_rendering_seqno,
2753                                 0,
2754                                 obj_priv->ring);
2755                 if (ret != 0)
2756                         return ret;
2757         }
2758
2759         i915_gem_object_flush_cpu_write_domain(obj);
2760
2761         old_write_domain = obj->write_domain;
2762         old_read_domains = obj->read_domains;
2763
2764         /* It should now be out of any other write domains, and we can update
2765          * the domain values for our changes.
2766          */
2767         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2768         obj->read_domains = I915_GEM_DOMAIN_GTT;
2769         obj->write_domain = I915_GEM_DOMAIN_GTT;
2770         obj_priv->dirty = 1;
2771
2772         trace_i915_gem_object_change_domain(obj,
2773                                             old_read_domains,
2774                                             old_write_domain);
2775
2776         return 0;
2777 }
2778
2779 /**
2780  * Moves a single object to the CPU read, and possibly write domain.
2781  *
2782  * This function returns when the move is complete, including waiting on
2783  * flushes to occur.
2784  */
2785 static int
2786 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2787 {
2788         uint32_t old_write_domain, old_read_domains;
2789         int ret;
2790
2791         ret = i915_gem_object_flush_gpu_write_domain(obj);
2792         if (ret)
2793                 return ret;
2794
2795         /* Wait on any GPU rendering and flushing to occur. */
2796         ret = i915_gem_object_wait_rendering(obj);
2797         if (ret != 0)
2798                 return ret;
2799
2800         i915_gem_object_flush_gtt_write_domain(obj);
2801
2802         /* If we have a partially-valid cache of the object in the CPU,
2803          * finish invalidating it and free the per-page flags.
2804          */
2805         i915_gem_object_set_to_full_cpu_read_domain(obj);
2806
2807         old_write_domain = obj->write_domain;
2808         old_read_domains = obj->read_domains;
2809
2810         /* Flush the CPU cache if it's still invalid. */
2811         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2812                 i915_gem_clflush_object(obj);
2813
2814                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2815         }
2816
2817         /* It should now be out of any other write domains, and we can update
2818          * the domain values for our changes.
2819          */
2820         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2821
2822         /* If we're writing through the CPU, then the GPU read domains will
2823          * need to be invalidated at next use.
2824          */
2825         if (write) {
2826                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2827                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2828         }
2829
2830         trace_i915_gem_object_change_domain(obj,
2831                                             old_read_domains,
2832                                             old_write_domain);
2833
2834         return 0;
2835 }
2836
2837 /*
2838  * Set the next domain for the specified object. This
2839  * may not actually perform the necessary flushing/invaliding though,
2840  * as that may want to be batched with other set_domain operations
2841  *
2842  * This is (we hope) the only really tricky part of gem. The goal
2843  * is fairly simple -- track which caches hold bits of the object
2844  * and make sure they remain coherent. A few concrete examples may
2845  * help to explain how it works. For shorthand, we use the notation
2846  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2847  * a pair of read and write domain masks.
2848  *
2849  * Case 1: the batch buffer
2850  *
2851  *      1. Allocated
2852  *      2. Written by CPU
2853  *      3. Mapped to GTT
2854  *      4. Read by GPU
2855  *      5. Unmapped from GTT
2856  *      6. Freed
2857  *
2858  *      Let's take these a step at a time
2859  *
2860  *      1. Allocated
2861  *              Pages allocated from the kernel may still have
2862  *              cache contents, so we set them to (CPU, CPU) always.
2863  *      2. Written by CPU (using pwrite)
2864  *              The pwrite function calls set_domain (CPU, CPU) and
2865  *              this function does nothing (as nothing changes)
2866  *      3. Mapped by GTT
2867  *              This function asserts that the object is not
2868  *              currently in any GPU-based read or write domains
2869  *      4. Read by GPU
2870  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2871  *              As write_domain is zero, this function adds in the
2872  *              current read domains (CPU+COMMAND, 0).
2873  *              flush_domains is set to CPU.
2874  *              invalidate_domains is set to COMMAND
2875  *              clflush is run to get data out of the CPU caches
2876  *              then i915_dev_set_domain calls i915_gem_flush to
2877  *              emit an MI_FLUSH and drm_agp_chipset_flush
2878  *      5. Unmapped from GTT
2879  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2880  *              flush_domains and invalidate_domains end up both zero
2881  *              so no flushing/invalidating happens
2882  *      6. Freed
2883  *              yay, done
2884  *
2885  * Case 2: The shared render buffer
2886  *
2887  *      1. Allocated
2888  *      2. Mapped to GTT
2889  *      3. Read/written by GPU
2890  *      4. set_domain to (CPU,CPU)
2891  *      5. Read/written by CPU
2892  *      6. Read/written by GPU
2893  *
2894  *      1. Allocated
2895  *              Same as last example, (CPU, CPU)
2896  *      2. Mapped to GTT
2897  *              Nothing changes (assertions find that it is not in the GPU)
2898  *      3. Read/written by GPU
2899  *              execbuffer calls set_domain (RENDER, RENDER)
2900  *              flush_domains gets CPU
2901  *              invalidate_domains gets GPU
2902  *              clflush (obj)
2903  *              MI_FLUSH and drm_agp_chipset_flush
2904  *      4. set_domain (CPU, CPU)
2905  *              flush_domains gets GPU
2906  *              invalidate_domains gets CPU
2907  *              wait_rendering (obj) to make sure all drawing is complete.
2908  *              This will include an MI_FLUSH to get the data from GPU
2909  *              to memory
2910  *              clflush (obj) to invalidate the CPU cache
2911  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2912  *      5. Read/written by CPU
2913  *              cache lines are loaded and dirtied
2914  *      6. Read written by GPU
2915  *              Same as last GPU access
2916  *
2917  * Case 3: The constant buffer
2918  *
2919  *      1. Allocated
2920  *      2. Written by CPU
2921  *      3. Read by GPU
2922  *      4. Updated (written) by CPU again
2923  *      5. Read by GPU
2924  *
2925  *      1. Allocated
2926  *              (CPU, CPU)
2927  *      2. Written by CPU
2928  *              (CPU, CPU)
2929  *      3. Read by GPU
2930  *              (CPU+RENDER, 0)
2931  *              flush_domains = CPU
2932  *              invalidate_domains = RENDER
2933  *              clflush (obj)
2934  *              MI_FLUSH
2935  *              drm_agp_chipset_flush
2936  *      4. Updated (written) by CPU again
2937  *              (CPU, CPU)
2938  *              flush_domains = 0 (no previous write domain)
2939  *              invalidate_domains = 0 (no new read domains)
2940  *      5. Read by GPU
2941  *              (CPU+RENDER, 0)
2942  *              flush_domains = CPU
2943  *              invalidate_domains = RENDER
2944  *              clflush (obj)
2945  *              MI_FLUSH
2946  *              drm_agp_chipset_flush
2947  */
2948 static void
2949 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2950 {
2951         struct drm_device               *dev = obj->dev;
2952         drm_i915_private_t              *dev_priv = dev->dev_private;
2953         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2954         uint32_t                        invalidate_domains = 0;
2955         uint32_t                        flush_domains = 0;
2956         uint32_t                        old_read_domains;
2957
2958         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2959         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2960
2961         intel_mark_busy(dev, obj);
2962
2963 #if WATCH_BUF
2964         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2965                  __func__, obj,
2966                  obj->read_domains, obj->pending_read_domains,
2967                  obj->write_domain, obj->pending_write_domain);
2968 #endif
2969         /*
2970          * If the object isn't moving to a new write domain,
2971          * let the object stay in multiple read domains
2972          */
2973         if (obj->pending_write_domain == 0)
2974                 obj->pending_read_domains |= obj->read_domains;
2975         else
2976                 obj_priv->dirty = 1;
2977
2978         /*
2979          * Flush the current write domain if
2980          * the new read domains don't match. Invalidate
2981          * any read domains which differ from the old
2982          * write domain
2983          */
2984         if (obj->write_domain &&
2985             obj->write_domain != obj->pending_read_domains) {
2986                 flush_domains |= obj->write_domain;
2987                 invalidate_domains |=
2988                         obj->pending_read_domains & ~obj->write_domain;
2989         }
2990         /*
2991          * Invalidate any read caches which may have
2992          * stale data. That is, any new read domains.
2993          */
2994         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
2995         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
2996 #if WATCH_BUF
2997                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
2998                          __func__, flush_domains, invalidate_domains);
2999 #endif
3000                 i915_gem_clflush_object(obj);
3001         }
3002
3003         old_read_domains = obj->read_domains;
3004
3005         /* The actual obj->write_domain will be updated with
3006          * pending_write_domain after we emit the accumulated flush for all
3007          * of our domain changes in execbuffers (which clears objects'
3008          * write_domains).  So if we have a current write domain that we
3009          * aren't changing, set pending_write_domain to that.
3010          */
3011         if (flush_domains == 0 && obj->pending_write_domain == 0)
3012                 obj->pending_write_domain = obj->write_domain;
3013         obj->read_domains = obj->pending_read_domains;
3014
3015         if (flush_domains & I915_GEM_GPU_DOMAINS) {
3016                 if (obj_priv->ring == &dev_priv->render_ring)
3017                         dev_priv->flush_rings |= FLUSH_RENDER_RING;
3018                 else if (obj_priv->ring == &dev_priv->bsd_ring)
3019                         dev_priv->flush_rings |= FLUSH_BSD_RING;
3020         }
3021
3022         dev->invalidate_domains |= invalidate_domains;
3023         dev->flush_domains |= flush_domains;
3024 #if WATCH_BUF
3025         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3026                  __func__,
3027                  obj->read_domains, obj->write_domain,
3028                  dev->invalidate_domains, dev->flush_domains);
3029 #endif
3030
3031         trace_i915_gem_object_change_domain(obj,
3032                                             old_read_domains,
3033                                             obj->write_domain);
3034 }
3035
3036 /**
3037  * Moves the object from a partially CPU read to a full one.
3038  *
3039  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3040  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3041  */
3042 static void
3043 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3044 {
3045         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3046
3047         if (!obj_priv->page_cpu_valid)
3048                 return;
3049
3050         /* If we're partially in the CPU read domain, finish moving it in.
3051          */
3052         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3053                 int i;
3054
3055                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3056                         if (obj_priv->page_cpu_valid[i])
3057                                 continue;
3058                         drm_clflush_pages(obj_priv->pages + i, 1);
3059                 }
3060         }
3061
3062         /* Free the page_cpu_valid mappings which are now stale, whether
3063          * or not we've got I915_GEM_DOMAIN_CPU.
3064          */
3065         kfree(obj_priv->page_cpu_valid);
3066         obj_priv->page_cpu_valid = NULL;
3067 }
3068
3069 /**
3070  * Set the CPU read domain on a range of the object.
3071  *
3072  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3073  * not entirely valid.  The page_cpu_valid member of the object flags which
3074  * pages have been flushed, and will be respected by
3075  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3076  * of the whole object.
3077  *
3078  * This function returns when the move is complete, including waiting on
3079  * flushes to occur.
3080  */
3081 static int
3082 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3083                                           uint64_t offset, uint64_t size)
3084 {
3085         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3086         uint32_t old_read_domains;
3087         int i, ret;
3088
3089         if (offset == 0 && size == obj->size)
3090                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3091
3092         ret = i915_gem_object_flush_gpu_write_domain(obj);
3093         if (ret)
3094                 return ret;
3095
3096         /* Wait on any GPU rendering and flushing to occur. */
3097         ret = i915_gem_object_wait_rendering(obj);
3098         if (ret != 0)
3099                 return ret;
3100         i915_gem_object_flush_gtt_write_domain(obj);
3101
3102         /* If we're already fully in the CPU read domain, we're done. */
3103         if (obj_priv->page_cpu_valid == NULL &&
3104             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3105                 return 0;
3106
3107         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3108          * newly adding I915_GEM_DOMAIN_CPU
3109          */
3110         if (obj_priv->page_cpu_valid == NULL) {
3111                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3112                                                    GFP_KERNEL);
3113                 if (obj_priv->page_cpu_valid == NULL)
3114                         return -ENOMEM;
3115         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3116                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3117
3118         /* Flush the cache on any pages that are still invalid from the CPU's
3119          * perspective.
3120          */
3121         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3122              i++) {
3123                 if (obj_priv->page_cpu_valid[i])
3124                         continue;
3125
3126                 drm_clflush_pages(obj_priv->pages + i, 1);
3127
3128                 obj_priv->page_cpu_valid[i] = 1;
3129         }
3130
3131         /* It should now be out of any other write domains, and we can update
3132          * the domain values for our changes.
3133          */
3134         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3135
3136         old_read_domains = obj->read_domains;
3137         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3138
3139         trace_i915_gem_object_change_domain(obj,
3140                                             old_read_domains,
3141                                             obj->write_domain);
3142
3143         return 0;
3144 }
3145
3146 /**
3147  * Pin an object to the GTT and evaluate the relocations landing in it.
3148  */
3149 static int
3150 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3151                                  struct drm_file *file_priv,
3152                                  struct drm_i915_gem_exec_object2 *entry,
3153                                  struct drm_i915_gem_relocation_entry *relocs)
3154 {
3155         struct drm_device *dev = obj->dev;
3156         drm_i915_private_t *dev_priv = dev->dev_private;
3157         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3158         int i, ret;
3159         void __iomem *reloc_page;
3160         bool need_fence;
3161
3162         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3163                      obj_priv->tiling_mode != I915_TILING_NONE;
3164
3165         /* Check fence reg constraints and rebind if necessary */
3166         if (need_fence &&
3167             !i915_gem_object_fence_offset_ok(obj,
3168                                              obj_priv->tiling_mode)) {
3169                 ret = i915_gem_object_unbind(obj);
3170                 if (ret)
3171                         return ret;
3172         }
3173
3174         /* Choose the GTT offset for our buffer and put it there. */
3175         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3176         if (ret)
3177                 return ret;
3178
3179         /*
3180          * Pre-965 chips need a fence register set up in order to
3181          * properly handle blits to/from tiled surfaces.
3182          */
3183         if (need_fence) {
3184                 ret = i915_gem_object_get_fence_reg(obj);
3185                 if (ret != 0) {
3186                         i915_gem_object_unpin(obj);
3187                         return ret;
3188                 }
3189         }
3190
3191         entry->offset = obj_priv->gtt_offset;
3192
3193         /* Apply the relocations, using the GTT aperture to avoid cache
3194          * flushing requirements.
3195          */
3196         for (i = 0; i < entry->relocation_count; i++) {
3197                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3198                 struct drm_gem_object *target_obj;
3199                 struct drm_i915_gem_object *target_obj_priv;
3200                 uint32_t reloc_val, reloc_offset;
3201                 uint32_t __iomem *reloc_entry;
3202
3203                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3204                                                    reloc->target_handle);
3205                 if (target_obj == NULL) {
3206                         i915_gem_object_unpin(obj);
3207                         return -ENOENT;
3208                 }
3209                 target_obj_priv = to_intel_bo(target_obj);
3210
3211 #if WATCH_RELOC
3212                 DRM_INFO("%s: obj %p offset %08x target %d "
3213                          "read %08x write %08x gtt %08x "
3214                          "presumed %08x delta %08x\n",
3215                          __func__,
3216                          obj,
3217                          (int) reloc->offset,
3218                          (int) reloc->target_handle,
3219                          (int) reloc->read_domains,
3220                          (int) reloc->write_domain,
3221                          (int) target_obj_priv->gtt_offset,
3222                          (int) reloc->presumed_offset,
3223                          reloc->delta);
3224 #endif
3225
3226                 /* The target buffer should have appeared before us in the
3227                  * exec_object list, so it should have a GTT space bound by now.
3228                  */
3229                 if (target_obj_priv->gtt_space == NULL) {
3230                         DRM_ERROR("No GTT space found for object %d\n",
3231                                   reloc->target_handle);
3232                         drm_gem_object_unreference(target_obj);
3233                         i915_gem_object_unpin(obj);
3234                         return -EINVAL;
3235                 }
3236
3237                 /* Validate that the target is in a valid r/w GPU domain */
3238                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3239                         DRM_ERROR("reloc with multiple write domains: "
3240                                   "obj %p target %d offset %d "
3241                                   "read %08x write %08x",
3242                                   obj, reloc->target_handle,
3243                                   (int) reloc->offset,
3244                                   reloc->read_domains,
3245                                   reloc->write_domain);
3246                         return -EINVAL;
3247                 }
3248                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3249                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3250                         DRM_ERROR("reloc with read/write CPU domains: "
3251                                   "obj %p target %d offset %d "
3252                                   "read %08x write %08x",
3253                                   obj, reloc->target_handle,
3254                                   (int) reloc->offset,
3255                                   reloc->read_domains,
3256                                   reloc->write_domain);
3257                         drm_gem_object_unreference(target_obj);
3258                         i915_gem_object_unpin(obj);
3259                         return -EINVAL;
3260                 }
3261                 if (reloc->write_domain && target_obj->pending_write_domain &&
3262                     reloc->write_domain != target_obj->pending_write_domain) {
3263                         DRM_ERROR("Write domain conflict: "
3264                                   "obj %p target %d offset %d "
3265                                   "new %08x old %08x\n",
3266                                   obj, reloc->target_handle,
3267                                   (int) reloc->offset,
3268                                   reloc->write_domain,
3269                                   target_obj->pending_write_domain);
3270                         drm_gem_object_unreference(target_obj);
3271                         i915_gem_object_unpin(obj);
3272                         return -EINVAL;
3273                 }
3274
3275                 target_obj->pending_read_domains |= reloc->read_domains;
3276                 target_obj->pending_write_domain |= reloc->write_domain;
3277
3278                 /* If the relocation already has the right value in it, no
3279                  * more work needs to be done.
3280                  */
3281                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3282                         drm_gem_object_unreference(target_obj);
3283                         continue;
3284                 }
3285
3286                 /* Check that the relocation address is valid... */
3287                 if (reloc->offset > obj->size - 4) {
3288                         DRM_ERROR("Relocation beyond object bounds: "
3289                                   "obj %p target %d offset %d size %d.\n",
3290                                   obj, reloc->target_handle,
3291                                   (int) reloc->offset, (int) obj->size);
3292                         drm_gem_object_unreference(target_obj);
3293                         i915_gem_object_unpin(obj);
3294                         return -EINVAL;
3295                 }
3296                 if (reloc->offset & 3) {
3297                         DRM_ERROR("Relocation not 4-byte aligned: "
3298                                   "obj %p target %d offset %d.\n",
3299                                   obj, reloc->target_handle,
3300                                   (int) reloc->offset);
3301                         drm_gem_object_unreference(target_obj);
3302                         i915_gem_object_unpin(obj);
3303                         return -EINVAL;
3304                 }
3305
3306                 /* and points to somewhere within the target object. */
3307                 if (reloc->delta >= target_obj->size) {
3308                         DRM_ERROR("Relocation beyond target object bounds: "
3309                                   "obj %p target %d delta %d size %d.\n",
3310                                   obj, reloc->target_handle,
3311                                   (int) reloc->delta, (int) target_obj->size);
3312                         drm_gem_object_unreference(target_obj);
3313                         i915_gem_object_unpin(obj);
3314                         return -EINVAL;
3315                 }
3316
3317                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3318                 if (ret != 0) {
3319                         drm_gem_object_unreference(target_obj);
3320                         i915_gem_object_unpin(obj);
3321                         return -EINVAL;
3322                 }
3323
3324                 /* Map the page containing the relocation we're going to
3325                  * perform.
3326                  */
3327                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3328                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3329                                                       (reloc_offset &
3330                                                        ~(PAGE_SIZE - 1)),
3331                                                       KM_USER0);
3332                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3333                                                    (reloc_offset & (PAGE_SIZE - 1)));
3334                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3335
3336 #if WATCH_BUF
3337                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3338                           obj, (unsigned int) reloc->offset,
3339                           readl(reloc_entry), reloc_val);
3340 #endif
3341                 writel(reloc_val, reloc_entry);
3342                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3343
3344                 /* The updated presumed offset for this entry will be
3345                  * copied back out to the user.
3346                  */
3347                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3348
3349                 drm_gem_object_unreference(target_obj);
3350         }
3351
3352 #if WATCH_BUF
3353         if (0)
3354                 i915_gem_dump_object(obj, 128, __func__, ~0);
3355 #endif
3356         return 0;
3357 }
3358
3359 /* Throttle our rendering by waiting until the ring has completed our requests
3360  * emitted over 20 msec ago.
3361  *
3362  * Note that if we were to use the current jiffies each time around the loop,
3363  * we wouldn't escape the function with any frames outstanding if the time to
3364  * render a frame was over 20ms.
3365  *
3366  * This should get us reasonable parallelism between CPU and GPU but also
3367  * relatively low latency when blocking on a particular request to finish.
3368  */
3369 static int
3370 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3371 {
3372         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3373         int ret = 0;
3374         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3375
3376         mutex_lock(&dev->struct_mutex);
3377         while (!list_empty(&i915_file_priv->mm.request_list)) {
3378                 struct drm_i915_gem_request *request;
3379
3380                 request = list_first_entry(&i915_file_priv->mm.request_list,
3381                                            struct drm_i915_gem_request,
3382                                            client_list);
3383
3384                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3385                         break;
3386
3387                 ret = i915_wait_request(dev, request->seqno, request->ring);
3388                 if (ret != 0)
3389                         break;
3390         }
3391         mutex_unlock(&dev->struct_mutex);
3392
3393         return ret;
3394 }
3395
3396 static int
3397 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3398                               uint32_t buffer_count,
3399                               struct drm_i915_gem_relocation_entry **relocs)
3400 {
3401         uint32_t reloc_count = 0, reloc_index = 0, i;
3402         int ret;
3403
3404         *relocs = NULL;
3405         for (i = 0; i < buffer_count; i++) {
3406                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3407                         return -EINVAL;
3408                 reloc_count += exec_list[i].relocation_count;
3409         }
3410
3411         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3412         if (*relocs == NULL) {
3413                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3414                 return -ENOMEM;
3415         }
3416
3417         for (i = 0; i < buffer_count; i++) {
3418                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3419
3420                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3421
3422                 ret = copy_from_user(&(*relocs)[reloc_index],
3423                                      user_relocs,
3424                                      exec_list[i].relocation_count *
3425                                      sizeof(**relocs));
3426                 if (ret != 0) {
3427                         drm_free_large(*relocs);
3428                         *relocs = NULL;
3429                         return -EFAULT;
3430                 }
3431
3432                 reloc_index += exec_list[i].relocation_count;
3433         }
3434
3435         return 0;
3436 }
3437
3438 static int
3439 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3440                             uint32_t buffer_count,
3441                             struct drm_i915_gem_relocation_entry *relocs)
3442 {
3443         uint32_t reloc_count = 0, i;
3444         int ret = 0;
3445
3446         if (relocs == NULL)
3447             return 0;
3448
3449         for (i = 0; i < buffer_count; i++) {
3450                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3451                 int unwritten;
3452
3453                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3454
3455                 unwritten = copy_to_user(user_relocs,
3456                                          &relocs[reloc_count],
3457                                          exec_list[i].relocation_count *
3458                                          sizeof(*relocs));
3459
3460                 if (unwritten) {
3461                         ret = -EFAULT;
3462                         goto err;
3463                 }
3464
3465                 reloc_count += exec_list[i].relocation_count;
3466         }
3467
3468 err:
3469         drm_free_large(relocs);
3470
3471         return ret;
3472 }
3473
3474 static int
3475 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3476                            uint64_t exec_offset)
3477 {
3478         uint32_t exec_start, exec_len;
3479
3480         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3481         exec_len = (uint32_t) exec->batch_len;
3482
3483         if ((exec_start | exec_len) & 0x7)
3484                 return -EINVAL;
3485
3486         if (!exec_start)
3487                 return -EINVAL;
3488
3489         return 0;
3490 }
3491
3492 static int
3493 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3494                                struct drm_gem_object **object_list,
3495                                int count)
3496 {
3497         drm_i915_private_t *dev_priv = dev->dev_private;
3498         struct drm_i915_gem_object *obj_priv;
3499         DEFINE_WAIT(wait);
3500         int i, ret = 0;
3501
3502         for (;;) {
3503                 prepare_to_wait(&dev_priv->pending_flip_queue,
3504                                 &wait, TASK_INTERRUPTIBLE);
3505                 for (i = 0; i < count; i++) {
3506                         obj_priv = to_intel_bo(object_list[i]);
3507                         if (atomic_read(&obj_priv->pending_flip) > 0)
3508                                 break;
3509                 }
3510                 if (i == count)
3511                         break;
3512
3513                 if (!signal_pending(current)) {
3514                         mutex_unlock(&dev->struct_mutex);
3515                         schedule();
3516                         mutex_lock(&dev->struct_mutex);
3517                         continue;
3518                 }
3519                 ret = -ERESTARTSYS;
3520                 break;
3521         }
3522         finish_wait(&dev_priv->pending_flip_queue, &wait);
3523
3524         return ret;
3525 }
3526
3527
3528 int
3529 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3530                        struct drm_file *file_priv,
3531                        struct drm_i915_gem_execbuffer2 *args,
3532                        struct drm_i915_gem_exec_object2 *exec_list)
3533 {
3534         drm_i915_private_t *dev_priv = dev->dev_private;
3535         struct drm_gem_object **object_list = NULL;
3536         struct drm_gem_object *batch_obj;
3537         struct drm_i915_gem_object *obj_priv;
3538         struct drm_clip_rect *cliprects = NULL;
3539         struct drm_i915_gem_relocation_entry *relocs = NULL;
3540         int ret = 0, ret2, i, pinned = 0;
3541         uint64_t exec_offset;
3542         uint32_t seqno, flush_domains, reloc_index;
3543         int pin_tries, flips;
3544
3545         struct intel_ring_buffer *ring = NULL;
3546
3547 #if WATCH_EXEC
3548         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3549                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3550 #endif
3551         if (args->flags & I915_EXEC_BSD) {
3552                 if (!HAS_BSD(dev)) {
3553                         DRM_ERROR("execbuf with wrong flag\n");
3554                         return -EINVAL;
3555                 }
3556                 ring = &dev_priv->bsd_ring;
3557         } else {
3558                 ring = &dev_priv->render_ring;
3559         }
3560
3561         if (args->buffer_count < 1) {
3562                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3563                 return -EINVAL;
3564         }
3565         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3566         if (object_list == NULL) {
3567                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3568                           args->buffer_count);
3569                 ret = -ENOMEM;
3570                 goto pre_mutex_err;
3571         }
3572
3573         if (args->num_cliprects != 0) {
3574                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3575                                     GFP_KERNEL);
3576                 if (cliprects == NULL) {
3577                         ret = -ENOMEM;
3578                         goto pre_mutex_err;
3579                 }
3580
3581                 ret = copy_from_user(cliprects,
3582                                      (struct drm_clip_rect __user *)
3583                                      (uintptr_t) args->cliprects_ptr,
3584                                      sizeof(*cliprects) * args->num_cliprects);
3585                 if (ret != 0) {
3586                         DRM_ERROR("copy %d cliprects failed: %d\n",
3587                                   args->num_cliprects, ret);
3588                         goto pre_mutex_err;
3589                 }
3590         }
3591
3592         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3593                                             &relocs);
3594         if (ret != 0)
3595                 goto pre_mutex_err;
3596
3597         mutex_lock(&dev->struct_mutex);
3598
3599         i915_verify_inactive(dev, __FILE__, __LINE__);
3600
3601         if (atomic_read(&dev_priv->mm.wedged)) {
3602                 mutex_unlock(&dev->struct_mutex);
3603                 ret = -EIO;
3604                 goto pre_mutex_err;
3605         }
3606
3607         if (dev_priv->mm.suspended) {
3608                 mutex_unlock(&dev->struct_mutex);
3609                 ret = -EBUSY;
3610                 goto pre_mutex_err;
3611         }
3612
3613         /* Look up object handles */
3614         flips = 0;
3615         for (i = 0; i < args->buffer_count; i++) {
3616                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3617                                                        exec_list[i].handle);
3618                 if (object_list[i] == NULL) {
3619                         DRM_ERROR("Invalid object handle %d at index %d\n",
3620                                    exec_list[i].handle, i);
3621                         /* prevent error path from reading uninitialized data */
3622                         args->buffer_count = i + 1;
3623                         ret = -ENOENT;
3624                         goto err;
3625                 }
3626
3627                 obj_priv = to_intel_bo(object_list[i]);
3628                 if (obj_priv->in_execbuffer) {
3629                         DRM_ERROR("Object %p appears more than once in object list\n",
3630                                    object_list[i]);
3631                         /* prevent error path from reading uninitialized data */
3632                         args->buffer_count = i + 1;
3633                         ret = -EINVAL;
3634                         goto err;
3635                 }
3636                 obj_priv->in_execbuffer = true;
3637                 flips += atomic_read(&obj_priv->pending_flip);
3638         }
3639
3640         if (flips > 0) {
3641                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3642                                                      args->buffer_count);
3643                 if (ret)
3644                         goto err;
3645         }
3646
3647         /* Pin and relocate */
3648         for (pin_tries = 0; ; pin_tries++) {
3649                 ret = 0;
3650                 reloc_index = 0;
3651
3652                 for (i = 0; i < args->buffer_count; i++) {
3653                         object_list[i]->pending_read_domains = 0;
3654                         object_list[i]->pending_write_domain = 0;
3655                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3656                                                                file_priv,
3657                                                                &exec_list[i],
3658                                                                &relocs[reloc_index]);
3659                         if (ret)
3660                                 break;
3661                         pinned = i + 1;
3662                         reloc_index += exec_list[i].relocation_count;
3663                 }
3664                 /* success */
3665                 if (ret == 0)
3666                         break;
3667
3668                 /* error other than GTT full, or we've already tried again */
3669                 if (ret != -ENOSPC || pin_tries >= 1) {
3670                         if (ret != -ERESTARTSYS) {
3671                                 unsigned long long total_size = 0;
3672                                 int num_fences = 0;
3673                                 for (i = 0; i < args->buffer_count; i++) {
3674                                         obj_priv = to_intel_bo(object_list[i]);
3675
3676                                         total_size += object_list[i]->size;
3677                                         num_fences +=
3678                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3679                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3680                                 }
3681                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3682                                           pinned+1, args->buffer_count,
3683                                           total_size, num_fences,
3684                                           ret);
3685                                 DRM_ERROR("%d objects [%d pinned], "
3686                                           "%d object bytes [%d pinned], "
3687                                           "%d/%d gtt bytes\n",
3688                                           atomic_read(&dev->object_count),
3689                                           atomic_read(&dev->pin_count),
3690                                           atomic_read(&dev->object_memory),
3691                                           atomic_read(&dev->pin_memory),
3692                                           atomic_read(&dev->gtt_memory),
3693                                           dev->gtt_total);
3694                         }
3695                         goto err;
3696                 }
3697
3698                 /* unpin all of our buffers */
3699                 for (i = 0; i < pinned; i++)
3700                         i915_gem_object_unpin(object_list[i]);
3701                 pinned = 0;
3702
3703                 /* evict everyone we can from the aperture */
3704                 ret = i915_gem_evict_everything(dev);
3705                 if (ret && ret != -ENOSPC)
3706                         goto err;
3707         }
3708
3709         /* Set the pending read domains for the batch buffer to COMMAND */
3710         batch_obj = object_list[args->buffer_count-1];
3711         if (batch_obj->pending_write_domain) {
3712                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3713                 ret = -EINVAL;
3714                 goto err;
3715         }
3716         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3717
3718         /* Sanity check the batch buffer, prior to moving objects */
3719         exec_offset = exec_list[args->buffer_count - 1].offset;
3720         ret = i915_gem_check_execbuffer (args, exec_offset);
3721         if (ret != 0) {
3722                 DRM_ERROR("execbuf with invalid offset/length\n");
3723                 goto err;
3724         }
3725
3726         i915_verify_inactive(dev, __FILE__, __LINE__);
3727
3728         /* Zero the global flush/invalidate flags. These
3729          * will be modified as new domains are computed
3730          * for each object
3731          */
3732         dev->invalidate_domains = 0;
3733         dev->flush_domains = 0;
3734         dev_priv->flush_rings = 0;
3735
3736         for (i = 0; i < args->buffer_count; i++) {
3737                 struct drm_gem_object *obj = object_list[i];
3738
3739                 /* Compute new gpu domains and update invalidate/flush */
3740                 i915_gem_object_set_to_gpu_domain(obj);
3741         }
3742
3743         i915_verify_inactive(dev, __FILE__, __LINE__);
3744
3745         if (dev->invalidate_domains | dev->flush_domains) {
3746 #if WATCH_EXEC
3747                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3748                           __func__,
3749                          dev->invalidate_domains,
3750                          dev->flush_domains);
3751 #endif
3752                 i915_gem_flush(dev,
3753                                dev->invalidate_domains,
3754                                dev->flush_domains);
3755                 if (dev_priv->flush_rings & FLUSH_RENDER_RING)
3756                         (void)i915_add_request(dev, file_priv,
3757                                                dev->flush_domains,
3758                                                &dev_priv->render_ring);
3759                 if (dev_priv->flush_rings & FLUSH_BSD_RING)
3760                         (void)i915_add_request(dev, file_priv,
3761                                                dev->flush_domains,
3762                                                &dev_priv->bsd_ring);
3763         }
3764
3765         for (i = 0; i < args->buffer_count; i++) {
3766                 struct drm_gem_object *obj = object_list[i];
3767                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3768                 uint32_t old_write_domain = obj->write_domain;
3769
3770                 obj->write_domain = obj->pending_write_domain;
3771                 if (obj->write_domain)
3772                         list_move_tail(&obj_priv->gpu_write_list,
3773                                        &dev_priv->mm.gpu_write_list);
3774                 else
3775                         list_del_init(&obj_priv->gpu_write_list);
3776
3777                 trace_i915_gem_object_change_domain(obj,
3778                                                     obj->read_domains,
3779                                                     old_write_domain);
3780         }
3781
3782         i915_verify_inactive(dev, __FILE__, __LINE__);
3783
3784 #if WATCH_COHERENCY
3785         for (i = 0; i < args->buffer_count; i++) {
3786                 i915_gem_object_check_coherency(object_list[i],
3787                                                 exec_list[i].handle);
3788         }
3789 #endif
3790
3791 #if WATCH_EXEC
3792         i915_gem_dump_object(batch_obj,
3793                               args->batch_len,
3794                               __func__,
3795                               ~0);
3796 #endif
3797
3798         /* Exec the batchbuffer */
3799         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3800                         cliprects, exec_offset);
3801         if (ret) {
3802                 DRM_ERROR("dispatch failed %d\n", ret);
3803                 goto err;
3804         }
3805
3806         /*
3807          * Ensure that the commands in the batch buffer are
3808          * finished before the interrupt fires
3809          */
3810         flush_domains = i915_retire_commands(dev, ring);
3811
3812         i915_verify_inactive(dev, __FILE__, __LINE__);
3813
3814         /*
3815          * Get a seqno representing the execution of the current buffer,
3816          * which we can wait on.  We would like to mitigate these interrupts,
3817          * likely by only creating seqnos occasionally (so that we have
3818          * *some* interrupts representing completion of buffers that we can
3819          * wait on when trying to clear up gtt space).
3820          */
3821         seqno = i915_add_request(dev, file_priv, flush_domains, ring);
3822         BUG_ON(seqno == 0);
3823         for (i = 0; i < args->buffer_count; i++) {
3824                 struct drm_gem_object *obj = object_list[i];
3825                 obj_priv = to_intel_bo(obj);
3826
3827                 i915_gem_object_move_to_active(obj, seqno, ring);
3828 #if WATCH_LRU
3829                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3830 #endif
3831         }
3832 #if WATCH_LRU
3833         i915_dump_lru(dev, __func__);
3834 #endif
3835
3836         i915_verify_inactive(dev, __FILE__, __LINE__);
3837
3838 err:
3839         for (i = 0; i < pinned; i++)
3840                 i915_gem_object_unpin(object_list[i]);
3841
3842         for (i = 0; i < args->buffer_count; i++) {
3843                 if (object_list[i]) {
3844                         obj_priv = to_intel_bo(object_list[i]);
3845                         obj_priv->in_execbuffer = false;
3846                 }
3847                 drm_gem_object_unreference(object_list[i]);
3848         }
3849
3850         mutex_unlock(&dev->struct_mutex);
3851
3852 pre_mutex_err:
3853         /* Copy the updated relocations out regardless of current error
3854          * state.  Failure to update the relocs would mean that the next
3855          * time userland calls execbuf, it would do so with presumed offset
3856          * state that didn't match the actual object state.
3857          */
3858         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3859                                            relocs);
3860         if (ret2 != 0) {
3861                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3862
3863                 if (ret == 0)
3864                         ret = ret2;
3865         }
3866
3867         drm_free_large(object_list);
3868         kfree(cliprects);
3869
3870         return ret;
3871 }
3872
3873 /*
3874  * Legacy execbuffer just creates an exec2 list from the original exec object
3875  * list array and passes it to the real function.
3876  */
3877 int
3878 i915_gem_execbuffer(struct drm_device *dev, void *data,
3879                     struct drm_file *file_priv)
3880 {
3881         struct drm_i915_gem_execbuffer *args = data;
3882         struct drm_i915_gem_execbuffer2 exec2;
3883         struct drm_i915_gem_exec_object *exec_list = NULL;
3884         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3885         int ret, i;
3886
3887 #if WATCH_EXEC
3888         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3889                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3890 #endif
3891
3892         if (args->buffer_count < 1) {
3893                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3894                 return -EINVAL;
3895         }
3896
3897         /* Copy in the exec list from userland */
3898         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3899         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3900         if (exec_list == NULL || exec2_list == NULL) {
3901                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3902                           args->buffer_count);
3903                 drm_free_large(exec_list);
3904                 drm_free_large(exec2_list);
3905                 return -ENOMEM;
3906         }
3907         ret = copy_from_user(exec_list,
3908                              (struct drm_i915_relocation_entry __user *)
3909                              (uintptr_t) args->buffers_ptr,
3910                              sizeof(*exec_list) * args->buffer_count);
3911         if (ret != 0) {
3912                 DRM_ERROR("copy %d exec entries failed %d\n",
3913                           args->buffer_count, ret);
3914                 drm_free_large(exec_list);
3915                 drm_free_large(exec2_list);
3916                 return -EFAULT;
3917         }
3918
3919         for (i = 0; i < args->buffer_count; i++) {
3920                 exec2_list[i].handle = exec_list[i].handle;
3921                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3922                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3923                 exec2_list[i].alignment = exec_list[i].alignment;
3924                 exec2_list[i].offset = exec_list[i].offset;
3925                 if (!IS_I965G(dev))
3926                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3927                 else
3928                         exec2_list[i].flags = 0;
3929         }
3930
3931         exec2.buffers_ptr = args->buffers_ptr;
3932         exec2.buffer_count = args->buffer_count;
3933         exec2.batch_start_offset = args->batch_start_offset;
3934         exec2.batch_len = args->batch_len;
3935         exec2.DR1 = args->DR1;
3936         exec2.DR4 = args->DR4;
3937         exec2.num_cliprects = args->num_cliprects;
3938         exec2.cliprects_ptr = args->cliprects_ptr;
3939         exec2.flags = I915_EXEC_RENDER;
3940
3941         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3942         if (!ret) {
3943                 /* Copy the new buffer offsets back to the user's exec list. */
3944                 for (i = 0; i < args->buffer_count; i++)
3945                         exec_list[i].offset = exec2_list[i].offset;
3946                 /* ... and back out to userspace */
3947                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3948                                    (uintptr_t) args->buffers_ptr,
3949                                    exec_list,
3950                                    sizeof(*exec_list) * args->buffer_count);
3951                 if (ret) {
3952                         ret = -EFAULT;
3953                         DRM_ERROR("failed to copy %d exec entries "
3954                                   "back to user (%d)\n",
3955                                   args->buffer_count, ret);
3956                 }
3957         }
3958
3959         drm_free_large(exec_list);
3960         drm_free_large(exec2_list);
3961         return ret;
3962 }
3963
3964 int
3965 i915_gem_execbuffer2(struct drm_device *dev, void *data,
3966                      struct drm_file *file_priv)
3967 {
3968         struct drm_i915_gem_execbuffer2 *args = data;
3969         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3970         int ret;
3971
3972 #if WATCH_EXEC
3973         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3974                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3975 #endif
3976
3977         if (args->buffer_count < 1) {
3978                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3979                 return -EINVAL;
3980         }
3981
3982         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3983         if (exec2_list == NULL) {
3984                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3985                           args->buffer_count);
3986                 return -ENOMEM;
3987         }
3988         ret = copy_from_user(exec2_list,
3989                              (struct drm_i915_relocation_entry __user *)
3990                              (uintptr_t) args->buffers_ptr,
3991                              sizeof(*exec2_list) * args->buffer_count);
3992         if (ret != 0) {
3993                 DRM_ERROR("copy %d exec entries failed %d\n",
3994                           args->buffer_count, ret);
3995                 drm_free_large(exec2_list);
3996                 return -EFAULT;
3997         }
3998
3999         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4000         if (!ret) {
4001                 /* Copy the new buffer offsets back to the user's exec list. */
4002                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4003                                    (uintptr_t) args->buffers_ptr,
4004                                    exec2_list,
4005                                    sizeof(*exec2_list) * args->buffer_count);
4006                 if (ret) {
4007                         ret = -EFAULT;
4008                         DRM_ERROR("failed to copy %d exec entries "
4009                                   "back to user (%d)\n",
4010                                   args->buffer_count, ret);
4011                 }
4012         }
4013
4014         drm_free_large(exec2_list);
4015         return ret;
4016 }
4017
4018 int
4019 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4020 {
4021         struct drm_device *dev = obj->dev;
4022         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4023         int ret;
4024
4025         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4026
4027         i915_verify_inactive(dev, __FILE__, __LINE__);
4028
4029         if (obj_priv->gtt_space != NULL) {
4030                 if (alignment == 0)
4031                         alignment = i915_gem_get_gtt_alignment(obj);
4032                 if (obj_priv->gtt_offset & (alignment - 1)) {
4033                         WARN(obj_priv->pin_count,
4034                              "bo is already pinned with incorrect alignment:"
4035                              " offset=%x, req.alignment=%x\n",
4036                              obj_priv->gtt_offset, alignment);
4037                         ret = i915_gem_object_unbind(obj);
4038                         if (ret)
4039                                 return ret;
4040                 }
4041         }
4042
4043         if (obj_priv->gtt_space == NULL) {
4044                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4045                 if (ret)
4046                         return ret;
4047         }
4048
4049         obj_priv->pin_count++;
4050
4051         /* If the object is not active and not pending a flush,
4052          * remove it from the inactive list
4053          */
4054         if (obj_priv->pin_count == 1) {
4055                 atomic_inc(&dev->pin_count);
4056                 atomic_add(obj->size, &dev->pin_memory);
4057                 if (!obj_priv->active &&
4058                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4059                         list_del_init(&obj_priv->list);
4060         }
4061         i915_verify_inactive(dev, __FILE__, __LINE__);
4062
4063         return 0;
4064 }
4065
4066 void
4067 i915_gem_object_unpin(struct drm_gem_object *obj)
4068 {
4069         struct drm_device *dev = obj->dev;
4070         drm_i915_private_t *dev_priv = dev->dev_private;
4071         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4072
4073         i915_verify_inactive(dev, __FILE__, __LINE__);
4074         obj_priv->pin_count--;
4075         BUG_ON(obj_priv->pin_count < 0);
4076         BUG_ON(obj_priv->gtt_space == NULL);
4077
4078         /* If the object is no longer pinned, and is
4079          * neither active nor being flushed, then stick it on
4080          * the inactive list
4081          */
4082         if (obj_priv->pin_count == 0) {
4083                 if (!obj_priv->active &&
4084                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4085                         list_move_tail(&obj_priv->list,
4086                                        &dev_priv->mm.inactive_list);
4087                 atomic_dec(&dev->pin_count);
4088                 atomic_sub(obj->size, &dev->pin_memory);
4089         }
4090         i915_verify_inactive(dev, __FILE__, __LINE__);
4091 }
4092
4093 int
4094 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4095                    struct drm_file *file_priv)
4096 {
4097         struct drm_i915_gem_pin *args = data;
4098         struct drm_gem_object *obj;
4099         struct drm_i915_gem_object *obj_priv;
4100         int ret;
4101
4102         mutex_lock(&dev->struct_mutex);
4103
4104         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4105         if (obj == NULL) {
4106                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4107                           args->handle);
4108                 mutex_unlock(&dev->struct_mutex);
4109                 return -ENOENT;
4110         }
4111         obj_priv = to_intel_bo(obj);
4112
4113         if (obj_priv->madv != I915_MADV_WILLNEED) {
4114                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4115                 drm_gem_object_unreference(obj);
4116                 mutex_unlock(&dev->struct_mutex);
4117                 return -EINVAL;
4118         }
4119
4120         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4121                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4122                           args->handle);
4123                 drm_gem_object_unreference(obj);
4124                 mutex_unlock(&dev->struct_mutex);
4125                 return -EINVAL;
4126         }
4127
4128         obj_priv->user_pin_count++;
4129         obj_priv->pin_filp = file_priv;
4130         if (obj_priv->user_pin_count == 1) {
4131                 ret = i915_gem_object_pin(obj, args->alignment);
4132                 if (ret != 0) {
4133                         drm_gem_object_unreference(obj);
4134                         mutex_unlock(&dev->struct_mutex);
4135                         return ret;
4136                 }
4137         }
4138
4139         /* XXX - flush the CPU caches for pinned objects
4140          * as the X server doesn't manage domains yet
4141          */
4142         i915_gem_object_flush_cpu_write_domain(obj);
4143         args->offset = obj_priv->gtt_offset;
4144         drm_gem_object_unreference(obj);
4145         mutex_unlock(&dev->struct_mutex);
4146
4147         return 0;
4148 }
4149
4150 int
4151 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4152                      struct drm_file *file_priv)
4153 {
4154         struct drm_i915_gem_pin *args = data;
4155         struct drm_gem_object *obj;
4156         struct drm_i915_gem_object *obj_priv;
4157
4158         mutex_lock(&dev->struct_mutex);
4159
4160         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4161         if (obj == NULL) {
4162                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4163                           args->handle);
4164                 mutex_unlock(&dev->struct_mutex);
4165                 return -ENOENT;
4166         }
4167
4168         obj_priv = to_intel_bo(obj);
4169         if (obj_priv->pin_filp != file_priv) {
4170                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4171                           args->handle);
4172                 drm_gem_object_unreference(obj);
4173                 mutex_unlock(&dev->struct_mutex);
4174                 return -EINVAL;
4175         }
4176         obj_priv->user_pin_count--;
4177         if (obj_priv->user_pin_count == 0) {
4178                 obj_priv->pin_filp = NULL;
4179                 i915_gem_object_unpin(obj);
4180         }
4181
4182         drm_gem_object_unreference(obj);
4183         mutex_unlock(&dev->struct_mutex);
4184         return 0;
4185 }
4186
4187 int
4188 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4189                     struct drm_file *file_priv)
4190 {
4191         struct drm_i915_gem_busy *args = data;
4192         struct drm_gem_object *obj;
4193         struct drm_i915_gem_object *obj_priv;
4194
4195         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4196         if (obj == NULL) {
4197                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4198                           args->handle);
4199                 return -ENOENT;
4200         }
4201
4202         mutex_lock(&dev->struct_mutex);
4203
4204         /* Count all active objects as busy, even if they are currently not used
4205          * by the gpu. Users of this interface expect objects to eventually
4206          * become non-busy without any further actions, therefore emit any
4207          * necessary flushes here.
4208          */
4209         obj_priv = to_intel_bo(obj);
4210         args->busy = obj_priv->active;
4211         if (args->busy) {
4212                 /* Unconditionally flush objects, even when the gpu still uses this
4213                  * object. Userspace calling this function indicates that it wants to
4214                  * use this buffer rather sooner than later, so issuing the required
4215                  * flush earlier is beneficial.
4216                  */
4217                 if (obj->write_domain) {
4218                         i915_gem_flush(dev, 0, obj->write_domain);
4219                         (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
4220                 }
4221
4222                 /* Update the active list for the hardware's current position.
4223                  * Otherwise this only updates on a delayed timer or when irqs
4224                  * are actually unmasked, and our working set ends up being
4225                  * larger than required.
4226                  */
4227                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4228
4229                 args->busy = obj_priv->active;
4230         }
4231
4232         drm_gem_object_unreference(obj);
4233         mutex_unlock(&dev->struct_mutex);
4234         return 0;
4235 }
4236
4237 int
4238 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4239                         struct drm_file *file_priv)
4240 {
4241     return i915_gem_ring_throttle(dev, file_priv);
4242 }
4243
4244 int
4245 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4246                        struct drm_file *file_priv)
4247 {
4248         struct drm_i915_gem_madvise *args = data;
4249         struct drm_gem_object *obj;
4250         struct drm_i915_gem_object *obj_priv;
4251
4252         switch (args->madv) {
4253         case I915_MADV_DONTNEED:
4254         case I915_MADV_WILLNEED:
4255             break;
4256         default:
4257             return -EINVAL;
4258         }
4259
4260         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4261         if (obj == NULL) {
4262                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4263                           args->handle);
4264                 return -ENOENT;
4265         }
4266
4267         mutex_lock(&dev->struct_mutex);
4268         obj_priv = to_intel_bo(obj);
4269
4270         if (obj_priv->pin_count) {
4271                 drm_gem_object_unreference(obj);
4272                 mutex_unlock(&dev->struct_mutex);
4273
4274                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4275                 return -EINVAL;
4276         }
4277
4278         if (obj_priv->madv != __I915_MADV_PURGED)
4279                 obj_priv->madv = args->madv;
4280
4281         /* if the object is no longer bound, discard its backing storage */
4282         if (i915_gem_object_is_purgeable(obj_priv) &&
4283             obj_priv->gtt_space == NULL)
4284                 i915_gem_object_truncate(obj);
4285
4286         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4287
4288         drm_gem_object_unreference(obj);
4289         mutex_unlock(&dev->struct_mutex);
4290
4291         return 0;
4292 }
4293
4294 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4295                                               size_t size)
4296 {
4297         struct drm_i915_gem_object *obj;
4298
4299         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4300         if (obj == NULL)
4301                 return NULL;
4302
4303         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4304                 kfree(obj);
4305                 return NULL;
4306         }
4307
4308         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4309         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4310
4311         obj->agp_type = AGP_USER_MEMORY;
4312         obj->base.driver_private = NULL;
4313         obj->fence_reg = I915_FENCE_REG_NONE;
4314         INIT_LIST_HEAD(&obj->list);
4315         INIT_LIST_HEAD(&obj->gpu_write_list);
4316         obj->madv = I915_MADV_WILLNEED;
4317
4318         trace_i915_gem_object_create(&obj->base);
4319
4320         return &obj->base;
4321 }
4322
4323 int i915_gem_init_object(struct drm_gem_object *obj)
4324 {
4325         BUG();
4326
4327         return 0;
4328 }
4329
4330 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4331 {
4332         struct drm_device *dev = obj->dev;
4333         drm_i915_private_t *dev_priv = dev->dev_private;
4334         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4335         int ret;
4336
4337         ret = i915_gem_object_unbind(obj);
4338         if (ret == -ERESTARTSYS) {
4339                 list_move(&obj_priv->list,
4340                           &dev_priv->mm.deferred_free_list);
4341                 return;
4342         }
4343
4344         if (obj_priv->mmap_offset)
4345                 i915_gem_free_mmap_offset(obj);
4346
4347         drm_gem_object_release(obj);
4348
4349         kfree(obj_priv->page_cpu_valid);
4350         kfree(obj_priv->bit_17);
4351         kfree(obj_priv);
4352 }
4353
4354 void i915_gem_free_object(struct drm_gem_object *obj)
4355 {
4356         struct drm_device *dev = obj->dev;
4357         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4358
4359         trace_i915_gem_object_destroy(obj);
4360
4361         while (obj_priv->pin_count > 0)
4362                 i915_gem_object_unpin(obj);
4363
4364         if (obj_priv->phys_obj)
4365                 i915_gem_detach_phys_object(dev, obj);
4366
4367         i915_gem_free_object_tail(obj);
4368 }
4369
4370 int
4371 i915_gem_idle(struct drm_device *dev)
4372 {
4373         drm_i915_private_t *dev_priv = dev->dev_private;
4374         int ret;
4375
4376         mutex_lock(&dev->struct_mutex);
4377
4378         if (dev_priv->mm.suspended ||
4379                         (dev_priv->render_ring.gem_object == NULL) ||
4380                         (HAS_BSD(dev) &&
4381                          dev_priv->bsd_ring.gem_object == NULL)) {
4382                 mutex_unlock(&dev->struct_mutex);
4383                 return 0;
4384         }
4385
4386         ret = i915_gpu_idle(dev);
4387         if (ret) {
4388                 mutex_unlock(&dev->struct_mutex);
4389                 return ret;
4390         }
4391
4392         /* Under UMS, be paranoid and evict. */
4393         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4394                 ret = i915_gem_evict_inactive(dev);
4395                 if (ret) {
4396                         mutex_unlock(&dev->struct_mutex);
4397                         return ret;
4398                 }
4399         }
4400
4401         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4402          * We need to replace this with a semaphore, or something.
4403          * And not confound mm.suspended!
4404          */
4405         dev_priv->mm.suspended = 1;
4406         del_timer(&dev_priv->hangcheck_timer);
4407
4408         i915_kernel_lost_context(dev);
4409         i915_gem_cleanup_ringbuffer(dev);
4410
4411         mutex_unlock(&dev->struct_mutex);
4412
4413         /* Cancel the retire work handler, which should be idle now. */
4414         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4415
4416         return 0;
4417 }
4418
4419 /*
4420  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4421  * over cache flushing.
4422  */
4423 static int
4424 i915_gem_init_pipe_control(struct drm_device *dev)
4425 {
4426         drm_i915_private_t *dev_priv = dev->dev_private;
4427         struct drm_gem_object *obj;
4428         struct drm_i915_gem_object *obj_priv;
4429         int ret;
4430
4431         obj = i915_gem_alloc_object(dev, 4096);
4432         if (obj == NULL) {
4433                 DRM_ERROR("Failed to allocate seqno page\n");
4434                 ret = -ENOMEM;
4435                 goto err;
4436         }
4437         obj_priv = to_intel_bo(obj);
4438         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4439
4440         ret = i915_gem_object_pin(obj, 4096);
4441         if (ret)
4442                 goto err_unref;
4443
4444         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4445         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4446         if (dev_priv->seqno_page == NULL)
4447                 goto err_unpin;
4448
4449         dev_priv->seqno_obj = obj;
4450         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4451
4452         return 0;
4453
4454 err_unpin:
4455         i915_gem_object_unpin(obj);
4456 err_unref:
4457         drm_gem_object_unreference(obj);
4458 err:
4459         return ret;
4460 }
4461
4462
4463 static void
4464 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4465 {
4466         drm_i915_private_t *dev_priv = dev->dev_private;
4467         struct drm_gem_object *obj;
4468         struct drm_i915_gem_object *obj_priv;
4469
4470         obj = dev_priv->seqno_obj;
4471         obj_priv = to_intel_bo(obj);
4472         kunmap(obj_priv->pages[0]);
4473         i915_gem_object_unpin(obj);
4474         drm_gem_object_unreference(obj);
4475         dev_priv->seqno_obj = NULL;
4476
4477         dev_priv->seqno_page = NULL;
4478 }
4479
4480 int
4481 i915_gem_init_ringbuffer(struct drm_device *dev)
4482 {
4483         drm_i915_private_t *dev_priv = dev->dev_private;
4484         int ret;
4485
4486         dev_priv->render_ring = render_ring;
4487
4488         if (!I915_NEED_GFX_HWS(dev)) {
4489                 dev_priv->render_ring.status_page.page_addr
4490                         = dev_priv->status_page_dmah->vaddr;
4491                 memset(dev_priv->render_ring.status_page.page_addr,
4492                                 0, PAGE_SIZE);
4493         }
4494
4495         if (HAS_PIPE_CONTROL(dev)) {
4496                 ret = i915_gem_init_pipe_control(dev);
4497                 if (ret)
4498                         return ret;
4499         }
4500
4501         ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
4502         if (ret)
4503                 goto cleanup_pipe_control;
4504
4505         if (HAS_BSD(dev)) {
4506                 dev_priv->bsd_ring = bsd_ring;
4507                 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
4508                 if (ret)
4509                         goto cleanup_render_ring;
4510         }
4511
4512         dev_priv->next_seqno = 1;
4513
4514         return 0;
4515
4516 cleanup_render_ring:
4517         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4518 cleanup_pipe_control:
4519         if (HAS_PIPE_CONTROL(dev))
4520                 i915_gem_cleanup_pipe_control(dev);
4521         return ret;
4522 }
4523
4524 void
4525 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4526 {
4527         drm_i915_private_t *dev_priv = dev->dev_private;
4528
4529         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4530         if (HAS_BSD(dev))
4531                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4532         if (HAS_PIPE_CONTROL(dev))
4533                 i915_gem_cleanup_pipe_control(dev);
4534 }
4535
4536 int
4537 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4538                        struct drm_file *file_priv)
4539 {
4540         drm_i915_private_t *dev_priv = dev->dev_private;
4541         int ret;
4542
4543         if (drm_core_check_feature(dev, DRIVER_MODESET))
4544                 return 0;
4545
4546         if (atomic_read(&dev_priv->mm.wedged)) {
4547                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4548                 atomic_set(&dev_priv->mm.wedged, 0);
4549         }
4550
4551         mutex_lock(&dev->struct_mutex);
4552         dev_priv->mm.suspended = 0;
4553
4554         ret = i915_gem_init_ringbuffer(dev);
4555         if (ret != 0) {
4556                 mutex_unlock(&dev->struct_mutex);
4557                 return ret;
4558         }
4559
4560         spin_lock(&dev_priv->mm.active_list_lock);
4561         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4562         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4563         spin_unlock(&dev_priv->mm.active_list_lock);
4564
4565         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4566         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4567         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4568         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4569         mutex_unlock(&dev->struct_mutex);
4570
4571         ret = drm_irq_install(dev);
4572         if (ret)
4573                 goto cleanup_ringbuffer;
4574
4575         return 0;
4576
4577 cleanup_ringbuffer:
4578         mutex_lock(&dev->struct_mutex);
4579         i915_gem_cleanup_ringbuffer(dev);
4580         dev_priv->mm.suspended = 1;
4581         mutex_unlock(&dev->struct_mutex);
4582
4583         return ret;
4584 }
4585
4586 int
4587 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4588                        struct drm_file *file_priv)
4589 {
4590         if (drm_core_check_feature(dev, DRIVER_MODESET))
4591                 return 0;
4592
4593         drm_irq_uninstall(dev);
4594         return i915_gem_idle(dev);
4595 }
4596
4597 void
4598 i915_gem_lastclose(struct drm_device *dev)
4599 {
4600         int ret;
4601
4602         if (drm_core_check_feature(dev, DRIVER_MODESET))
4603                 return;
4604
4605         ret = i915_gem_idle(dev);
4606         if (ret)
4607                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4608 }
4609
4610 void
4611 i915_gem_load(struct drm_device *dev)
4612 {
4613         int i;
4614         drm_i915_private_t *dev_priv = dev->dev_private;
4615
4616         spin_lock_init(&dev_priv->mm.active_list_lock);
4617         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4618         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4619         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4620         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4621         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4622         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4623         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4624         if (HAS_BSD(dev)) {
4625                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4626                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4627         }
4628         for (i = 0; i < 16; i++)
4629                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4630         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4631                           i915_gem_retire_work_handler);
4632         spin_lock(&shrink_list_lock);
4633         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4634         spin_unlock(&shrink_list_lock);
4635
4636         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4637         if (IS_GEN3(dev)) {
4638                 u32 tmp = I915_READ(MI_ARB_STATE);
4639                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4640                         /* arb state is a masked write, so set bit + bit in mask */
4641                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4642                         I915_WRITE(MI_ARB_STATE, tmp);
4643                 }
4644         }
4645
4646         /* Old X drivers will take 0-2 for front, back, depth buffers */
4647         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4648                 dev_priv->fence_reg_start = 3;
4649
4650         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4651                 dev_priv->num_fence_regs = 16;
4652         else
4653                 dev_priv->num_fence_regs = 8;
4654
4655         /* Initialize fence registers to zero */
4656         if (IS_I965G(dev)) {
4657                 for (i = 0; i < 16; i++)
4658                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4659         } else {
4660                 for (i = 0; i < 8; i++)
4661                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4662                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4663                         for (i = 0; i < 8; i++)
4664                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4665         }
4666         i915_gem_detect_bit_6_swizzle(dev);
4667         init_waitqueue_head(&dev_priv->pending_flip_queue);
4668 }
4669
4670 /*
4671  * Create a physically contiguous memory object for this object
4672  * e.g. for cursor + overlay regs
4673  */
4674 int i915_gem_init_phys_object(struct drm_device *dev,
4675                               int id, int size, int align)
4676 {
4677         drm_i915_private_t *dev_priv = dev->dev_private;
4678         struct drm_i915_gem_phys_object *phys_obj;
4679         int ret;
4680
4681         if (dev_priv->mm.phys_objs[id - 1] || !size)
4682                 return 0;
4683
4684         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4685         if (!phys_obj)
4686                 return -ENOMEM;
4687
4688         phys_obj->id = id;
4689
4690         phys_obj->handle = drm_pci_alloc(dev, size, align);
4691         if (!phys_obj->handle) {
4692                 ret = -ENOMEM;
4693                 goto kfree_obj;
4694         }
4695 #ifdef CONFIG_X86
4696         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4697 #endif
4698
4699         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4700
4701         return 0;
4702 kfree_obj:
4703         kfree(phys_obj);
4704         return ret;
4705 }
4706
4707 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4708 {
4709         drm_i915_private_t *dev_priv = dev->dev_private;
4710         struct drm_i915_gem_phys_object *phys_obj;
4711
4712         if (!dev_priv->mm.phys_objs[id - 1])
4713                 return;
4714
4715         phys_obj = dev_priv->mm.phys_objs[id - 1];
4716         if (phys_obj->cur_obj) {
4717                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4718         }
4719
4720 #ifdef CONFIG_X86
4721         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4722 #endif
4723         drm_pci_free(dev, phys_obj->handle);
4724         kfree(phys_obj);
4725         dev_priv->mm.phys_objs[id - 1] = NULL;
4726 }
4727
4728 void i915_gem_free_all_phys_object(struct drm_device *dev)
4729 {
4730         int i;
4731
4732         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4733                 i915_gem_free_phys_object(dev, i);
4734 }
4735
4736 void i915_gem_detach_phys_object(struct drm_device *dev,
4737                                  struct drm_gem_object *obj)
4738 {
4739         struct drm_i915_gem_object *obj_priv;
4740         int i;
4741         int ret;
4742         int page_count;
4743
4744         obj_priv = to_intel_bo(obj);
4745         if (!obj_priv->phys_obj)
4746                 return;
4747
4748         ret = i915_gem_object_get_pages(obj, 0);
4749         if (ret)
4750                 goto out;
4751
4752         page_count = obj->size / PAGE_SIZE;
4753
4754         for (i = 0; i < page_count; i++) {
4755                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4756                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4757
4758                 memcpy(dst, src, PAGE_SIZE);
4759                 kunmap_atomic(dst, KM_USER0);
4760         }
4761         drm_clflush_pages(obj_priv->pages, page_count);
4762         drm_agp_chipset_flush(dev);
4763
4764         i915_gem_object_put_pages(obj);
4765 out:
4766         obj_priv->phys_obj->cur_obj = NULL;
4767         obj_priv->phys_obj = NULL;
4768 }
4769
4770 int
4771 i915_gem_attach_phys_object(struct drm_device *dev,
4772                             struct drm_gem_object *obj,
4773                             int id,
4774                             int align)
4775 {
4776         drm_i915_private_t *dev_priv = dev->dev_private;
4777         struct drm_i915_gem_object *obj_priv;
4778         int ret = 0;
4779         int page_count;
4780         int i;
4781
4782         if (id > I915_MAX_PHYS_OBJECT)
4783                 return -EINVAL;
4784
4785         obj_priv = to_intel_bo(obj);
4786
4787         if (obj_priv->phys_obj) {
4788                 if (obj_priv->phys_obj->id == id)
4789                         return 0;
4790                 i915_gem_detach_phys_object(dev, obj);
4791         }
4792
4793         /* create a new object */
4794         if (!dev_priv->mm.phys_objs[id - 1]) {
4795                 ret = i915_gem_init_phys_object(dev, id,
4796                                                 obj->size, align);
4797                 if (ret) {
4798                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4799                         goto out;
4800                 }
4801         }
4802
4803         /* bind to the object */
4804         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4805         obj_priv->phys_obj->cur_obj = obj;
4806
4807         ret = i915_gem_object_get_pages(obj, 0);
4808         if (ret) {
4809                 DRM_ERROR("failed to get page list\n");
4810                 goto out;
4811         }
4812
4813         page_count = obj->size / PAGE_SIZE;
4814
4815         for (i = 0; i < page_count; i++) {
4816                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4817                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4818
4819                 memcpy(dst, src, PAGE_SIZE);
4820                 kunmap_atomic(src, KM_USER0);
4821         }
4822
4823         i915_gem_object_put_pages(obj);
4824
4825         return 0;
4826 out:
4827         return ret;
4828 }
4829
4830 static int
4831 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4832                      struct drm_i915_gem_pwrite *args,
4833                      struct drm_file *file_priv)
4834 {
4835         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4836         void *obj_addr;
4837         int ret;
4838         char __user *user_data;
4839
4840         user_data = (char __user *) (uintptr_t) args->data_ptr;
4841         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4842
4843         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4844         ret = copy_from_user(obj_addr, user_data, args->size);
4845         if (ret)
4846                 return -EFAULT;
4847
4848         drm_agp_chipset_flush(dev);
4849         return 0;
4850 }
4851
4852 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4853 {
4854         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4855
4856         /* Clean up our request list when the client is going away, so that
4857          * later retire_requests won't dereference our soon-to-be-gone
4858          * file_priv.
4859          */
4860         mutex_lock(&dev->struct_mutex);
4861         while (!list_empty(&i915_file_priv->mm.request_list))
4862                 list_del_init(i915_file_priv->mm.request_list.next);
4863         mutex_unlock(&dev->struct_mutex);
4864 }
4865
4866 static int
4867 i915_gpu_is_active(struct drm_device *dev)
4868 {
4869         drm_i915_private_t *dev_priv = dev->dev_private;
4870         int lists_empty;
4871
4872         spin_lock(&dev_priv->mm.active_list_lock);
4873         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4874                       list_empty(&dev_priv->render_ring.active_list);
4875         if (HAS_BSD(dev))
4876                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4877         spin_unlock(&dev_priv->mm.active_list_lock);
4878
4879         return !lists_empty;
4880 }
4881
4882 static int
4883 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
4884 {
4885         drm_i915_private_t *dev_priv, *next_dev;
4886         struct drm_i915_gem_object *obj_priv, *next_obj;
4887         int cnt = 0;
4888         int would_deadlock = 1;
4889
4890         /* "fast-path" to count number of available objects */
4891         if (nr_to_scan == 0) {
4892                 spin_lock(&shrink_list_lock);
4893                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4894                         struct drm_device *dev = dev_priv->dev;
4895
4896                         if (mutex_trylock(&dev->struct_mutex)) {
4897                                 list_for_each_entry(obj_priv,
4898                                                     &dev_priv->mm.inactive_list,
4899                                                     list)
4900                                         cnt++;
4901                                 mutex_unlock(&dev->struct_mutex);
4902                         }
4903                 }
4904                 spin_unlock(&shrink_list_lock);
4905
4906                 return (cnt / 100) * sysctl_vfs_cache_pressure;
4907         }
4908
4909         spin_lock(&shrink_list_lock);
4910
4911 rescan:
4912         /* first scan for clean buffers */
4913         list_for_each_entry_safe(dev_priv, next_dev,
4914                                  &shrink_list, mm.shrink_list) {
4915                 struct drm_device *dev = dev_priv->dev;
4916
4917                 if (! mutex_trylock(&dev->struct_mutex))
4918                         continue;
4919
4920                 spin_unlock(&shrink_list_lock);
4921                 i915_gem_retire_requests(dev);
4922
4923                 list_for_each_entry_safe(obj_priv, next_obj,
4924                                          &dev_priv->mm.inactive_list,
4925                                          list) {
4926                         if (i915_gem_object_is_purgeable(obj_priv)) {
4927                                 i915_gem_object_unbind(&obj_priv->base);
4928                                 if (--nr_to_scan <= 0)
4929                                         break;
4930                         }
4931                 }
4932
4933                 spin_lock(&shrink_list_lock);
4934                 mutex_unlock(&dev->struct_mutex);
4935
4936                 would_deadlock = 0;
4937
4938                 if (nr_to_scan <= 0)
4939                         break;
4940         }
4941
4942         /* second pass, evict/count anything still on the inactive list */
4943         list_for_each_entry_safe(dev_priv, next_dev,
4944                                  &shrink_list, mm.shrink_list) {
4945                 struct drm_device *dev = dev_priv->dev;
4946
4947                 if (! mutex_trylock(&dev->struct_mutex))
4948                         continue;
4949
4950                 spin_unlock(&shrink_list_lock);
4951
4952                 list_for_each_entry_safe(obj_priv, next_obj,
4953                                          &dev_priv->mm.inactive_list,
4954                                          list) {
4955                         if (nr_to_scan > 0) {
4956                                 i915_gem_object_unbind(&obj_priv->base);
4957                                 nr_to_scan--;
4958                         } else
4959                                 cnt++;
4960                 }
4961
4962                 spin_lock(&shrink_list_lock);
4963                 mutex_unlock(&dev->struct_mutex);
4964
4965                 would_deadlock = 0;
4966         }
4967
4968         if (nr_to_scan) {
4969                 int active = 0;
4970
4971                 /*
4972                  * We are desperate for pages, so as a last resort, wait
4973                  * for the GPU to finish and discard whatever we can.
4974                  * This has a dramatic impact to reduce the number of
4975                  * OOM-killer events whilst running the GPU aggressively.
4976                  */
4977                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4978                         struct drm_device *dev = dev_priv->dev;
4979
4980                         if (!mutex_trylock(&dev->struct_mutex))
4981                                 continue;
4982
4983                         spin_unlock(&shrink_list_lock);
4984
4985                         if (i915_gpu_is_active(dev)) {
4986                                 i915_gpu_idle(dev);
4987                                 active++;
4988                         }
4989
4990                         spin_lock(&shrink_list_lock);
4991                         mutex_unlock(&dev->struct_mutex);
4992                 }
4993
4994                 if (active)
4995                         goto rescan;
4996         }
4997
4998         spin_unlock(&shrink_list_lock);
4999
5000         if (would_deadlock)
5001                 return -1;
5002         else if (cnt > 0)
5003                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5004         else
5005                 return 0;
5006 }
5007
5008 static struct shrinker shrinker = {
5009         .shrink = i915_gem_shrink,
5010         .seeks = DEFAULT_SEEKS,
5011 };
5012
5013 __init void
5014 i915_gem_shrinker_init(void)
5015 {
5016     register_shrinker(&shrinker);
5017 }
5018
5019 __exit void
5020 i915_gem_shrinker_exit(void)
5021 {
5022     unregister_shrinker(&shrinker);
5023 }