drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 /*
  38  * 965+ support PIPE_CONTROL commands, which provide finer grained control
  39  * over cache flushing.
  40  */
  41 struct pipe_control {
  42         struct drm_i915_gem_object *obj;
  43         volatile u32 *cpu_page;
  44         u32 gtt_offset;
  45 };
  46
  47 static inline int ring_space(struct intel_ring_buffer *ring)
  48 {
  49         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  50         if (space < 0)
  51                 space += ring->size;
  52         return space;
  53 }
  54
  55 static int
  56 gen2_render_ring_flush(struct intel_ring_buffer *ring,
  57                        u32      invalidate_domains,
  58                        u32      flush_domains)
  59 {
  60         u32 cmd;
  61         int ret;
  62
  63         cmd = MI_FLUSH;
  64         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  65                 cmd |= MI_NO_WRITE_FLUSH;
  66
  67         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  68                 cmd |= MI_READ_FLUSH;
  69
  70         ret = intel_ring_begin(ring, 2);
  71         if (ret)
  72                 return ret;
  73
  74         intel_ring_emit(ring, cmd);
  75         intel_ring_emit(ring, MI_NOOP);
  76         intel_ring_advance(ring);
  77
  78         return 0;
  79 }
  80
  81 static int
  82 gen4_render_ring_flush(struct intel_ring_buffer *ring,
  83                        u32      invalidate_domains,
  84                        u32      flush_domains)
  85 {
  86         struct drm_device *dev = ring->dev;
  87         u32 cmd;
  88         int ret;
  89
  90         /*
  91          * read/write caches:
  92          *
  93          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  94          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  95          * also flushed at 2d versus 3d pipeline switches.
  96          *
  97          * read-only caches:
  98          *
  99          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 100          * MI_READ_FLUSH is set, and is always flushed on 965.
 101          *
 102          * I915_GEM_DOMAIN_COMMAND may not exist?
 103          *
 104          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 105          * invalidated when MI_EXE_FLUSH is set.
 106          *
 107          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 108          * invalidated with every MI_FLUSH.
 109          *
 110          * TLBs:
 111          *
 112          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 113          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 114          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 115          * are flushed at any MI_FLUSH.
 116          */
 117
 118         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 119         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 120                 cmd &= ~MI_NO_WRITE_FLUSH;
 121         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 122                 cmd |= MI_EXE_FLUSH;
 123
 124         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 125             (IS_G4X(dev) || IS_GEN5(dev)))
 126                 cmd |= MI_INVALIDATE_ISP;
 127
 128         ret = intel_ring_begin(ring, 2);
 129         if (ret)
 130                 return ret;
 131
 132         intel_ring_emit(ring, cmd);
 133         intel_ring_emit(ring, MI_NOOP);
 134         intel_ring_advance(ring);
 135
 136         return 0;
 137 }
 138
 139 /**
 140  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 141  * implementing two workarounds on gen6.  From section 1.4.7.1
 142  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 143  *
 144  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 145  * produced by non-pipelined state commands), software needs to first
 146  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 147  * 0.
 148  *
 149  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 150  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 151  *
 152  * And the workaround for these two requires this workaround first:
 153  *
 154  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 155  * BEFORE the pipe-control with a post-sync op and no write-cache
 156  * flushes.
 157  *
 158  * And this last workaround is tricky because of the requirements on
 159  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 160  * volume 2 part 1:
 161  *
 162  *     "1 of the following must also be set:
 163  *      - Render Target Cache Flush Enable ([12] of DW1)
 164  *      - Depth Cache Flush Enable ([0] of DW1)
 165  *      - Stall at Pixel Scoreboard ([1] of DW1)
 166  *      - Depth Stall ([13] of DW1)
 167  *      - Post-Sync Operation ([13] of DW1)
 168  *      - Notify Enable ([8] of DW1)"
 169  *
 170  * The cache flushes require the workaround flush that triggered this
 171  * one, so we can't use it.  Depth stall would trigger the same.
 172  * Post-sync nonzero is what triggered this second workaround, so we
 173  * can't use that one either.  Notify enable is IRQs, which aren't
 174  * really our business.  That leaves only stall at scoreboard.
 175  */
 176 static int
 177 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
 178 {
 179         struct pipe_control *pc = ring->private;
 180         u32 scratch_addr = pc->gtt_offset + 128;
 181         int ret;
 182
 183
 184         ret = intel_ring_begin(ring, 6);
 185         if (ret)
 186                 return ret;
 187
 188         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 189         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 190                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 191         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 192         intel_ring_emit(ring, 0); /* low dword */
 193         intel_ring_emit(ring, 0); /* high dword */
 194         intel_ring_emit(ring, MI_NOOP);
 195         intel_ring_advance(ring);
 196
 197         ret = intel_ring_begin(ring, 6);
 198         if (ret)
 199                 return ret;
 200
 201         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 202         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 203         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 204         intel_ring_emit(ring, 0);
 205         intel_ring_emit(ring, 0);
 206         intel_ring_emit(ring, MI_NOOP);
 207         intel_ring_advance(ring);
 208
 209         return 0;
 210 }
 211
 212 static int
 213 gen6_render_ring_flush(struct intel_ring_buffer *ring,
 214                          u32 invalidate_domains, u32 flush_domains)
 215 {
 216         u32 flags = 0;
 217         struct pipe_control *pc = ring->private;
 218         u32 scratch_addr = pc->gtt_offset + 128;
 219         int ret;
 220
 221         /* Force SNB workarounds for PIPE_CONTROL flushes */
 222         intel_emit_post_sync_nonzero_flush(ring);
 223
 224         /* Just flush everything.  Experiments have shown that reducing the
 225          * number of bits based on the write domains has little performance
 226          * impact.
 227          */
 228         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 229         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 230         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 231         flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 232         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 233         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 234         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 235
 236         ret = intel_ring_begin(ring, 6);
 237         if (ret)
 238                 return ret;
 239
 240         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 241         intel_ring_emit(ring, flags);
 242         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 243         intel_ring_emit(ring, 0); /* lower dword */
 244         intel_ring_emit(ring, 0); /* uppwer dword */
 245         intel_ring_emit(ring, MI_NOOP);
 246         intel_ring_advance(ring);
 247
 248         return 0;
 249 }
 250
 251 static void ring_write_tail(struct intel_ring_buffer *ring,
 252                             u32 value)
 253 {
 254         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 255         I915_WRITE_TAIL(ring, value);
 256 }
 257
 258 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 259 {
 260         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 261         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 262                         RING_ACTHD(ring->mmio_base) : ACTHD;
 263
 264         return I915_READ(acthd_reg);
 265 }
 266
 267 static int init_ring_common(struct intel_ring_buffer *ring)
 268 {
 269         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 270         struct drm_i915_gem_object *obj = ring->obj;
 271         u32 head;
 272
 273         /* Stop the ring if it's running. */
 274         I915_WRITE_CTL(ring, 0);
 275         I915_WRITE_HEAD(ring, 0);
 276         ring->write_tail(ring, 0);
 277
 278         /* Initialize the ring. */
 279         I915_WRITE_START(ring, obj->gtt_offset);
 280         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 281
 282         /* G45 ring initialization fails to reset head to zero */
 283         if (head != 0) {
 284                 DRM_DEBUG_KMS("%s head not reset to zero "
 285                               "ctl %08x head %08x tail %08x start %08x\n",
 286                               ring->name,
 287                               I915_READ_CTL(ring),
 288                               I915_READ_HEAD(ring),
 289                               I915_READ_TAIL(ring),
 290                               I915_READ_START(ring));
 291
 292                 I915_WRITE_HEAD(ring, 0);
 293
 294                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
 295                         DRM_ERROR("failed to set %s head to zero "
 296                                   "ctl %08x head %08x tail %08x start %08x\n",
 297                                   ring->name,
 298                                   I915_READ_CTL(ring),
 299                                   I915_READ_HEAD(ring),
 300                                   I915_READ_TAIL(ring),
 301                                   I915_READ_START(ring));
 302                 }
 303         }
 304
 305         I915_WRITE_CTL(ring,
 306                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 307                         | RING_VALID);
 308
 309         /* If the head is still not zero, the ring is dead */
 310         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
 311                      I915_READ_START(ring) == obj->gtt_offset &&
 312                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
 313                 DRM_ERROR("%s initialization failed "
 314                                 "ctl %08x head %08x tail %08x start %08x\n",
 315                                 ring->name,
 316                                 I915_READ_CTL(ring),
 317                                 I915_READ_HEAD(ring),
 318                                 I915_READ_TAIL(ring),
 319                                 I915_READ_START(ring));
 320                 return -EIO;
 321         }
 322
 323         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 324                 i915_kernel_lost_context(ring->dev);
 325         else {
 326                 ring->head = I915_READ_HEAD(ring);
 327                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 328                 ring->space = ring_space(ring);
 329                 ring->last_retired_head = -1;
 330         }
 331
 332         return 0;
 333 }
 334
 335 static int
 336 init_pipe_control(struct intel_ring_buffer *ring)
 337 {
 338         struct pipe_control *pc;
 339         struct drm_i915_gem_object *obj;
 340         int ret;
 341
 342         if (ring->private)
 343                 return 0;
 344
 345         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
 346         if (!pc)
 347                 return -ENOMEM;
 348
 349         obj = i915_gem_alloc_object(ring->dev, 4096);
 350         if (obj == NULL) {
 351                 DRM_ERROR("Failed to allocate seqno page\n");
 352                 ret = -ENOMEM;
 353                 goto err;
 354         }
 355
 356         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 357
 358         ret = i915_gem_object_pin(obj, 4096, true);
 359         if (ret)
 360                 goto err_unref;
 361
 362         pc->gtt_offset = obj->gtt_offset;
 363         pc->cpu_page =  kmap(obj->pages[0]);
 364         if (pc->cpu_page == NULL)
 365                 goto err_unpin;
 366
 367         pc->obj = obj;
 368         ring->private = pc;
 369         return 0;
 370
 371 err_unpin:
 372         i915_gem_object_unpin(obj);
 373 err_unref:
 374         drm_gem_object_unreference(&obj->base);
 375 err:
 376         kfree(pc);
 377         return ret;
 378 }
 379
 380 static void
 381 cleanup_pipe_control(struct intel_ring_buffer *ring)
 382 {
 383         struct pipe_control *pc = ring->private;
 384         struct drm_i915_gem_object *obj;
 385
 386         if (!ring->private)
 387                 return;
 388
 389         obj = pc->obj;
 390         kunmap(obj->pages[0]);
 391         i915_gem_object_unpin(obj);
 392         drm_gem_object_unreference(&obj->base);
 393
 394         kfree(pc);
 395         ring->private = NULL;
 396 }
 397
 398 static int init_render_ring(struct intel_ring_buffer *ring)
 399 {
 400         struct drm_device *dev = ring->dev;
 401         struct drm_i915_private *dev_priv = dev->dev_private;
 402         int ret = init_ring_common(ring);
 403
 404         if (INTEL_INFO(dev)->gen > 3) {
 405                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 406                 if (IS_GEN7(dev))
 407                         I915_WRITE(GFX_MODE_GEN7,
 408                                    _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
 409                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 410         }
 411
 412         if (INTEL_INFO(dev)->gen >= 5) {
 413                 ret = init_pipe_control(ring);
 414                 if (ret)
 415                         return ret;
 416         }
 417
 418         if (IS_GEN6(dev)) {
 419                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
 420                  * "If this bit is set, STCunit will have LRA as replacement
 421                  *  policy. [...] This bit must be reset.  LRA replacement
 422                  *  policy is not supported."
 423                  */
 424                 I915_WRITE(CACHE_MODE_0,
 425                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
 426         }
 427
 428         if (INTEL_INFO(dev)->gen >= 6)
 429                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 430
 431         return ret;
 432 }
 433
 434 static void render_ring_cleanup(struct intel_ring_buffer *ring)
 435 {
 436         if (!ring->private)
 437                 return;
 438
 439         cleanup_pipe_control(ring);
 440 }
 441
 442 static void
 443 update_mboxes(struct intel_ring_buffer *ring,
 444             u32 seqno,
 445             u32 mmio_offset)
 446 {
 447         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
 448                               MI_SEMAPHORE_GLOBAL_GTT |
 449                               MI_SEMAPHORE_REGISTER |
 450                               MI_SEMAPHORE_UPDATE);
 451         intel_ring_emit(ring, seqno);
 452         intel_ring_emit(ring, mmio_offset);
 453 }
 454
 455 /**
 456  * gen6_add_request - Update the semaphore mailbox registers
 457  *
 458  * @ring - ring that is adding a request
 459  * @seqno - return seqno stuck into the ring
 460  *
 461  * Update the mailbox registers in the *other* rings with the current seqno.
 462  * This acts like a signal in the canonical semaphore.
 463  */
 464 static int
 465 gen6_add_request(struct intel_ring_buffer *ring,
 466                  u32 *seqno)
 467 {
 468         u32 mbox1_reg;
 469         u32 mbox2_reg;
 470         int ret;
 471
 472         ret = intel_ring_begin(ring, 10);
 473         if (ret)
 474                 return ret;
 475
 476         mbox1_reg = ring->signal_mbox[0];
 477         mbox2_reg = ring->signal_mbox[1];
 478
 479         *seqno = i915_gem_next_request_seqno(ring);
 480
 481         update_mboxes(ring, *seqno, mbox1_reg);
 482         update_mboxes(ring, *seqno, mbox2_reg);
 483         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 484         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 485         intel_ring_emit(ring, *seqno);
 486         intel_ring_emit(ring, MI_USER_INTERRUPT);
 487         intel_ring_advance(ring);
 488
 489         return 0;
 490 }
 491
 492 /**
 493  * intel_ring_sync - sync the waiter to the signaller on seqno
 494  *
 495  * @waiter - ring that is waiting
 496  * @signaller - ring which has, or will signal
 497  * @seqno - seqno which the waiter will block on
 498  */
 499 static int
 500 gen6_ring_sync(struct intel_ring_buffer *waiter,
 501                struct intel_ring_buffer *signaller,
 502                u32 seqno)
 503 {
 504         int ret;
 505         u32 dw1 = MI_SEMAPHORE_MBOX |
 506                   MI_SEMAPHORE_COMPARE |
 507                   MI_SEMAPHORE_REGISTER;
 508
 509         /* Throughout all of the GEM code, seqno passed implies our current
 510          * seqno is >= the last seqno executed. However for hardware the
 511          * comparison is strictly greater than.
 512          */
 513         seqno -= 1;
 514
 515         WARN_ON(signaller->semaphore_register[waiter->id] ==
 516                 MI_SEMAPHORE_SYNC_INVALID);
 517
 518         ret = intel_ring_begin(waiter, 4);
 519         if (ret)
 520                 return ret;
 521
 522         intel_ring_emit(waiter,
 523                         dw1 | signaller->semaphore_register[waiter->id]);
 524         intel_ring_emit(waiter, seqno);
 525         intel_ring_emit(waiter, 0);
 526         intel_ring_emit(waiter, MI_NOOP);
 527         intel_ring_advance(waiter);
 528
 529         return 0;
 530 }
 531
 532 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 533 do {                                                                    \
 534         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
 535                  PIPE_CONTROL_DEPTH_STALL);                             \
 536         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 537         intel_ring_emit(ring__, 0);                                                     \
 538         intel_ring_emit(ring__, 0);                                                     \
 539 } while (0)
 540
 541 static int
 542 pc_render_add_request(struct intel_ring_buffer *ring,
 543                       u32 *result)
 544 {
 545         u32 seqno = i915_gem_next_request_seqno(ring);
 546         struct pipe_control *pc = ring->private;
 547         u32 scratch_addr = pc->gtt_offset + 128;
 548         int ret;
 549
 550         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
 551          * incoherent with writes to memory, i.e. completely fubar,
 552          * so we need to use PIPE_NOTIFY instead.
 553          *
 554          * However, we also need to workaround the qword write
 555          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
 556          * memory before requesting an interrupt.
 557          */
 558         ret = intel_ring_begin(ring, 32);
 559         if (ret)
 560                 return ret;
 561
 562         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 563                         PIPE_CONTROL_WRITE_FLUSH |
 564                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 565         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 566         intel_ring_emit(ring, seqno);
 567         intel_ring_emit(ring, 0);
 568         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 569         scratch_addr += 128; /* write to separate cachelines */
 570         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 571         scratch_addr += 128;
 572         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 573         scratch_addr += 128;
 574         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 575         scratch_addr += 128;
 576         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 577         scratch_addr += 128;
 578         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 579
 580         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 581                         PIPE_CONTROL_WRITE_FLUSH |
 582                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 583                         PIPE_CONTROL_NOTIFY);
 584         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 585         intel_ring_emit(ring, seqno);
 586         intel_ring_emit(ring, 0);
 587         intel_ring_advance(ring);
 588
 589         *result = seqno;
 590         return 0;
 591 }
 592
 593 static u32
 594 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
 595 {
 596         struct drm_device *dev = ring->dev;
 597
 598         /* Workaround to force correct ordering between irq and seqno writes on
 599          * ivb (and maybe also on snb) by reading from a CS register (like
 600          * ACTHD) before reading the status page. */
 601         if (IS_GEN6(dev) || IS_GEN7(dev))
 602                 intel_ring_get_active_head(ring);
 603         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 604 }
 605
 606 static u32
 607 ring_get_seqno(struct intel_ring_buffer *ring)
 608 {
 609         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 610 }
 611
 612 static u32
 613 pc_render_get_seqno(struct intel_ring_buffer *ring)
 614 {
 615         struct pipe_control *pc = ring->private;
 616         return pc->cpu_page[0];
 617 }
 618
 619 static bool
 620 gen5_ring_get_irq(struct intel_ring_buffer *ring)
 621 {
 622         struct drm_device *dev = ring->dev;
 623         drm_i915_private_t *dev_priv = dev->dev_private;
 624         unsigned long flags;
 625
 626         if (!dev->irq_enabled)
 627                 return false;
 628
 629         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 630         if (ring->irq_refcount++ == 0) {
 631                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 632                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 633                 POSTING_READ(GTIMR);
 634         }
 635         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 636
 637         return true;
 638 }
 639
 640 static void
 641 gen5_ring_put_irq(struct intel_ring_buffer *ring)
 642 {
 643         struct drm_device *dev = ring->dev;
 644         drm_i915_private_t *dev_priv = dev->dev_private;
 645         unsigned long flags;
 646
 647         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 648         if (--ring->irq_refcount == 0) {
 649                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 650                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 651                 POSTING_READ(GTIMR);
 652         }
 653         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 654 }
 655
 656 static bool
 657 i9xx_ring_get_irq(struct intel_ring_buffer *ring)
 658 {
 659         struct drm_device *dev = ring->dev;
 660         drm_i915_private_t *dev_priv = dev->dev_private;
 661         unsigned long flags;
 662
 663         if (!dev->irq_enabled)
 664                 return false;
 665
 666         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 667         if (ring->irq_refcount++ == 0) {
 668                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 669                 I915_WRITE(IMR, dev_priv->irq_mask);
 670                 POSTING_READ(IMR);
 671         }
 672         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 673
 674         return true;
 675 }
 676
 677 static void
 678 i9xx_ring_put_irq(struct intel_ring_buffer *ring)
 679 {
 680         struct drm_device *dev = ring->dev;
 681         drm_i915_private_t *dev_priv = dev->dev_private;
 682         unsigned long flags;
 683
 684         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 685         if (--ring->irq_refcount == 0) {
 686                 dev_priv->irq_mask |= ring->irq_enable_mask;
 687                 I915_WRITE(IMR, dev_priv->irq_mask);
 688                 POSTING_READ(IMR);
 689         }
 690         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 691 }
 692
 693 static bool
 694 i8xx_ring_get_irq(struct intel_ring_buffer *ring)
 695 {
 696         struct drm_device *dev = ring->dev;
 697         drm_i915_private_t *dev_priv = dev->dev_private;
 698         unsigned long flags;
 699
 700         if (!dev->irq_enabled)
 701                 return false;
 702
 703         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 704         if (ring->irq_refcount++ == 0) {
 705                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 706                 I915_WRITE16(IMR, dev_priv->irq_mask);
 707                 POSTING_READ16(IMR);
 708         }
 709         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 710
 711         return true;
 712 }
 713
 714 static void
 715 i8xx_ring_put_irq(struct intel_ring_buffer *ring)
 716 {
 717         struct drm_device *dev = ring->dev;
 718         drm_i915_private_t *dev_priv = dev->dev_private;
 719         unsigned long flags;
 720
 721         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 722         if (--ring->irq_refcount == 0) {
 723                 dev_priv->irq_mask |= ring->irq_enable_mask;
 724                 I915_WRITE16(IMR, dev_priv->irq_mask);
 725                 POSTING_READ16(IMR);
 726         }
 727         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 728 }
 729
 730 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 731 {
 732         struct drm_device *dev = ring->dev;
 733         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 734         u32 mmio = 0;
 735
 736         /* The ring status page addresses are no longer next to the rest of
 737          * the ring registers as of gen7.
 738          */
 739         if (IS_GEN7(dev)) {
 740                 switch (ring->id) {
 741                 case RCS:
 742                         mmio = RENDER_HWS_PGA_GEN7;
 743                         break;
 744                 case BCS:
 745                         mmio = BLT_HWS_PGA_GEN7;
 746                         break;
 747                 case VCS:
 748                         mmio = BSD_HWS_PGA_GEN7;
 749                         break;
 750                 }
 751         } else if (IS_GEN6(ring->dev)) {
 752                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
 753         } else {
 754                 mmio = RING_HWS_PGA(ring->mmio_base);
 755         }
 756
 757         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 758         POSTING_READ(mmio);
 759 }
 760
 761 static int
 762 bsd_ring_flush(struct intel_ring_buffer *ring,
 763                u32     invalidate_domains,
 764                u32     flush_domains)
 765 {
 766         int ret;
 767
 768         ret = intel_ring_begin(ring, 2);
 769         if (ret)
 770                 return ret;
 771
 772         intel_ring_emit(ring, MI_FLUSH);
 773         intel_ring_emit(ring, MI_NOOP);
 774         intel_ring_advance(ring);
 775         return 0;
 776 }
 777
 778 static int
 779 i9xx_add_request(struct intel_ring_buffer *ring,
 780                  u32 *result)
 781 {
 782         u32 seqno;
 783         int ret;
 784
 785         ret = intel_ring_begin(ring, 4);
 786         if (ret)
 787                 return ret;
 788
 789         seqno = i915_gem_next_request_seqno(ring);
 790
 791         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 792         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 793         intel_ring_emit(ring, seqno);
 794         intel_ring_emit(ring, MI_USER_INTERRUPT);
 795         intel_ring_advance(ring);
 796
 797         *result = seqno;
 798         return 0;
 799 }
 800
 801 static bool
 802 gen6_ring_get_irq(struct intel_ring_buffer *ring)
 803 {
 804         struct drm_device *dev = ring->dev;
 805         drm_i915_private_t *dev_priv = dev->dev_private;
 806         unsigned long flags;
 807
 808         if (!dev->irq_enabled)
 809                return false;
 810
 811         /* It looks like we need to prevent the gt from suspending while waiting
 812          * for an notifiy irq, otherwise irqs seem to get lost on at least the
 813          * blt/bsd rings on ivb. */
 814         gen6_gt_force_wake_get(dev_priv);
 815
 816         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 817         if (ring->irq_refcount++ == 0) {
 818                 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
 819                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 820                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 821                 POSTING_READ(GTIMR);
 822         }
 823         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 824
 825         return true;
 826 }
 827
 828 static void
 829 gen6_ring_put_irq(struct intel_ring_buffer *ring)
 830 {
 831         struct drm_device *dev = ring->dev;
 832         drm_i915_private_t *dev_priv = dev->dev_private;
 833         unsigned long flags;
 834
 835         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 836         if (--ring->irq_refcount == 0) {
 837                 I915_WRITE_IMR(ring, ~0);
 838                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 839                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 840                 POSTING_READ(GTIMR);
 841         }
 842         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 843
 844         gen6_gt_force_wake_put(dev_priv);
 845 }
 846
 847 static int
 848 i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 849 {
 850         int ret;
 851
 852         ret = intel_ring_begin(ring, 2);
 853         if (ret)
 854                 return ret;
 855
 856         intel_ring_emit(ring,
 857                         MI_BATCH_BUFFER_START |
 858                         MI_BATCH_GTT |
 859                         MI_BATCH_NON_SECURE_I965);
 860         intel_ring_emit(ring, offset);
 861         intel_ring_advance(ring);
 862
 863         return 0;
 864 }
 865
 866 static int
 867 i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
 868                                 u32 offset, u32 len)
 869 {
 870         int ret;
 871
 872         ret = intel_ring_begin(ring, 4);
 873         if (ret)
 874                 return ret;
 875
 876         intel_ring_emit(ring, MI_BATCH_BUFFER);
 877         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 878         intel_ring_emit(ring, offset + len - 8);
 879         intel_ring_emit(ring, 0);
 880         intel_ring_advance(ring);
 881
 882         return 0;
 883 }
 884
 885 static int
 886 i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
 887                                 u32 offset, u32 len)
 888 {
 889         int ret;
 890
 891         ret = intel_ring_begin(ring, 2);
 892         if (ret)
 893                 return ret;
 894
 895         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
 896         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 897         intel_ring_advance(ring);
 898
 899         return 0;
 900 }
 901
 902 static void cleanup_status_page(struct intel_ring_buffer *ring)
 903 {
 904         struct drm_i915_gem_object *obj;
 905
 906         obj = ring->status_page.obj;
 907         if (obj == NULL)
 908                 return;
 909
 910         kunmap(obj->pages[0]);
 911         i915_gem_object_unpin(obj);
 912         drm_gem_object_unreference(&obj->base);
 913         ring->status_page.obj = NULL;
 914 }
 915
 916 static int init_status_page(struct intel_ring_buffer *ring)
 917 {
 918         struct drm_device *dev = ring->dev;
 919         struct drm_i915_gem_object *obj;
 920         int ret;
 921
 922         obj = i915_gem_alloc_object(dev, 4096);
 923         if (obj == NULL) {
 924                 DRM_ERROR("Failed to allocate status page\n");
 925                 ret = -ENOMEM;
 926                 goto err;
 927         }
 928
 929         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 930
 931         ret = i915_gem_object_pin(obj, 4096, true);
 932         if (ret != 0) {
 933                 goto err_unref;
 934         }
 935
 936         ring->status_page.gfx_addr = obj->gtt_offset;
 937         ring->status_page.page_addr = kmap(obj->pages[0]);
 938         if (ring->status_page.page_addr == NULL) {
 939                 goto err_unpin;
 940         }
 941         ring->status_page.obj = obj;
 942         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 943
 944         intel_ring_setup_status_page(ring);
 945         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 946                         ring->name, ring->status_page.gfx_addr);
 947
 948         return 0;
 949
 950 err_unpin:
 951         i915_gem_object_unpin(obj);
 952 err_unref:
 953         drm_gem_object_unreference(&obj->base);
 954 err:
 955         return ret;
 956 }
 957
 958 static int intel_init_ring_buffer(struct drm_device *dev,
 959                                   struct intel_ring_buffer *ring)
 960 {
 961         struct drm_i915_gem_object *obj;
 962         int ret;
 963
 964         ring->dev = dev;
 965         INIT_LIST_HEAD(&ring->active_list);
 966         INIT_LIST_HEAD(&ring->request_list);
 967         INIT_LIST_HEAD(&ring->gpu_write_list);
 968         ring->size = 32 * PAGE_SIZE;
 969
 970         init_waitqueue_head(&ring->irq_queue);
 971
 972         if (I915_NEED_GFX_HWS(dev)) {
 973                 ret = init_status_page(ring);
 974                 if (ret)
 975                         return ret;
 976         }
 977
 978         obj = i915_gem_alloc_object(dev, ring->size);
 979         if (obj == NULL) {
 980                 DRM_ERROR("Failed to allocate ringbuffer\n");
 981                 ret = -ENOMEM;
 982                 goto err_hws;
 983         }
 984
 985         ring->obj = obj;
 986
 987         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
 988         if (ret)
 989                 goto err_unref;
 990
 991         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 992         if (ret)
 993                 goto err_unpin;
 994
 995         ring->virtual_start = ioremap_wc(dev->agp->base + obj->gtt_offset,
 996                                          ring->size);
 997         if (ring->virtual_start == NULL) {
 998                 DRM_ERROR("Failed to map ringbuffer.\n");
 999                 ret = -EINVAL;
1000                 goto err_unpin;
1001         }
1002
1003         ret = ring->init(ring);
1004         if (ret)
1005                 goto err_unmap;
1006
1007         /* Workaround an erratum on the i830 which causes a hang if
1008          * the TAIL pointer points to within the last 2 cachelines
1009          * of the buffer.
1010          */
1011         ring->effective_size = ring->size;
1012         if (IS_I830(ring->dev) || IS_845G(ring->dev))
1013                 ring->effective_size -= 128;
1014
1015         return 0;
1016
1017 err_unmap:
1018         iounmap(ring->virtual_start);
1019 err_unpin:
1020         i915_gem_object_unpin(obj);
1021 err_unref:
1022         drm_gem_object_unreference(&obj->base);
1023         ring->obj = NULL;
1024 err_hws:
1025         cleanup_status_page(ring);
1026         return ret;
1027 }
1028
1029 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1030 {
1031         struct drm_i915_private *dev_priv;
1032         int ret;
1033
1034         if (ring->obj == NULL)
1035                 return;
1036
1037         /* Disable the ring buffer. The ring must be idle at this point */
1038         dev_priv = ring->dev->dev_private;
1039         ret = intel_wait_ring_idle(ring);
1040         if (ret)
1041                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1042                           ring->name, ret);
1043
1044         I915_WRITE_CTL(ring, 0);
1045
1046         iounmap(ring->virtual_start);
1047
1048         i915_gem_object_unpin(ring->obj);
1049         drm_gem_object_unreference(&ring->obj->base);
1050         ring->obj = NULL;
1051
1052         if (ring->cleanup)
1053                 ring->cleanup(ring);
1054
1055         cleanup_status_page(ring);
1056 }
1057
1058 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1059 {
1060         uint32_t __iomem *virt;
1061         int rem = ring->size - ring->tail;
1062
1063         if (ring->space < rem) {
1064                 int ret = intel_wait_ring_buffer(ring, rem);
1065                 if (ret)
1066                         return ret;
1067         }
1068
1069         virt = ring->virtual_start + ring->tail;
1070         rem /= 4;
1071         while (rem--)
1072                 iowrite32(MI_NOOP, virt++);
1073
1074         ring->tail = 0;
1075         ring->space = ring_space(ring);
1076
1077         return 0;
1078 }
1079
1080 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1081 {
1082         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1083         bool was_interruptible;
1084         int ret;
1085
1086         /* XXX As we have not yet audited all the paths to check that
1087          * they are ready for ERESTARTSYS from intel_ring_begin, do not
1088          * allow us to be interruptible by a signal.
1089          */
1090         was_interruptible = dev_priv->mm.interruptible;
1091         dev_priv->mm.interruptible = false;
1092
1093         ret = i915_wait_request(ring, seqno);
1094
1095         dev_priv->mm.interruptible = was_interruptible;
1096         if (!ret)
1097                 i915_gem_retire_requests_ring(ring);
1098
1099         return ret;
1100 }
1101
1102 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1103 {
1104         struct drm_i915_gem_request *request;
1105         u32 seqno = 0;
1106         int ret;
1107
1108         i915_gem_retire_requests_ring(ring);
1109
1110         if (ring->last_retired_head != -1) {
1111                 ring->head = ring->last_retired_head;
1112                 ring->last_retired_head = -1;
1113                 ring->space = ring_space(ring);
1114                 if (ring->space >= n)
1115                         return 0;
1116         }
1117
1118         list_for_each_entry(request, &ring->request_list, list) {
1119                 int space;
1120
1121                 if (request->tail == -1)
1122                         continue;
1123
1124                 space = request->tail - (ring->tail + 8);
1125                 if (space < 0)
1126                         space += ring->size;
1127                 if (space >= n) {
1128                         seqno = request->seqno;
1129                         break;
1130                 }
1131
1132                 /* Consume this request in case we need more space than
1133                  * is available and so need to prevent a race between
1134                  * updating last_retired_head and direct reads of
1135                  * I915_RING_HEAD. It also provides a nice sanity check.
1136                  */
1137                 request->tail = -1;
1138         }
1139
1140         if (seqno == 0)
1141                 return -ENOSPC;
1142
1143         ret = intel_ring_wait_seqno(ring, seqno);
1144         if (ret)
1145                 return ret;
1146
1147         if (WARN_ON(ring->last_retired_head == -1))
1148                 return -ENOSPC;
1149
1150         ring->head = ring->last_retired_head;
1151         ring->last_retired_head = -1;
1152         ring->space = ring_space(ring);
1153         if (WARN_ON(ring->space < n))
1154                 return -ENOSPC;
1155
1156         return 0;
1157 }
1158
1159 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1160 {
1161         struct drm_device *dev = ring->dev;
1162         struct drm_i915_private *dev_priv = dev->dev_private;
1163         unsigned long end;
1164         int ret;
1165
1166         ret = intel_ring_wait_request(ring, n);
1167         if (ret != -ENOSPC)
1168                 return ret;
1169
1170         trace_i915_ring_wait_begin(ring);
1171         /* With GEM the hangcheck timer should kick us out of the loop,
1172          * leaving it early runs the risk of corrupting GEM state (due
1173          * to running on almost untested codepaths). But on resume
1174          * timers don't work yet, so prevent a complete hang in that
1175          * case by choosing an insanely large timeout. */
1176         end = jiffies + 60 * HZ;
1177
1178         do {
1179                 ring->head = I915_READ_HEAD(ring);
1180                 ring->space = ring_space(ring);
1181                 if (ring->space >= n) {
1182                         trace_i915_ring_wait_end(ring);
1183                         return 0;
1184                 }
1185
1186                 if (dev->primary->master) {
1187                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1188                         if (master_priv->sarea_priv)
1189                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1190                 }
1191
1192                 msleep(1);
1193                 if (atomic_read(&dev_priv->mm.wedged))
1194                         return -EAGAIN;
1195         } while (!time_after(jiffies, end));
1196         trace_i915_ring_wait_end(ring);
1197         return -EBUSY;
1198 }
1199
1200 int intel_ring_begin(struct intel_ring_buffer *ring,
1201                      int num_dwords)
1202 {
1203         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1204         int n = 4*num_dwords;
1205         int ret;
1206
1207         if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1208                 return -EIO;
1209
1210         if (unlikely(ring->tail + n > ring->effective_size)) {
1211                 ret = intel_wrap_ring_buffer(ring);
1212                 if (unlikely(ret))
1213                         return ret;
1214         }
1215
1216         if (unlikely(ring->space < n)) {
1217                 ret = intel_wait_ring_buffer(ring, n);
1218                 if (unlikely(ret))
1219                         return ret;
1220         }
1221
1222         ring->space -= n;
1223         return 0;
1224 }
1225
1226 void intel_ring_advance(struct intel_ring_buffer *ring)
1227 {
1228         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1229
1230         ring->tail &= ring->size - 1;
1231         if (dev_priv->stop_rings & intel_ring_flag(ring))
1232                 return;
1233         ring->write_tail(ring, ring->tail);
1234 }
1235
1236
1237 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1238                                      u32 value)
1239 {
1240         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1241
1242        /* Every tail move must follow the sequence below */
1243         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1244                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1245                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1246         I915_WRITE(GEN6_BSD_RNCID, 0x0);
1247
1248         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1249                 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1250                 50))
1251         DRM_ERROR("timed out waiting for IDLE Indicator\n");
1252
1253         I915_WRITE_TAIL(ring, value);
1254         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1255                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1256                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1257 }
1258
1259 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1260                            u32 invalidate, u32 flush)
1261 {
1262         uint32_t cmd;
1263         int ret;
1264
1265         ret = intel_ring_begin(ring, 4);
1266         if (ret)
1267                 return ret;
1268
1269         cmd = MI_FLUSH_DW;
1270         if (invalidate & I915_GEM_GPU_DOMAINS)
1271                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1272         intel_ring_emit(ring, cmd);
1273         intel_ring_emit(ring, 0);
1274         intel_ring_emit(ring, 0);
1275         intel_ring_emit(ring, MI_NOOP);
1276         intel_ring_advance(ring);
1277         return 0;
1278 }
1279
1280 static int
1281 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1282                               u32 offset, u32 len)
1283 {
1284         int ret;
1285
1286         ret = intel_ring_begin(ring, 2);
1287         if (ret)
1288                 return ret;
1289
1290         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1291         /* bit0-7 is the length on GEN6+ */
1292         intel_ring_emit(ring, offset);
1293         intel_ring_advance(ring);
1294
1295         return 0;
1296 }
1297
1298 /* Blitter support (SandyBridge+) */
1299
1300 static int blt_ring_flush(struct intel_ring_buffer *ring,
1301                           u32 invalidate, u32 flush)
1302 {
1303         uint32_t cmd;
1304         int ret;
1305
1306         ret = intel_ring_begin(ring, 4);
1307         if (ret)
1308                 return ret;
1309
1310         cmd = MI_FLUSH_DW;
1311         if (invalidate & I915_GEM_DOMAIN_RENDER)
1312                 cmd |= MI_INVALIDATE_TLB;
1313         intel_ring_emit(ring, cmd);
1314         intel_ring_emit(ring, 0);
1315         intel_ring_emit(ring, 0);
1316         intel_ring_emit(ring, MI_NOOP);
1317         intel_ring_advance(ring);
1318         return 0;
1319 }
1320
1321 int intel_init_render_ring_buffer(struct drm_device *dev)
1322 {
1323         drm_i915_private_t *dev_priv = dev->dev_private;
1324         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1325
1326         ring->name = "render ring";
1327         ring->id = RCS;
1328         ring->mmio_base = RENDER_RING_BASE;
1329
1330         if (INTEL_INFO(dev)->gen >= 6) {
1331                 ring->add_request = gen6_add_request;
1332                 ring->flush = gen6_render_ring_flush;
1333                 ring->irq_get = gen6_ring_get_irq;
1334                 ring->irq_put = gen6_ring_put_irq;
1335                 ring->irq_enable_mask = GT_USER_INTERRUPT;
1336                 ring->get_seqno = gen6_ring_get_seqno;
1337                 ring->sync_to = gen6_ring_sync;
1338                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
1339                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
1340                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
1341                 ring->signal_mbox[0] = GEN6_VRSYNC;
1342                 ring->signal_mbox[1] = GEN6_BRSYNC;
1343         } else if (IS_GEN5(dev)) {
1344                 ring->add_request = pc_render_add_request;
1345                 ring->flush = gen4_render_ring_flush;
1346                 ring->get_seqno = pc_render_get_seqno;
1347                 ring->irq_get = gen5_ring_get_irq;
1348                 ring->irq_put = gen5_ring_put_irq;
1349                 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
1350         } else {
1351                 ring->add_request = i9xx_add_request;
1352                 if (INTEL_INFO(dev)->gen < 4)
1353                         ring->flush = gen2_render_ring_flush;
1354                 else
1355                         ring->flush = gen4_render_ring_flush;
1356                 ring->get_seqno = ring_get_seqno;
1357                 if (IS_GEN2(dev)) {
1358                         ring->irq_get = i8xx_ring_get_irq;
1359                         ring->irq_put = i8xx_ring_put_irq;
1360                 } else {
1361                         ring->irq_get = i9xx_ring_get_irq;
1362                         ring->irq_put = i9xx_ring_put_irq;
1363                 }
1364                 ring->irq_enable_mask = I915_USER_INTERRUPT;
1365         }
1366         ring->write_tail = ring_write_tail;
1367         if (INTEL_INFO(dev)->gen >= 6)
1368                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1369         else if (INTEL_INFO(dev)->gen >= 4)
1370                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1371         else if (IS_I830(dev) || IS_845G(dev))
1372                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1373         else
1374                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1375         ring->init = init_render_ring;
1376         ring->cleanup = render_ring_cleanup;
1377
1378
1379         if (!I915_NEED_GFX_HWS(dev)) {
1380                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1381                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1382         }
1383
1384         return intel_init_ring_buffer(dev, ring);
1385 }
1386
1387 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1388 {
1389         drm_i915_private_t *dev_priv = dev->dev_private;
1390         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1391
1392         ring->name = "render ring";
1393         ring->id = RCS;
1394         ring->mmio_base = RENDER_RING_BASE;
1395
1396         if (INTEL_INFO(dev)->gen >= 6) {
1397                 /* non-kms not supported on gen6+ */
1398                 return -ENODEV;
1399         }
1400
1401         /* Note: gem is not supported on gen5/ilk without kms (the corresponding
1402          * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
1403          * the special gen5 functions. */
1404         ring->add_request = i9xx_add_request;
1405         if (INTEL_INFO(dev)->gen < 4)
1406                 ring->flush = gen2_render_ring_flush;
1407         else
1408                 ring->flush = gen4_render_ring_flush;
1409         ring->get_seqno = ring_get_seqno;
1410         if (IS_GEN2(dev)) {
1411                 ring->irq_get = i8xx_ring_get_irq;
1412                 ring->irq_put = i8xx_ring_put_irq;
1413         } else {
1414                 ring->irq_get = i9xx_ring_get_irq;
1415                 ring->irq_put = i9xx_ring_put_irq;
1416         }
1417         ring->irq_enable_mask = I915_USER_INTERRUPT;
1418         ring->write_tail = ring_write_tail;
1419         if (INTEL_INFO(dev)->gen >= 4)
1420                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1421         else if (IS_I830(dev) || IS_845G(dev))
1422                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1423         else
1424                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1425         ring->init = init_render_ring;
1426         ring->cleanup = render_ring_cleanup;
1427
1428         if (!I915_NEED_GFX_HWS(dev))
1429                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1430
1431         ring->dev = dev;
1432         INIT_LIST_HEAD(&ring->active_list);
1433         INIT_LIST_HEAD(&ring->request_list);
1434         INIT_LIST_HEAD(&ring->gpu_write_list);
1435
1436         ring->size = size;
1437         ring->effective_size = ring->size;
1438         if (IS_I830(ring->dev))
1439                 ring->effective_size -= 128;
1440
1441         ring->virtual_start = ioremap_wc(start, size);
1442         if (ring->virtual_start == NULL) {
1443                 DRM_ERROR("can not ioremap virtual address for"
1444                           " ring buffer\n");
1445                 return -ENOMEM;
1446         }
1447
1448         return 0;
1449 }
1450
1451 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1452 {
1453         drm_i915_private_t *dev_priv = dev->dev_private;
1454         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1455
1456         ring->name = "bsd ring";
1457         ring->id = VCS;
1458
1459         ring->write_tail = ring_write_tail;
1460         if (IS_GEN6(dev) || IS_GEN7(dev)) {
1461                 ring->mmio_base = GEN6_BSD_RING_BASE;
1462                 /* gen6 bsd needs a special wa for tail updates */
1463                 if (IS_GEN6(dev))
1464                         ring->write_tail = gen6_bsd_ring_write_tail;
1465                 ring->flush = gen6_ring_flush;
1466                 ring->add_request = gen6_add_request;
1467                 ring->get_seqno = gen6_ring_get_seqno;
1468                 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
1469                 ring->irq_get = gen6_ring_get_irq;
1470                 ring->irq_put = gen6_ring_put_irq;
1471                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1472                 ring->sync_to = gen6_ring_sync;
1473                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
1474                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
1475                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
1476                 ring->signal_mbox[0] = GEN6_RVSYNC;
1477                 ring->signal_mbox[1] = GEN6_BVSYNC;
1478         } else {
1479                 ring->mmio_base = BSD_RING_BASE;
1480                 ring->flush = bsd_ring_flush;
1481                 ring->add_request = i9xx_add_request;
1482                 ring->get_seqno = ring_get_seqno;
1483                 if (IS_GEN5(dev)) {
1484                         ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1485                         ring->irq_get = gen5_ring_get_irq;
1486                         ring->irq_put = gen5_ring_put_irq;
1487                 } else {
1488                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1489                         ring->irq_get = i9xx_ring_get_irq;
1490                         ring->irq_put = i9xx_ring_put_irq;
1491                 }
1492                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1493         }
1494         ring->init = init_ring_common;
1495
1496
1497         return intel_init_ring_buffer(dev, ring);
1498 }
1499
1500 int intel_init_blt_ring_buffer(struct drm_device *dev)
1501 {
1502         drm_i915_private_t *dev_priv = dev->dev_private;
1503         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1504
1505         ring->name = "blitter ring";
1506         ring->id = BCS;
1507
1508         ring->mmio_base = BLT_RING_BASE;
1509         ring->write_tail = ring_write_tail;
1510         ring->flush = blt_ring_flush;
1511         ring->add_request = gen6_add_request;
1512         ring->get_seqno = gen6_ring_get_seqno;
1513         ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
1514         ring->irq_get = gen6_ring_get_irq;
1515         ring->irq_put = gen6_ring_put_irq;
1516         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1517         ring->sync_to = gen6_ring_sync;
1518         ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
1519         ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
1520         ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
1521         ring->signal_mbox[0] = GEN6_RBSYNC;
1522         ring->signal_mbox[1] = GEN6_VBSYNC;
1523         ring->init = init_ring_common;
1524
1525         return intel_init_ring_buffer(dev, ring);
1526 }