shared-core/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #include "drmP.h"
  30 #include "drm.h"
  31 #include "i915_drm.h"
  32 #include "i915_drv.h"
  33
  34 #define MAX_NOPID ((u32)~0)
  35
  36 /*
  37  * These are the interrupts used by the driver
  38  */
  39 #define I915_INTERRUPT_ENABLE_MASK (I915_USER_INTERRUPT | \
  40                                     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
  41                                     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)
  42
  43 static inline void
  44 i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
  45 {
  46         if (dev_priv->irq_use_mask) {
  47                 if ((dev_priv->irq_mask_reg & mask) != 0) {
  48                         dev_priv->irq_mask_reg &= ~mask;
  49                         I915_WRITE(I915REG_INT_MASK_R, dev_priv->irq_mask_reg);
  50                         (void) I915_READ(I915REG_INT_MASK_R);
  51                 }
  52         } else {
  53                 if ((dev_priv->irq_enable_reg & mask) != mask) {
  54                         dev_priv->irq_enable_reg |= mask;
  55                         I915_WRITE(I915REG_INT_ENABLE_R, dev_priv->irq_enable_reg);
  56                         (void) I915_READ(I915REG_INT_ENABLE_R);
  57                 }
  58         }
  59 }
  60
  61 static inline void
  62 i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
  63 {
  64         if (dev_priv->irq_use_mask) {
  65                 if ((dev_priv->irq_enable_reg & mask) != mask) {
  66                         dev_priv->irq_mask_reg |= mask;
  67                         I915_WRITE(I915REG_INT_MASK_R, dev_priv->irq_mask_reg);
  68                         (void) I915_READ(I915REG_INT_MASK_R);
  69                 }
  70         } else {
  71                 if ((dev_priv->irq_enable_reg & mask) != 0) {
  72                         dev_priv->irq_enable_reg &= ~mask;
  73                         I915_WRITE(I915REG_INT_ENABLE_R, dev_priv->irq_enable_reg);
  74                         (void) I915_READ(I915REG_INT_ENABLE_R);
  75                 }
  76         }
  77 }
  78
  79 static inline void
  80 i915_enable_irqs(drm_i915_private_t *dev_priv)
  81 {
  82         if (dev_priv->irq_use_mask) {
  83                 I915_WRITE(I915REG_INT_MASK_R, dev_priv->irq_mask_reg);
  84                 (void) I915_READ(I915REG_INT_MASK_R);
  85         } else {
  86                 I915_WRITE(I915REG_INT_ENABLE_R, dev_priv->irq_enable_reg);
  87                 (void) I915_READ(I915REG_INT_ENABLE_R);
  88         }
  89 }
  90
  91 static inline void
  92 i915_disable_irqs(drm_i915_private_t *dev_priv)
  93 {
  94         if (dev_priv->irq_use_mask)
  95                 I915_WRITE(I915REG_INT_MASK_R, I915_INTERRUPT_ENABLE_MASK);
  96         else
  97                 I915_WRITE(I915REG_INT_ENABLE_R, 0);
  98 }
  99
 100 /**
 101  * i915_get_pipe - return the the pipe associated with a given plane
 102  * @dev: DRM device
 103  * @plane: plane to look for
 104  *
 105  * The Intel Mesa & 2D drivers call the vblank routines with a plane number
 106  * rather than a pipe number, since they may not always be equal.  This routine
 107  * maps the given @plane back to a pipe number.
 108  */
 109 static int
 110 i915_get_pipe(struct drm_device *dev, int plane)
 111 {
 112         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 113         u32 dspcntr;
 114
 115         dspcntr = plane ? I915_READ(DSPBCNTR) : I915_READ(DSPACNTR);
 116
 117         return dspcntr & DISPPLANE_SEL_PIPE_MASK ? 1 : 0;
 118 }
 119
 120 /**
 121  * i915_get_plane - return the the plane associated with a given pipe
 122  * @dev: DRM device
 123  * @pipe: pipe to look for
 124  *
 125  * The Intel Mesa & 2D drivers call the vblank routines with a plane number
 126  * rather than a plane number, since they may not always be equal.  This routine
 127  * maps the given @pipe back to a plane number.
 128  */
 129 static int
 130 i915_get_plane(struct drm_device *dev, int pipe)
 131 {
 132         if (i915_get_pipe(dev, 0) == pipe)
 133                 return 0;
 134         return 1;
 135 }
 136
 137 /**
 138  * i915_pipe_enabled - check if a pipe is enabled
 139  * @dev: DRM device
 140  * @pipe: pipe to check
 141  *
 142  * Reading certain registers when the pipe is disabled can hang the chip.
 143  * Use this routine to make sure the PLL is running and the pipe is active
 144  * before reading such registers if unsure.
 145  */
 146 static int
 147 i915_pipe_enabled(struct drm_device *dev, int pipe)
 148 {
 149         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 150         unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;
 151
 152         if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
 153                 return 1;
 154
 155         return 0;
 156 }
 157
 158 /**
 159  * Emit a synchronous flip.
 160  *
 161  * This function must be called with the drawable spinlock held.
 162  */
 163 static void
 164 i915_dispatch_vsync_flip(struct drm_device *dev, struct drm_drawable_info *drw,
 165                          int plane)
 166 {
 167         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 168         drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
 169         u16 x1, y1, x2, y2;
 170         int pf_planes = 1 << plane;
 171
 172         DRM_SPINLOCK_ASSERT(&dev->drw_lock);
 173
 174         /* If the window is visible on the other plane, we have to flip on that
 175          * plane as well.
 176          */
 177         if (plane == 1) {
 178                 x1 = sarea_priv->planeA_x;
 179                 y1 = sarea_priv->planeA_y;
 180                 x2 = x1 + sarea_priv->planeA_w;
 181                 y2 = y1 + sarea_priv->planeA_h;
 182         } else {
 183                 x1 = sarea_priv->planeB_x;
 184                 y1 = sarea_priv->planeB_y;
 185                 x2 = x1 + sarea_priv->planeB_w;
 186                 y2 = y1 + sarea_priv->planeB_h;
 187         }
 188
 189         if (x2 > 0 && y2 > 0) {
 190                 int i, num_rects = drw->num_rects;
 191                 struct drm_clip_rect *rect = drw->rects;
 192
 193                 for (i = 0; i < num_rects; i++)
 194                         if (!(rect[i].x1 >= x2 || rect[i].y1 >= y2 ||
 195                               rect[i].x2 <= x1 || rect[i].y2 <= y1)) {
 196                                 pf_planes = 0x3;
 197
 198                                 break;
 199                         }
 200         }
 201
 202         i915_dispatch_flip(dev, pf_planes, 1);
 203 }
 204
 205 /**
 206  * Emit blits for scheduled buffer swaps.
 207  *
 208  * This function will be called with the HW lock held.
 209  */
 210 static void i915_vblank_tasklet(struct drm_device *dev)
 211 {
 212         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 213         struct list_head *list, *tmp, hits, *hit;
 214         int nhits, nrects, slice[2], upper[2], lower[2], i, num_pages;
 215         unsigned counter[2];
 216         struct drm_drawable_info *drw;
 217         drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
 218         u32 cpp = dev_priv->cpp,  offsets[3];
 219         u32 cmd = (cpp == 4) ? (XY_SRC_COPY_BLT_CMD |
 220                                 XY_SRC_COPY_BLT_WRITE_ALPHA |
 221                                 XY_SRC_COPY_BLT_WRITE_RGB)
 222                              : XY_SRC_COPY_BLT_CMD;
 223         u32 src_pitch = sarea_priv->pitch * cpp;
 224         u32 dst_pitch = sarea_priv->pitch * cpp;
 225         /* COPY rop (0xcc), map cpp to magic color depth constants */
 226         u32 ropcpp = (0xcc << 16) | ((cpp - 1) << 24);
 227         RING_LOCALS;
 228
 229         if (sarea_priv->front_tiled) {
 230                 cmd |= XY_SRC_COPY_BLT_DST_TILED;
 231                 dst_pitch >>= 2;
 232         }
 233         if (sarea_priv->back_tiled) {
 234                 cmd |= XY_SRC_COPY_BLT_SRC_TILED;
 235                 src_pitch >>= 2;
 236         }
 237
 238         counter[0] = drm_vblank_count(dev, 0);
 239         counter[1] = drm_vblank_count(dev, 1);
 240
 241         DRM_DEBUG("\n");
 242
 243         INIT_LIST_HEAD(&hits);
 244
 245         nhits = nrects = 0;
 246
 247         /* No irqsave/restore necessary.  This tasklet may be run in an
 248          * interrupt context or normal context, but we don't have to worry
 249          * about getting interrupted by something acquiring the lock, because
 250          * we are the interrupt context thing that acquires the lock.
 251          */
 252         DRM_SPINLOCK(&dev_priv->swaps_lock);
 253
 254         /* Find buffer swaps scheduled for this vertical blank */
 255         list_for_each_safe(list, tmp, &dev_priv->vbl_swaps.head) {
 256                 drm_i915_vbl_swap_t *vbl_swap =
 257                         list_entry(list, drm_i915_vbl_swap_t, head);
 258                 int pipe = i915_get_pipe(dev, vbl_swap->plane);
 259
 260                 if ((counter[pipe] - vbl_swap->sequence) > (1<<23))
 261                         continue;
 262
 263                 list_del(list);
 264                 dev_priv->swaps_pending--;
 265                 drm_vblank_put(dev, pipe);
 266
 267                 DRM_SPINUNLOCK(&dev_priv->swaps_lock);
 268                 DRM_SPINLOCK(&dev->drw_lock);
 269
 270                 drw = drm_get_drawable_info(dev, vbl_swap->drw_id);
 271
 272                 if (!drw) {
 273                         DRM_SPINUNLOCK(&dev->drw_lock);
 274                         drm_free(vbl_swap, sizeof(*vbl_swap), DRM_MEM_DRIVER);
 275                         DRM_SPINLOCK(&dev_priv->swaps_lock);
 276                         continue;
 277                 }
 278
 279                 list_for_each(hit, &hits) {
 280                         drm_i915_vbl_swap_t *swap_cmp =
 281                                 list_entry(hit, drm_i915_vbl_swap_t, head);
 282                         struct drm_drawable_info *drw_cmp =
 283                                 drm_get_drawable_info(dev, swap_cmp->drw_id);
 284
 285                         if (drw_cmp &&
 286                             drw_cmp->rects[0].y1 > drw->rects[0].y1) {
 287                                 list_add_tail(list, hit);
 288                                 break;
 289                         }
 290                 }
 291
 292                 DRM_SPINUNLOCK(&dev->drw_lock);
 293
 294                 /* List of hits was empty, or we reached the end of it */
 295                 if (hit == &hits)
 296                         list_add_tail(list, hits.prev);
 297
 298                 nhits++;
 299
 300                 DRM_SPINLOCK(&dev_priv->swaps_lock);
 301         }
 302
 303         DRM_SPINUNLOCK(&dev_priv->swaps_lock);
 304
 305         if (nhits == 0) {
 306                 return;
 307         }
 308
 309         i915_kernel_lost_context(dev);
 310
 311         upper[0] = upper[1] = 0;
 312         slice[0] = max(sarea_priv->planeA_h / nhits, 1);
 313         slice[1] = max(sarea_priv->planeB_h / nhits, 1);
 314         lower[0] = sarea_priv->planeA_y + slice[0];
 315         lower[1] = sarea_priv->planeB_y + slice[0];
 316
 317         offsets[0] = sarea_priv->front_offset;
 318         offsets[1] = sarea_priv->back_offset;
 319         offsets[2] = sarea_priv->third_offset;
 320         num_pages = sarea_priv->third_handle ? 3 : 2;
 321
 322         DRM_SPINLOCK(&dev->drw_lock);
 323
 324         /* Emit blits for buffer swaps, partitioning both outputs into as many
 325          * slices as there are buffer swaps scheduled in order to avoid tearing
 326          * (based on the assumption that a single buffer swap would always
 327          * complete before scanout starts).
 328          */
 329         for (i = 0; i++ < nhits;
 330              upper[0] = lower[0], lower[0] += slice[0],
 331              upper[1] = lower[1], lower[1] += slice[1]) {
 332                 int init_drawrect = 1;
 333
 334                 if (i == nhits)
 335                         lower[0] = lower[1] = sarea_priv->height;
 336
 337                 list_for_each(hit, &hits) {
 338                         drm_i915_vbl_swap_t *swap_hit =
 339                                 list_entry(hit, drm_i915_vbl_swap_t, head);
 340                         struct drm_clip_rect *rect;
 341                         int num_rects, plane, front, back;
 342                         unsigned short top, bottom;
 343
 344                         drw = drm_get_drawable_info(dev, swap_hit->drw_id);
 345
 346                         if (!drw)
 347                                 continue;
 348
 349                         plane = swap_hit->plane;
 350
 351                         if (swap_hit->flip) {
 352                                 i915_dispatch_vsync_flip(dev, drw, plane);
 353                                 continue;
 354                         }
 355
 356                         if (init_drawrect) {
 357                                 int width  = sarea_priv->width;
 358                                 int height = sarea_priv->height;
 359                                 if (IS_I965G(dev)) {
 360                                         BEGIN_LP_RING(4);
 361
 362                                         OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
 363                                         OUT_RING(0);
 364                                         OUT_RING(((width - 1) & 0xffff) | ((height - 1) << 16));
 365                                         OUT_RING(0);
 366
 367                                         ADVANCE_LP_RING();
 368                                 } else {
 369                                         BEGIN_LP_RING(6);
 370
 371                                         OUT_RING(GFX_OP_DRAWRECT_INFO);
 372                                         OUT_RING(0);
 373                                         OUT_RING(0);
 374                                         OUT_RING(((width - 1) & 0xffff) | ((height - 1) << 16));
 375                                         OUT_RING(0);
 376                                         OUT_RING(0);
 377
 378                                         ADVANCE_LP_RING();
 379                                 }
 380
 381                                 sarea_priv->ctxOwner = DRM_KERNEL_CONTEXT;
 382
 383                                 init_drawrect = 0;
 384                         }
 385
 386                         rect = drw->rects;
 387                         top = upper[plane];
 388                         bottom = lower[plane];
 389
 390                         front = (dev_priv->sarea_priv->pf_current_page >>
 391                                  (2 * plane)) & 0x3;
 392                         back = (front + 1) % num_pages;
 393
 394                         for (num_rects = drw->num_rects; num_rects--; rect++) {
 395                                 int y1 = max(rect->y1, top);
 396                                 int y2 = min(rect->y2, bottom);
 397
 398                                 if (y1 >= y2)
 399                                         continue;
 400
 401                                 BEGIN_LP_RING(8);
 402
 403                                 OUT_RING(cmd);
 404                                 OUT_RING(ropcpp | dst_pitch);
 405                                 OUT_RING((y1 << 16) | rect->x1);
 406                                 OUT_RING((y2 << 16) | rect->x2);
 407                                 OUT_RING(offsets[front]);
 408                                 OUT_RING((y1 << 16) | rect->x1);
 409                                 OUT_RING(src_pitch);
 410                                 OUT_RING(offsets[back]);
 411
 412                                 ADVANCE_LP_RING();
 413                         }
 414                 }
 415         }
 416
 417         DRM_SPINUNLOCK(&dev->drw_lock);
 418
 419         list_for_each_safe(hit, tmp, &hits) {
 420                 drm_i915_vbl_swap_t *swap_hit =
 421                         list_entry(hit, drm_i915_vbl_swap_t, head);
 422
 423                 list_del(hit);
 424
 425                 drm_free(swap_hit, sizeof(*swap_hit), DRM_MEM_DRIVER);
 426         }
 427 }
 428 #if 0
 429 static int i915_in_vblank(struct drm_device *dev, int pipe)
 430 {
 431         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 432         unsigned long pipedsl, vblank, vtotal;
 433         unsigned long vbl_start, vbl_end, cur_line;
 434
 435         pipedsl = pipe ? PIPEBDSL : PIPEADSL;
 436         vblank = pipe ? VBLANK_B : VBLANK_A;
 437         vtotal = pipe ? VTOTAL_B : VTOTAL_A;
 438
 439         vbl_start = I915_READ(vblank) & VBLANK_START_MASK;
 440         vbl_end = (I915_READ(vblank) >> VBLANK_END_SHIFT) & VBLANK_END_MASK;
 441
 442         cur_line = I915_READ(pipedsl);
 443
 444         if (cur_line >= vbl_start)
 445                 return 1;
 446
 447         return 0;
 448 }
 449 #endif
 450 u32 i915_get_vblank_counter(struct drm_device *dev, int plane)
 451 {
 452         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 453         unsigned long high_frame;
 454         unsigned long low_frame;
 455         u32 high1, high2, low, count;
 456         int pipe;
 457
 458         pipe = i915_get_pipe(dev, plane);
 459         high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
 460         low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
 461
 462         if (!i915_pipe_enabled(dev, pipe)) {
 463             DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
 464             return 0;
 465         }
 466
 467         /*
 468          * High & low register fields aren't synchronized, so make sure
 469          * we get a low value that's stable across two reads of the high
 470          * register.
 471          */
 472         do {
 473                 high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
 474                          PIPE_FRAME_HIGH_SHIFT);
 475                 low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
 476                         PIPE_FRAME_LOW_SHIFT);
 477                 high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
 478                          PIPE_FRAME_HIGH_SHIFT);
 479         } while (high1 != high2);
 480
 481         count = (high1 << 8) | low;
 482
 483         /*
 484          * If we're in the middle of the vblank period, the
 485          * above regs won't have been updated yet, so return
 486          * an incremented count to stay accurate
 487          */
 488 #if 0
 489         if (i915_in_vblank(dev, pipe))
 490                 count++;
 491 #endif
 492         /* count may be reset by other driver(e.g. 2D driver),
 493            we have no way to know if it is wrapped or resetted
 494            when count is zero. do a rough guess.
 495         */
 496         if (count == 0 && dev->last_vblank[pipe] < dev->max_vblank_count/2)
 497                 dev->last_vblank[pipe] = 0;
 498
 499         return count;
 500 }
 501
 502 irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
 503 {
 504         struct drm_device *dev = (struct drm_device *) arg;
 505         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 506         u32 iir;
 507         u32 pipea_stats = 0, pipeb_stats = 0;
 508         int vblank = 0;
 509
 510         DRM_SPINLOCK(&dev_priv->user_irq_lock);
 511         if (dev->pdev->msi_enabled)
 512                 i915_disable_irqs(dev_priv);
 513         iir = I915_READ(I915REG_INT_IDENTITY_R);
 514 #if 0
 515         DRM_DEBUG("flag=%08x\n", iir);
 516 #endif
 517         atomic_inc(&dev_priv->irq_received);
 518         if (iir == 0) {
 519                 DRM_DEBUG ("iir 0x%08x im 0x%08x ie 0x%08x pipea 0x%08x pipeb 0x%08x\n",
 520                            iir,
 521                            I915_READ(I915REG_INT_MASK_R),
 522                            I915_READ(I915REG_INT_ENABLE_R),
 523                            I915_READ(I915REG_PIPEASTAT),
 524                            I915_READ(I915REG_PIPEBSTAT));
 525                 if (dev->pdev->msi_enabled)
 526                         i915_enable_irqs(dev_priv);
 527
 528                 DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 529                 return IRQ_NONE;
 530         }
 531
 532         /*
 533          * Clear the PIPE(A|B)STAT regs before the IIR otherwise
 534          * we may get extra interrupts.
 535          */
 536         if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
 537                 pipea_stats = I915_READ(I915REG_PIPEASTAT);
 538                 I915_WRITE(I915REG_PIPEASTAT, pipea_stats);
 539         }
 540         if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
 541                 pipeb_stats = I915_READ(I915REG_PIPEBSTAT);
 542                 I915_WRITE(I915REG_PIPEBSTAT, pipeb_stats);
 543         }
 544
 545         I915_WRITE(I915REG_INT_IDENTITY_R, iir);
 546         (void) I915_READ(I915REG_INT_IDENTITY_R); /* Flush posted writes */
 547
 548         if (dev->pdev->msi_enabled)
 549                 i915_enable_irqs(dev_priv);
 550
 551         DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 552
 553         if (dev_priv->sarea_priv)
 554             dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 555
 556         if (iir & I915_USER_INTERRUPT) {
 557                 dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
 558                 DRM_WAKEUP(&dev_priv->irq_queue);
 559 #ifdef I915_HAVE_FENCE
 560                 i915_fence_handler(dev);
 561 #endif
 562         }
 563
 564         if (pipea_stats & (I915_START_VBLANK_INTERRUPT_STATUS|
 565                            I915_VBLANK_INTERRUPT_STATUS)) {
 566                 vblank = 1;
 567                 drm_handle_vblank(dev, i915_get_plane(dev, 0));
 568         }
 569         if (pipeb_stats & (I915_START_VBLANK_INTERRUPT_STATUS|
 570                            I915_VBLANK_INTERRUPT_STATUS)) {
 571                 vblank = 1;
 572                 drm_handle_vblank(dev, i915_get_plane(dev, 1));
 573         }
 574         if (vblank) {
 575                 if (dev_priv->swaps_pending > 0)
 576                         drm_locked_tasklet(dev, i915_vblank_tasklet);
 577         }
 578
 579         return IRQ_HANDLED;
 580 }
 581
 582 int i915_emit_irq(struct drm_device *dev)
 583 {
 584         drm_i915_private_t *dev_priv = dev->dev_private;
 585         RING_LOCALS;
 586
 587         i915_kernel_lost_context(dev);
 588
 589         DRM_DEBUG("\n");
 590
 591         i915_emit_breadcrumb(dev);
 592
 593         BEGIN_LP_RING(2);
 594         OUT_RING(0);
 595         OUT_RING(GFX_OP_USER_INTERRUPT);
 596         ADVANCE_LP_RING();
 597
 598         return dev_priv->counter;
 599 }
 600
 601 void i915_user_irq_on(drm_i915_private_t *dev_priv)
 602 {
 603         DRM_SPINLOCK(&dev_priv->user_irq_lock);
 604         if (dev_priv->irq_enabled && (++dev_priv->user_irq_refcount == 1))
 605                 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 606         DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 607
 608 }
 609
 610 void i915_user_irq_off(drm_i915_private_t *dev_priv)
 611 {
 612         DRM_SPINLOCK(&dev_priv->user_irq_lock);
 613         BUG_ON(dev_priv->irq_enabled && dev_priv->user_irq_refcount <= 0);
 614         if (dev_priv->irq_enabled && (--dev_priv->user_irq_refcount == 0))
 615                 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 616         DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 617 }
 618
 619
 620 int i915_wait_irq(struct drm_device * dev, int irq_nr)
 621 {
 622         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 623         int ret = 0;
 624
 625         if (!dev_priv) {
 626                 DRM_ERROR("called with no initialization\n");
 627                 return -EINVAL;
 628         }
 629
 630         DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
 631                   READ_BREADCRUMB(dev_priv));
 632
 633         if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
 634                 if (dev_priv->sarea_priv)
 635                         dev_priv->sarea_priv->last_dispatch =
 636                                 READ_BREADCRUMB(dev_priv);
 637                 return 0;
 638         }
 639
 640         i915_user_irq_on(dev_priv);
 641         DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ,
 642                     READ_BREADCRUMB(dev_priv) >= irq_nr);
 643         i915_user_irq_off(dev_priv);
 644
 645         if (ret == -EBUSY) {
 646                 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
 647                           READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
 648         }
 649
 650         if (dev_priv->sarea_priv)
 651                 dev_priv->sarea_priv->last_dispatch =
 652                         READ_BREADCRUMB(dev_priv);
 653         return ret;
 654 }
 655
 656 /* Needs the lock as it touches the ring.
 657  */
 658 int i915_irq_emit(struct drm_device *dev, void *data,
 659                          struct drm_file *file_priv)
 660 {
 661         drm_i915_private_t *dev_priv = dev->dev_private;
 662         drm_i915_irq_emit_t *emit = data;
 663         int result;
 664
 665         LOCK_TEST_WITH_RETURN(dev, file_priv);
 666
 667         if (!dev_priv) {
 668                 DRM_ERROR("called with no initialization\n");
 669                 return -EINVAL;
 670         }
 671
 672         result = i915_emit_irq(dev);
 673
 674         if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
 675                 DRM_ERROR("copy_to_user\n");
 676                 return -EFAULT;
 677         }
 678
 679         return 0;
 680 }
 681
 682 /* Doesn't need the hardware lock.
 683  */
 684 int i915_irq_wait(struct drm_device *dev, void *data,
 685                   struct drm_file *file_priv)
 686 {
 687         drm_i915_private_t *dev_priv = dev->dev_private;
 688         drm_i915_irq_wait_t *irqwait = data;
 689
 690         if (!dev_priv) {
 691                 DRM_ERROR("called with no initialization\n");
 692                 return -EINVAL;
 693         }
 694
 695         return i915_wait_irq(dev, irqwait->irq_seq);
 696 }
 697
 698 int i915_enable_vblank(struct drm_device *dev, int plane)
 699 {
 700         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 701         int pipe = i915_get_pipe(dev, plane);
 702         u32     pipestat_reg = 0;
 703         u32     mask_reg = 0;
 704         u32     pipestat;
 705
 706         switch (pipe) {
 707         case 0:
 708                 pipestat_reg = I915REG_PIPEASTAT;
 709                 mask_reg |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
 710                 break;
 711         case 1:
 712                 pipestat_reg = I915REG_PIPEBSTAT;
 713                 mask_reg |= I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
 714                 break;
 715         default:
 716                 DRM_ERROR("tried to enable vblank on non-existent pipe %d\n",
 717                           pipe);
 718                 break;
 719         }
 720
 721         if (pipestat_reg)
 722         {
 723                 pipestat = I915_READ (pipestat_reg);
 724                 /*
 725                  * Older chips didn't have the start vblank interrupt,
 726                  * but
 727                  */
 728                 if (IS_I965G (dev))
 729                         pipestat |= I915_START_VBLANK_INTERRUPT_ENABLE;
 730                 else
 731                         pipestat |= I915_VBLANK_INTERRUPT_ENABLE;
 732                 /*
 733                  * Clear any pending status
 734                  */
 735                 pipestat |= (I915_START_VBLANK_INTERRUPT_STATUS |
 736                              I915_VBLANK_INTERRUPT_STATUS);
 737                 I915_WRITE(pipestat_reg, pipestat);
 738         }
 739         DRM_SPINLOCK(&dev_priv->user_irq_lock);
 740         i915_enable_irq(dev_priv, mask_reg);
 741         DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 742
 743         return 0;
 744 }
 745
 746 void i915_disable_vblank(struct drm_device *dev, int plane)
 747 {
 748         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 749         int pipe = i915_get_pipe(dev, plane);
 750         u32     pipestat_reg = 0;
 751         u32     mask_reg = 0;
 752         u32     pipestat;
 753
 754         switch (pipe) {
 755         case 0:
 756                 pipestat_reg = I915REG_PIPEASTAT;
 757                 mask_reg |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
 758                 break;
 759         case 1:
 760                 pipestat_reg = I915REG_PIPEBSTAT;
 761                 mask_reg |= I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
 762                 break;
 763         default:
 764                 DRM_ERROR("tried to disable vblank on non-existent pipe %d\n",
 765                           pipe);
 766                 break;
 767         }
 768
 769         DRM_SPINLOCK(&dev_priv->user_irq_lock);
 770         i915_disable_irq(dev_priv, mask_reg);
 771         if (pipestat_reg)
 772         {
 773                 pipestat = I915_READ (pipestat_reg);
 774                 pipestat &= ~(I915_START_VBLANK_INTERRUPT_ENABLE |
 775                               I915_VBLANK_INTERRUPT_ENABLE);
 776                 /*
 777                  * Clear any pending status
 778                  */
 779                 pipestat |= (I915_START_VBLANK_INTERRUPT_STATUS |
 780                              I915_VBLANK_INTERRUPT_STATUS);
 781                 I915_WRITE(pipestat_reg, pipestat);
 782                 (void) I915_READ(pipestat_reg);
 783         }
 784         DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 785 }
 786
 787 static void i915_enable_interrupt (struct drm_device *dev)
 788 {
 789         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 790
 791         dev_priv->irq_use_mask = 1;
 792         if (dev_priv->irq_use_mask) {
 793                 dev_priv->irq_mask_reg = I915_INTERRUPT_ENABLE_MASK;
 794                 dev_priv->irq_enable_reg = I915_INTERRUPT_ENABLE_MASK;
 795         } else {
 796                 dev_priv->irq_mask_reg = 0;
 797                 dev_priv->irq_enable_reg = 0;
 798         }
 799         I915_WRITE(I915REG_INT_IDENTITY_R, I915_READ(I915REG_INT_IDENTITY_R));
 800         I915_WRITE(I915REG_INT_MASK_R, dev_priv->irq_mask_reg);
 801         I915_WRITE(I915REG_INT_ENABLE_R, dev_priv->irq_enable_reg);
 802         (void) I915_READ (I915REG_INT_ENABLE_R);
 803         dev_priv->irq_enabled = 1;
 804 }
 805
 806 static void i915_disable_interrupt (struct drm_device *dev)
 807 {
 808         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 809
 810         I915_WRITE(I915REG_HWSTAM, 0xffffffff);
 811         I915_WRITE(I915REG_INT_MASK_R, I915_INTERRUPT_ENABLE_MASK);
 812         I915_WRITE(I915REG_INT_ENABLE_R, 0);
 813         I915_WRITE(I915REG_INT_IDENTITY_R, I915_READ(I915REG_INT_IDENTITY_R));
 814         (void) I915_READ (I915REG_INT_IDENTITY_R);
 815         dev_priv->irq_enabled = 0;
 816 }
 817
 818 /* Set the vblank monitor pipe
 819  */
 820 int i915_vblank_pipe_set(struct drm_device *dev, void *data,
 821                          struct drm_file *file_priv)
 822 {
 823         drm_i915_private_t *dev_priv = dev->dev_private;
 824         drm_i915_vblank_pipe_t *pipe = data;
 825
 826         if (!dev_priv) {
 827                 DRM_ERROR("called with no initialization\n");
 828                 return -EINVAL;
 829         }
 830
 831         if (pipe->pipe & ~(DRM_I915_VBLANK_PIPE_A|DRM_I915_VBLANK_PIPE_B)) {
 832                 DRM_ERROR("called with invalid pipe 0x%x\n", pipe->pipe);
 833                 return -EINVAL;
 834         }
 835
 836         dev_priv->vblank_pipe = pipe->pipe;
 837
 838         return 0;
 839 }
 840
 841 int i915_vblank_pipe_get(struct drm_device *dev, void *data,
 842                          struct drm_file *file_priv)
 843 {
 844         drm_i915_private_t *dev_priv = dev->dev_private;
 845         drm_i915_vblank_pipe_t *pipe = data;
 846         u16 flag;
 847
 848         if (!dev_priv) {
 849                 DRM_ERROR("called with no initialization\n");
 850                 return -EINVAL;
 851         }
 852
 853         if (dev_priv->irq_use_mask)
 854                 flag = ~dev_priv->irq_mask_reg;
 855         else
 856                 flag = dev_priv->irq_enable_reg;
 857         pipe->pipe = 0;
 858         if (flag & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT)
 859                 pipe->pipe |= DRM_I915_VBLANK_PIPE_A;
 860         if (flag & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)
 861                 pipe->pipe |= DRM_I915_VBLANK_PIPE_B;
 862
 863         return 0;
 864 }
 865
 866 /**
 867  * Schedule buffer swap at given vertical blank.
 868  */
 869 int i915_vblank_swap(struct drm_device *dev, void *data,
 870                      struct drm_file *file_priv)
 871 {
 872         drm_i915_private_t *dev_priv = dev->dev_private;
 873         drm_i915_vblank_swap_t *swap = data;
 874         drm_i915_vbl_swap_t *vbl_swap;
 875         unsigned int pipe, seqtype, curseq, plane;
 876         unsigned long irqflags;
 877         struct list_head *list;
 878         int ret;
 879
 880         if (!dev_priv) {
 881                 DRM_ERROR("%s called with no initialization\n", __func__);
 882                 return -EINVAL;
 883         }
 884
 885         if (!dev_priv->sarea_priv || dev_priv->sarea_priv->rotation) {
 886                 DRM_DEBUG("Rotation not supported\n");
 887                 return -EINVAL;
 888         }
 889
 890         if (swap->seqtype & ~(_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE |
 891                              _DRM_VBLANK_SECONDARY | _DRM_VBLANK_NEXTONMISS |
 892                              _DRM_VBLANK_FLIP)) {
 893                 DRM_ERROR("Invalid sequence type 0x%x\n", swap->seqtype);
 894                 return -EINVAL;
 895         }
 896
 897         plane = (swap->seqtype & _DRM_VBLANK_SECONDARY) ? 1 : 0;
 898         pipe = i915_get_pipe(dev, plane);
 899
 900         seqtype = swap->seqtype & (_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE);
 901
 902         if (!(dev_priv->vblank_pipe & (1 << pipe))) {
 903                 DRM_ERROR("Invalid pipe %d\n", pipe);
 904                 return -EINVAL;
 905         }
 906
 907         DRM_SPINLOCK_IRQSAVE(&dev->drw_lock, irqflags);
 908
 909         /* It makes no sense to schedule a swap for a drawable that doesn't have
 910          * valid information at this point. E.g. this could mean that the X
 911          * server is too old to push drawable information to the DRM, in which
 912          * case all such swaps would become ineffective.
 913          */
 914         if (!drm_get_drawable_info(dev, swap->drawable)) {
 915                 DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock, irqflags);
 916                 DRM_DEBUG("Invalid drawable ID %d\n", swap->drawable);
 917                 return -EINVAL;
 918         }
 919
 920         DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock, irqflags);
 921
 922         drm_update_vblank_count(dev, pipe);
 923         curseq = drm_vblank_count(dev, pipe);
 924
 925         if (seqtype == _DRM_VBLANK_RELATIVE)
 926                 swap->sequence += curseq;
 927
 928         if ((curseq - swap->sequence) <= (1<<23)) {
 929                 if (swap->seqtype & _DRM_VBLANK_NEXTONMISS) {
 930                         swap->sequence = curseq + 1;
 931                 } else {
 932                         DRM_DEBUG("Missed target sequence\n");
 933                         return -EINVAL;
 934                 }
 935         }
 936
 937         if (swap->seqtype & _DRM_VBLANK_FLIP) {
 938                 swap->sequence--;
 939
 940                 if ((curseq - swap->sequence) <= (1<<23)) {
 941                         struct drm_drawable_info *drw;
 942
 943                         LOCK_TEST_WITH_RETURN(dev, file_priv);
 944
 945                         DRM_SPINLOCK_IRQSAVE(&dev->drw_lock, irqflags);
 946
 947                         drw = drm_get_drawable_info(dev, swap->drawable);
 948
 949                         if (!drw) {
 950                                 DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock,
 951                                     irqflags);
 952                                 DRM_DEBUG("Invalid drawable ID %d\n",
 953                                           swap->drawable);
 954                                 return -EINVAL;
 955                         }
 956
 957                         i915_dispatch_vsync_flip(dev, drw, plane);
 958
 959                         DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock, irqflags);
 960
 961                         return 0;
 962                 }
 963         }
 964
 965         DRM_SPINLOCK_IRQSAVE(&dev_priv->swaps_lock, irqflags);
 966
 967         list_for_each(list, &dev_priv->vbl_swaps.head) {
 968                 vbl_swap = list_entry(list, drm_i915_vbl_swap_t, head);
 969
 970                 if (vbl_swap->drw_id == swap->drawable &&
 971                     vbl_swap->plane == plane &&
 972                     vbl_swap->sequence == swap->sequence) {
 973                         vbl_swap->flip = (swap->seqtype & _DRM_VBLANK_FLIP);
 974                         DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->swaps_lock, irqflags);
 975                         DRM_DEBUG("Already scheduled\n");
 976                         return 0;
 977                 }
 978         }
 979
 980         DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->swaps_lock, irqflags);
 981
 982         if (dev_priv->swaps_pending >= 100) {
 983                 DRM_DEBUG("Too many swaps queued\n");
 984                 return -EBUSY;
 985         }
 986
 987         vbl_swap = drm_calloc(1, sizeof(*vbl_swap), DRM_MEM_DRIVER);
 988
 989         if (!vbl_swap) {
 990                 DRM_ERROR("Failed to allocate memory to queue swap\n");
 991                 return -ENOMEM;
 992         }
 993
 994         DRM_DEBUG("\n");
 995
 996         ret = drm_vblank_get(dev, pipe);
 997         if (ret) {
 998                 drm_free(vbl_swap, sizeof(*vbl_swap), DRM_MEM_DRIVER);
 999                 return ret;
1000         }
1001
1002         vbl_swap->drw_id = swap->drawable;
1003         vbl_swap->plane = plane;
1004         vbl_swap->sequence = swap->sequence;
1005         vbl_swap->flip = (swap->seqtype & _DRM_VBLANK_FLIP);
1006
1007         if (vbl_swap->flip)
1008                 swap->sequence++;
1009
1010         DRM_SPINLOCK_IRQSAVE(&dev_priv->swaps_lock, irqflags);
1011
1012         list_add_tail(&vbl_swap->head, &dev_priv->vbl_swaps.head);
1013         dev_priv->swaps_pending++;
1014
1015         DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->swaps_lock, irqflags);
1016
1017         return 0;
1018 }
1019
1020 /* drm_dma.h hooks
1021 */
1022 void i915_driver_irq_preinstall(struct drm_device * dev)
1023 {
1024         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1025
1026         atomic_set(&dev_priv->irq_received, 0);
1027         I915_WRITE(I915REG_HWSTAM, 0xffff);
1028         I915_WRITE(I915REG_INT_ENABLE_R, 0x0);
1029         I915_WRITE(I915REG_INT_MASK_R, 0xffffffff);
1030         I915_WRITE(I915REG_INT_IDENTITY_R, 0xffffffff);
1031         (void) I915_READ(I915REG_INT_IDENTITY_R);
1032 }
1033
1034 int i915_driver_irq_postinstall(struct drm_device * dev)
1035 {
1036         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1037         int ret, num_pipes = 2;
1038
1039         DRM_SPININIT(&dev_priv->swaps_lock, "swap");
1040         INIT_LIST_HEAD(&dev_priv->vbl_swaps.head);
1041         dev_priv->swaps_pending = 0;
1042
1043         DRM_SPININIT(&dev_priv->user_irq_lock, "userirq");
1044         dev_priv->user_irq_refcount = 0;
1045         dev_priv->irq_mask_reg = 0;
1046
1047         ret = drm_vblank_init(dev, num_pipes);
1048         if (ret)
1049                 return ret;
1050
1051         dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
1052
1053         i915_enable_interrupt(dev);
1054         DRM_INIT_WAITQUEUE(&dev_priv->irq_queue);
1055
1056         /*
1057          * Initialize the hardware status page IRQ location.
1058          */
1059
1060         I915_WRITE(I915REG_INSTPM, (1 << 5) | (1 << 21));
1061         return 0;
1062 }
1063
1064 void i915_driver_irq_uninstall(struct drm_device * dev)
1065 {
1066         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1067         u32 temp;
1068
1069         if (!dev_priv)
1070                 return;
1071
1072         i915_disable_interrupt (dev);
1073
1074         temp = I915_READ(I915REG_PIPEASTAT);
1075         I915_WRITE(I915REG_PIPEASTAT, temp);
1076         temp = I915_READ(I915REG_PIPEBSTAT);
1077         I915_WRITE(I915REG_PIPEBSTAT, temp);
1078 }