shared-core/r128_state.c

   1 /* r128_state.c -- State support for r128 -*- linux-c -*-
   2  * Created: Thu Jan 27 02:53:43 2000 by gareth@valinux.com
   3  *
   4  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24  * DEALINGS IN THE SOFTWARE.
  25  *
  26  * Authors:
  27  *    Gareth Hughes <gareth@valinux.com>
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "r128_drm.h"
  33 #include "r128_drv.h"
  34
  35 /* Interface history:
  36  *
  37  * ??  - ??
  38  * 2.4 - Add support for ycbcr textures (no new ioctls)
  39  * 2.5 - Add FLIP ioctl, disable FULLSCREEN.
  40  */
  41 drm_ioctl_desc_t r128_ioctls[] = {
  42         [DRM_IOCTL_NR(DRM_R128_INIT)] = {r128_cce_init, 1, 1},
  43         [DRM_IOCTL_NR(DRM_R128_CCE_START)] = {r128_cce_start, 1, 1},
  44         [DRM_IOCTL_NR(DRM_R128_CCE_STOP)] = {r128_cce_stop, 1, 1},
  45         [DRM_IOCTL_NR(DRM_R128_CCE_RESET)] = {r128_cce_reset, 1, 1},
  46         [DRM_IOCTL_NR(DRM_R128_CCE_IDLE)] = {r128_cce_idle, 1, 0},
  47         [DRM_IOCTL_NR(DRM_R128_RESET)] = {r128_engine_reset, 1, 0},
  48         [DRM_IOCTL_NR(DRM_R128_FULLSCREEN)] = {r128_fullscreen, 1, 0},
  49         [DRM_IOCTL_NR(DRM_R128_SWAP)] = {r128_cce_swap, 1, 0},
  50         [DRM_IOCTL_NR(DRM_R128_FLIP)] = {r128_cce_flip, 1, 0},
  51         [DRM_IOCTL_NR(DRM_R128_CLEAR)] = {r128_cce_clear, 1, 0},
  52         [DRM_IOCTL_NR(DRM_R128_VERTEX)] = {r128_cce_vertex, 1, 0},
  53         [DRM_IOCTL_NR(DRM_R128_INDICES)] = {r128_cce_indices, 1, 0},
  54         [DRM_IOCTL_NR(DRM_R128_BLIT)] = {r128_cce_blit, 1, 0},
  55         [DRM_IOCTL_NR(DRM_R128_DEPTH)] = {r128_cce_depth, 1, 0},
  56         [DRM_IOCTL_NR(DRM_R128_STIPPLE)] = {r128_cce_stipple, 1, 0},
  57         [DRM_IOCTL_NR(DRM_R128_INDIRECT)] = {r128_cce_indirect, 1, 1},
  58         [DRM_IOCTL_NR(DRM_R128_GETPARAM)] = {r128_getparam, 1, 0},
  59 };
  60
  61 int r128_max_ioctl = DRM_ARRAY_SIZE(r128_ioctls);
  62
  63 /* ================================================================
  64  * CCE hardware state programming functions
  65  */
  66
  67 static void r128_emit_clip_rects(drm_r128_private_t * dev_priv,
  68                                  drm_clip_rect_t * boxes, int count)
  69 {
  70         u32 aux_sc_cntl = 0x00000000;
  71         RING_LOCALS;
  72         DRM_DEBUG("    %s\n", __FUNCTION__);
  73
  74         BEGIN_RING((count < 3 ? count : 3) * 5 + 2);
  75
  76         if (count >= 1) {
  77                 OUT_RING(CCE_PACKET0(R128_AUX1_SC_LEFT, 3));
  78                 OUT_RING(boxes[0].x1);
  79                 OUT_RING(boxes[0].x2 - 1);
  80                 OUT_RING(boxes[0].y1);
  81                 OUT_RING(boxes[0].y2 - 1);
  82
  83                 aux_sc_cntl |= (R128_AUX1_SC_EN | R128_AUX1_SC_MODE_OR);
  84         }
  85         if (count >= 2) {
  86                 OUT_RING(CCE_PACKET0(R128_AUX2_SC_LEFT, 3));
  87                 OUT_RING(boxes[1].x1);
  88                 OUT_RING(boxes[1].x2 - 1);
  89                 OUT_RING(boxes[1].y1);
  90                 OUT_RING(boxes[1].y2 - 1);
  91
  92                 aux_sc_cntl |= (R128_AUX2_SC_EN | R128_AUX2_SC_MODE_OR);
  93         }
  94         if (count >= 3) {
  95                 OUT_RING(CCE_PACKET0(R128_AUX3_SC_LEFT, 3));
  96                 OUT_RING(boxes[2].x1);
  97                 OUT_RING(boxes[2].x2 - 1);
  98                 OUT_RING(boxes[2].y1);
  99                 OUT_RING(boxes[2].y2 - 1);
 100
 101                 aux_sc_cntl |= (R128_AUX3_SC_EN | R128_AUX3_SC_MODE_OR);
 102         }
 103
 104         OUT_RING(CCE_PACKET0(R128_AUX_SC_CNTL, 0));
 105         OUT_RING(aux_sc_cntl);
 106
 107         ADVANCE_RING();
 108 }
 109
 110 static __inline__ void r128_emit_core(drm_r128_private_t * dev_priv)
 111 {
 112         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 113         drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 114         RING_LOCALS;
 115         DRM_DEBUG("    %s\n", __FUNCTION__);
 116
 117         BEGIN_RING(2);
 118
 119         OUT_RING(CCE_PACKET0(R128_SCALE_3D_CNTL, 0));
 120         OUT_RING(ctx->scale_3d_cntl);
 121
 122         ADVANCE_RING();
 123 }
 124
 125 static __inline__ void r128_emit_context(drm_r128_private_t * dev_priv)
 126 {
 127         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 128         drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 129         RING_LOCALS;
 130         DRM_DEBUG("    %s\n", __FUNCTION__);
 131
 132         BEGIN_RING(13);
 133
 134         OUT_RING(CCE_PACKET0(R128_DST_PITCH_OFFSET_C, 11));
 135         OUT_RING(ctx->dst_pitch_offset_c);
 136         OUT_RING(ctx->dp_gui_master_cntl_c);
 137         OUT_RING(ctx->sc_top_left_c);
 138         OUT_RING(ctx->sc_bottom_right_c);
 139         OUT_RING(ctx->z_offset_c);
 140         OUT_RING(ctx->z_pitch_c);
 141         OUT_RING(ctx->z_sten_cntl_c);
 142         OUT_RING(ctx->tex_cntl_c);
 143         OUT_RING(ctx->misc_3d_state_cntl_reg);
 144         OUT_RING(ctx->texture_clr_cmp_clr_c);
 145         OUT_RING(ctx->texture_clr_cmp_msk_c);
 146         OUT_RING(ctx->fog_color_c);
 147
 148         ADVANCE_RING();
 149 }
 150
 151 static __inline__ void r128_emit_setup(drm_r128_private_t * dev_priv)
 152 {
 153         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 154         drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 155         RING_LOCALS;
 156         DRM_DEBUG("    %s\n", __FUNCTION__);
 157
 158         BEGIN_RING(3);
 159
 160         OUT_RING(CCE_PACKET1(R128_SETUP_CNTL, R128_PM4_VC_FPU_SETUP));
 161         OUT_RING(ctx->setup_cntl);
 162         OUT_RING(ctx->pm4_vc_fpu_setup);
 163
 164         ADVANCE_RING();
 165 }
 166
 167 static __inline__ void r128_emit_masks(drm_r128_private_t * dev_priv)
 168 {
 169         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 170         drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 171         RING_LOCALS;
 172         DRM_DEBUG("    %s\n", __FUNCTION__);
 173
 174         BEGIN_RING(5);
 175
 176         OUT_RING(CCE_PACKET0(R128_DP_WRITE_MASK, 0));
 177         OUT_RING(ctx->dp_write_mask);
 178
 179         OUT_RING(CCE_PACKET0(R128_STEN_REF_MASK_C, 1));
 180         OUT_RING(ctx->sten_ref_mask_c);
 181         OUT_RING(ctx->plane_3d_mask_c);
 182
 183         ADVANCE_RING();
 184 }
 185
 186 static __inline__ void r128_emit_window(drm_r128_private_t * dev_priv)
 187 {
 188         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 189         drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 190         RING_LOCALS;
 191         DRM_DEBUG("    %s\n", __FUNCTION__);
 192
 193         BEGIN_RING(2);
 194
 195         OUT_RING(CCE_PACKET0(R128_WINDOW_XY_OFFSET, 0));
 196         OUT_RING(ctx->window_xy_offset);
 197
 198         ADVANCE_RING();
 199 }
 200
 201 static __inline__ void r128_emit_tex0(drm_r128_private_t * dev_priv)
 202 {
 203         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 204         drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 205         drm_r128_texture_regs_t *tex = &sarea_priv->tex_state[0];
 206         int i;
 207         RING_LOCALS;
 208         DRM_DEBUG("    %s\n", __FUNCTION__);
 209
 210         BEGIN_RING(7 + R128_MAX_TEXTURE_LEVELS);
 211
 212         OUT_RING(CCE_PACKET0(R128_PRIM_TEX_CNTL_C,
 213                              2 + R128_MAX_TEXTURE_LEVELS));
 214         OUT_RING(tex->tex_cntl);
 215         OUT_RING(tex->tex_combine_cntl);
 216         OUT_RING(ctx->tex_size_pitch_c);
 217         for (i = 0; i < R128_MAX_TEXTURE_LEVELS; i++) {
 218                 OUT_RING(tex->tex_offset[i]);
 219         }
 220
 221         OUT_RING(CCE_PACKET0(R128_CONSTANT_COLOR_C, 1));
 222         OUT_RING(ctx->constant_color_c);
 223         OUT_RING(tex->tex_border_color);
 224
 225         ADVANCE_RING();
 226 }
 227
 228 static __inline__ void r128_emit_tex1(drm_r128_private_t * dev_priv)
 229 {
 230         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 231         drm_r128_texture_regs_t *tex = &sarea_priv->tex_state[1];
 232         int i;
 233         RING_LOCALS;
 234         DRM_DEBUG("    %s\n", __FUNCTION__);
 235
 236         BEGIN_RING(5 + R128_MAX_TEXTURE_LEVELS);
 237
 238         OUT_RING(CCE_PACKET0(R128_SEC_TEX_CNTL_C, 1 + R128_MAX_TEXTURE_LEVELS));
 239         OUT_RING(tex->tex_cntl);
 240         OUT_RING(tex->tex_combine_cntl);
 241         for (i = 0; i < R128_MAX_TEXTURE_LEVELS; i++) {
 242                 OUT_RING(tex->tex_offset[i]);
 243         }
 244
 245         OUT_RING(CCE_PACKET0(R128_SEC_TEXTURE_BORDER_COLOR_C, 0));
 246         OUT_RING(tex->tex_border_color);
 247
 248         ADVANCE_RING();
 249 }
 250
 251 static __inline__ void r128_emit_state(drm_r128_private_t * dev_priv)
 252 {
 253         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 254         unsigned int dirty = sarea_priv->dirty;
 255
 256         DRM_DEBUG("%s: dirty=0x%08x\n", __FUNCTION__, dirty);
 257
 258         if (dirty & R128_UPLOAD_CORE) {
 259                 r128_emit_core(dev_priv);
 260                 sarea_priv->dirty &= ~R128_UPLOAD_CORE;
 261         }
 262
 263         if (dirty & R128_UPLOAD_CONTEXT) {
 264                 r128_emit_context(dev_priv);
 265                 sarea_priv->dirty &= ~R128_UPLOAD_CONTEXT;
 266         }
 267
 268         if (dirty & R128_UPLOAD_SETUP) {
 269                 r128_emit_setup(dev_priv);
 270                 sarea_priv->dirty &= ~R128_UPLOAD_SETUP;
 271         }
 272
 273         if (dirty & R128_UPLOAD_MASKS) {
 274                 r128_emit_masks(dev_priv);
 275                 sarea_priv->dirty &= ~R128_UPLOAD_MASKS;
 276         }
 277
 278         if (dirty & R128_UPLOAD_WINDOW) {
 279                 r128_emit_window(dev_priv);
 280                 sarea_priv->dirty &= ~R128_UPLOAD_WINDOW;
 281         }
 282
 283         if (dirty & R128_UPLOAD_TEX0) {
 284                 r128_emit_tex0(dev_priv);
 285                 sarea_priv->dirty &= ~R128_UPLOAD_TEX0;
 286         }
 287
 288         if (dirty & R128_UPLOAD_TEX1) {
 289                 r128_emit_tex1(dev_priv);
 290                 sarea_priv->dirty &= ~R128_UPLOAD_TEX1;
 291         }
 292
 293         /* Turn off the texture cache flushing */
 294         sarea_priv->context_state.tex_cntl_c &= ~R128_TEX_CACHE_FLUSH;
 295
 296         sarea_priv->dirty &= ~R128_REQUIRE_QUIESCENCE;
 297 }
 298
 299 #if R128_PERFORMANCE_BOXES
 300 /* ================================================================
 301  * Performance monitoring functions
 302  */
 303
 304 static void r128_clear_box(drm_r128_private_t * dev_priv,
 305                            int x, int y, int w, int h, int r, int g, int b)
 306 {
 307         u32 pitch, offset;
 308         u32 fb_bpp, color;
 309         RING_LOCALS;
 310
 311         switch (dev_priv->fb_bpp) {
 312         case 16:
 313                 fb_bpp = R128_GMC_DST_16BPP;
 314                 color = (((r & 0xf8) << 8) |
 315                          ((g & 0xfc) << 3) | ((b & 0xf8) >> 3));
 316                 break;
 317         case 24:
 318                 fb_bpp = R128_GMC_DST_24BPP;
 319                 color = ((r << 16) | (g << 8) | b);
 320                 break;
 321         case 32:
 322                 fb_bpp = R128_GMC_DST_32BPP;
 323                 color = (((0xff) << 24) | (r << 16) | (g << 8) | b);
 324                 break;
 325         default:
 326                 return;
 327         }
 328
 329         offset = dev_priv->back_offset;
 330         pitch = dev_priv->back_pitch >> 3;
 331
 332         BEGIN_RING(6);
 333
 334         OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
 335         OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
 336                  R128_GMC_BRUSH_SOLID_COLOR |
 337                  fb_bpp |
 338                  R128_GMC_SRC_DATATYPE_COLOR |
 339                  R128_ROP3_P |
 340                  R128_GMC_CLR_CMP_CNTL_DIS | R128_GMC_AUX_CLIP_DIS);
 341
 342         OUT_RING((pitch << 21) | (offset >> 5));
 343         OUT_RING(color);
 344
 345         OUT_RING((x << 16) | y);
 346         OUT_RING((w << 16) | h);
 347
 348         ADVANCE_RING();
 349 }
 350
 351 static void r128_cce_performance_boxes(drm_r128_private_t * dev_priv)
 352 {
 353         if (atomic_read(&dev_priv->idle_count) == 0) {
 354                 r128_clear_box(dev_priv, 64, 4, 8, 8, 0, 255, 0);
 355         } else {
 356                 atomic_set(&dev_priv->idle_count, 0);
 357         }
 358 }
 359
 360 #endif
 361
 362 /* ================================================================
 363  * CCE command dispatch functions
 364  */
 365
 366 static void r128_print_dirty(const char *msg, unsigned int flags)
 367 {
 368         DRM_INFO("%s: (0x%x) %s%s%s%s%s%s%s%s%s\n",
 369                  msg,
 370                  flags,
 371                  (flags & R128_UPLOAD_CORE) ? "core, " : "",
 372                  (flags & R128_UPLOAD_CONTEXT) ? "context, " : "",
 373                  (flags & R128_UPLOAD_SETUP) ? "setup, " : "",
 374                  (flags & R128_UPLOAD_TEX0) ? "tex0, " : "",
 375                  (flags & R128_UPLOAD_TEX1) ? "tex1, " : "",
 376                  (flags & R128_UPLOAD_MASKS) ? "masks, " : "",
 377                  (flags & R128_UPLOAD_WINDOW) ? "window, " : "",
 378                  (flags & R128_UPLOAD_CLIPRECTS) ? "cliprects, " : "",
 379                  (flags & R128_REQUIRE_QUIESCENCE) ? "quiescence, " : "");
 380 }
 381
 382 static void r128_cce_dispatch_clear(drm_device_t * dev,
 383                                     drm_r128_clear_t * clear)
 384 {
 385         drm_r128_private_t *dev_priv = dev->dev_private;
 386         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 387         int nbox = sarea_priv->nbox;
 388         drm_clip_rect_t *pbox = sarea_priv->boxes;
 389         unsigned int flags = clear->flags;
 390         int i;
 391         RING_LOCALS;
 392         DRM_DEBUG("%s\n", __FUNCTION__);
 393
 394         if (dev_priv->page_flipping && dev_priv->current_page == 1) {
 395                 unsigned int tmp = flags;
 396
 397                 flags &= ~(R128_FRONT | R128_BACK);
 398                 if (tmp & R128_FRONT)
 399                         flags |= R128_BACK;
 400                 if (tmp & R128_BACK)
 401                         flags |= R128_FRONT;
 402         }
 403
 404         for (i = 0; i < nbox; i++) {
 405                 int x = pbox[i].x1;
 406                 int y = pbox[i].y1;
 407                 int w = pbox[i].x2 - x;
 408                 int h = pbox[i].y2 - y;
 409
 410                 DRM_DEBUG("dispatch clear %d,%d-%d,%d flags 0x%x\n",
 411                           pbox[i].x1, pbox[i].y1, pbox[i].x2,
 412                           pbox[i].y2, flags);
 413
 414                 if (flags & (R128_FRONT | R128_BACK)) {
 415                         BEGIN_RING(2);
 416
 417                         OUT_RING(CCE_PACKET0(R128_DP_WRITE_MASK, 0));
 418                         OUT_RING(clear->color_mask);
 419
 420                         ADVANCE_RING();
 421                 }
 422
 423                 if (flags & R128_FRONT) {
 424                         BEGIN_RING(6);
 425
 426                         OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
 427                         OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
 428                                  R128_GMC_BRUSH_SOLID_COLOR |
 429                                  (dev_priv->color_fmt << 8) |
 430                                  R128_GMC_SRC_DATATYPE_COLOR |
 431                                  R128_ROP3_P |
 432                                  R128_GMC_CLR_CMP_CNTL_DIS |
 433                                  R128_GMC_AUX_CLIP_DIS);
 434
 435                         OUT_RING(dev_priv->front_pitch_offset_c);
 436                         OUT_RING(clear->clear_color);
 437
 438                         OUT_RING((x << 16) | y);
 439                         OUT_RING((w << 16) | h);
 440
 441                         ADVANCE_RING();
 442                 }
 443
 444                 if (flags & R128_BACK) {
 445                         BEGIN_RING(6);
 446
 447                         OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
 448                         OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
 449                                  R128_GMC_BRUSH_SOLID_COLOR |
 450                                  (dev_priv->color_fmt << 8) |
 451                                  R128_GMC_SRC_DATATYPE_COLOR |
 452                                  R128_ROP3_P |
 453                                  R128_GMC_CLR_CMP_CNTL_DIS |
 454                                  R128_GMC_AUX_CLIP_DIS);
 455
 456                         OUT_RING(dev_priv->back_pitch_offset_c);
 457                         OUT_RING(clear->clear_color);
 458
 459                         OUT_RING((x << 16) | y);
 460                         OUT_RING((w << 16) | h);
 461
 462                         ADVANCE_RING();
 463                 }
 464
 465                 if (flags & R128_DEPTH) {
 466                         BEGIN_RING(6);
 467
 468                         OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
 469                         OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
 470                                  R128_GMC_BRUSH_SOLID_COLOR |
 471                                  (dev_priv->depth_fmt << 8) |
 472                                  R128_GMC_SRC_DATATYPE_COLOR |
 473                                  R128_ROP3_P |
 474                                  R128_GMC_CLR_CMP_CNTL_DIS |
 475                                  R128_GMC_AUX_CLIP_DIS | R128_GMC_WR_MSK_DIS);
 476
 477                         OUT_RING(dev_priv->depth_pitch_offset_c);
 478                         OUT_RING(clear->clear_depth);
 479
 480                         OUT_RING((x << 16) | y);
 481                         OUT_RING((w << 16) | h);
 482
 483                         ADVANCE_RING();
 484                 }
 485         }
 486 }
 487
 488 static void r128_cce_dispatch_swap(drm_device_t * dev)
 489 {
 490         drm_r128_private_t *dev_priv = dev->dev_private;
 491         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 492         int nbox = sarea_priv->nbox;
 493         drm_clip_rect_t *pbox = sarea_priv->boxes;
 494         int i;
 495         RING_LOCALS;
 496         DRM_DEBUG("%s\n", __FUNCTION__);
 497
 498 #if R128_PERFORMANCE_BOXES
 499         /* Do some trivial performance monitoring...
 500          */
 501         r128_cce_performance_boxes(dev_priv);
 502 #endif
 503
 504         for (i = 0; i < nbox; i++) {
 505                 int x = pbox[i].x1;
 506                 int y = pbox[i].y1;
 507                 int w = pbox[i].x2 - x;
 508                 int h = pbox[i].y2 - y;
 509
 510                 BEGIN_RING(7);
 511
 512                 OUT_RING(CCE_PACKET3(R128_CNTL_BITBLT_MULTI, 5));
 513                 OUT_RING(R128_GMC_SRC_PITCH_OFFSET_CNTL |
 514                          R128_GMC_DST_PITCH_OFFSET_CNTL |
 515                          R128_GMC_BRUSH_NONE |
 516                          (dev_priv->color_fmt << 8) |
 517                          R128_GMC_SRC_DATATYPE_COLOR |
 518                          R128_ROP3_S |
 519                          R128_DP_SRC_SOURCE_MEMORY |
 520                          R128_GMC_CLR_CMP_CNTL_DIS |
 521                          R128_GMC_AUX_CLIP_DIS | R128_GMC_WR_MSK_DIS);
 522
 523                 /* Make this work even if front & back are flipped:
 524                  */
 525                 if (dev_priv->current_page == 0) {
 526                         OUT_RING(dev_priv->back_pitch_offset_c);
 527                         OUT_RING(dev_priv->front_pitch_offset_c);
 528                 } else {
 529                         OUT_RING(dev_priv->front_pitch_offset_c);
 530                         OUT_RING(dev_priv->back_pitch_offset_c);
 531                 }
 532
 533                 OUT_RING((x << 16) | y);
 534                 OUT_RING((x << 16) | y);
 535                 OUT_RING((w << 16) | h);
 536
 537                 ADVANCE_RING();
 538         }
 539
 540         /* Increment the frame counter.  The client-side 3D driver must
 541          * throttle the framerate by waiting for this value before
 542          * performing the swapbuffer ioctl.
 543          */
 544         dev_priv->sarea_priv->last_frame++;
 545
 546         BEGIN_RING(2);
 547
 548         OUT_RING(CCE_PACKET0(R128_LAST_FRAME_REG, 0));
 549         OUT_RING(dev_priv->sarea_priv->last_frame);
 550
 551         ADVANCE_RING();
 552 }
 553
 554 static void r128_cce_dispatch_flip(drm_device_t * dev)
 555 {
 556         drm_r128_private_t *dev_priv = dev->dev_private;
 557         RING_LOCALS;
 558         DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
 559                   __FUNCTION__,
 560                   dev_priv->current_page, dev_priv->sarea_priv->pfCurrentPage);
 561
 562 #if R128_PERFORMANCE_BOXES
 563         /* Do some trivial performance monitoring...
 564          */
 565         r128_cce_performance_boxes(dev_priv);
 566 #endif
 567
 568         BEGIN_RING(4);
 569
 570         R128_WAIT_UNTIL_PAGE_FLIPPED();
 571         OUT_RING(CCE_PACKET0(R128_CRTC_OFFSET, 0));
 572
 573         if (dev_priv->current_page == 0) {
 574                 OUT_RING(dev_priv->back_offset);
 575         } else {
 576                 OUT_RING(dev_priv->front_offset);
 577         }
 578
 579         ADVANCE_RING();
 580
 581         /* Increment the frame counter.  The client-side 3D driver must
 582          * throttle the framerate by waiting for this value before
 583          * performing the swapbuffer ioctl.
 584          */
 585         dev_priv->sarea_priv->last_frame++;
 586         dev_priv->sarea_priv->pfCurrentPage = dev_priv->current_page =
 587             1 - dev_priv->current_page;
 588
 589         BEGIN_RING(2);
 590
 591         OUT_RING(CCE_PACKET0(R128_LAST_FRAME_REG, 0));
 592         OUT_RING(dev_priv->sarea_priv->last_frame);
 593
 594         ADVANCE_RING();
 595 }
 596
 597 static void r128_cce_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
 598 {
 599         drm_r128_private_t *dev_priv = dev->dev_private;
 600         drm_r128_buf_priv_t *buf_priv = buf->dev_private;
 601         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 602         int format = sarea_priv->vc_format;
 603         int offset = buf->bus_address;
 604         int size = buf->used;
 605         int prim = buf_priv->prim;
 606         int i = 0;
 607         RING_LOCALS;
 608         DRM_DEBUG("buf=%d nbox=%d\n", buf->idx, sarea_priv->nbox);
 609
 610         if (0)
 611                 r128_print_dirty("dispatch_vertex", sarea_priv->dirty);
 612
 613         if (buf->used) {
 614                 buf_priv->dispatched = 1;
 615
 616                 if (sarea_priv->dirty & ~R128_UPLOAD_CLIPRECTS) {
 617                         r128_emit_state(dev_priv);
 618                 }
 619
 620                 do {
 621                         /* Emit the next set of up to three cliprects */
 622                         if (i < sarea_priv->nbox) {
 623                                 r128_emit_clip_rects(dev_priv,
 624                                                      &sarea_priv->boxes[i],
 625                                                      sarea_priv->nbox - i);
 626                         }
 627
 628                         /* Emit the vertex buffer rendering commands */
 629                         BEGIN_RING(5);
 630
 631                         OUT_RING(CCE_PACKET3(R128_3D_RNDR_GEN_INDX_PRIM, 3));
 632                         OUT_RING(offset);
 633                         OUT_RING(size);
 634                         OUT_RING(format);
 635                         OUT_RING(prim | R128_CCE_VC_CNTL_PRIM_WALK_LIST |
 636                                  (size << R128_CCE_VC_CNTL_NUM_SHIFT));
 637
 638                         ADVANCE_RING();
 639
 640                         i += 3;
 641                 } while (i < sarea_priv->nbox);
 642         }
 643
 644         if (buf_priv->discard) {
 645                 buf_priv->age = dev_priv->sarea_priv->last_dispatch;
 646
 647                 /* Emit the vertex buffer age */
 648                 BEGIN_RING(2);
 649
 650                 OUT_RING(CCE_PACKET0(R128_LAST_DISPATCH_REG, 0));
 651                 OUT_RING(buf_priv->age);
 652
 653                 ADVANCE_RING();
 654
 655                 buf->pending = 1;
 656                 buf->used = 0;
 657                 /* FIXME: Check dispatched field */
 658                 buf_priv->dispatched = 0;
 659         }
 660
 661         dev_priv->sarea_priv->last_dispatch++;
 662
 663         sarea_priv->dirty &= ~R128_UPLOAD_CLIPRECTS;
 664         sarea_priv->nbox = 0;
 665 }
 666
 667 static void r128_cce_dispatch_indirect(drm_device_t * dev,
 668                                        drm_buf_t * buf, int start, int end)
 669 {
 670         drm_r128_private_t *dev_priv = dev->dev_private;
 671         drm_r128_buf_priv_t *buf_priv = buf->dev_private;
 672         RING_LOCALS;
 673         DRM_DEBUG("indirect: buf=%d s=0x%x e=0x%x\n", buf->idx, start, end);
 674
 675         if (start != end) {
 676                 int offset = buf->bus_address + start;
 677                 int dwords = (end - start + 3) / sizeof(u32);
 678
 679                 /* Indirect buffer data must be an even number of
 680                  * dwords, so if we've been given an odd number we must
 681                  * pad the data with a Type-2 CCE packet.
 682                  */
 683                 if (dwords & 1) {
 684                         u32 *data = (u32 *)
 685                             ((char *)dev->agp_buffer_map->handle
 686                              + buf->offset + start);
 687                         data[dwords++] = cpu_to_le32(R128_CCE_PACKET2);
 688                 }
 689
 690                 buf_priv->dispatched = 1;
 691
 692                 /* Fire off the indirect buffer */
 693                 BEGIN_RING(3);
 694
 695                 OUT_RING(CCE_PACKET0(R128_PM4_IW_INDOFF, 1));
 696                 OUT_RING(offset);
 697                 OUT_RING(dwords);
 698
 699                 ADVANCE_RING();
 700         }
 701
 702         if (buf_priv->discard) {
 703                 buf_priv->age = dev_priv->sarea_priv->last_dispatch;
 704
 705                 /* Emit the indirect buffer age */
 706                 BEGIN_RING(2);
 707
 708                 OUT_RING(CCE_PACKET0(R128_LAST_DISPATCH_REG, 0));
 709                 OUT_RING(buf_priv->age);
 710
 711                 ADVANCE_RING();
 712
 713                 buf->pending = 1;
 714                 buf->used = 0;
 715                 /* FIXME: Check dispatched field */
 716                 buf_priv->dispatched = 0;
 717         }
 718
 719         dev_priv->sarea_priv->last_dispatch++;
 720 }
 721
 722 static void r128_cce_dispatch_indices(drm_device_t * dev,
 723                                       drm_buf_t * buf,
 724                                       int start, int end, int count)
 725 {
 726         drm_r128_private_t *dev_priv = dev->dev_private;
 727         drm_r128_buf_priv_t *buf_priv = buf->dev_private;
 728         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 729         int format = sarea_priv->vc_format;
 730         int offset = dev->agp_buffer_map->offset - dev_priv->cce_buffers_offset;
 731         int prim = buf_priv->prim;
 732         u32 *data;
 733         int dwords;
 734         int i = 0;
 735         RING_LOCALS;
 736         DRM_DEBUG("indices: s=%d e=%d c=%d\n", start, end, count);
 737
 738         if (0)
 739                 r128_print_dirty("dispatch_indices", sarea_priv->dirty);
 740
 741         if (start != end) {
 742                 buf_priv->dispatched = 1;
 743
 744                 if (sarea_priv->dirty & ~R128_UPLOAD_CLIPRECTS) {
 745                         r128_emit_state(dev_priv);
 746                 }
 747
 748                 dwords = (end - start + 3) / sizeof(u32);
 749
 750                 data = (u32 *) ((char *)dev->agp_buffer_map->handle
 751                                 + buf->offset + start);
 752
 753                 data[0] = cpu_to_le32(CCE_PACKET3(R128_3D_RNDR_GEN_INDX_PRIM,
 754                                                   dwords - 2));
 755
 756                 data[1] = cpu_to_le32(offset);
 757                 data[2] = cpu_to_le32(R128_MAX_VB_VERTS);
 758                 data[3] = cpu_to_le32(format);
 759                 data[4] = cpu_to_le32((prim | R128_CCE_VC_CNTL_PRIM_WALK_IND |
 760                                        (count << 16)));
 761
 762                 if (count & 0x1) {
 763 #ifdef __LITTLE_ENDIAN
 764                         data[dwords - 1] &= 0x0000ffff;
 765 #else
 766                         data[dwords - 1] &= 0xffff0000;
 767 #endif
 768                 }
 769
 770                 do {
 771                         /* Emit the next set of up to three cliprects */
 772                         if (i < sarea_priv->nbox) {
 773                                 r128_emit_clip_rects(dev_priv,
 774                                                      &sarea_priv->boxes[i],
 775                                                      sarea_priv->nbox - i);
 776                         }
 777
 778                         r128_cce_dispatch_indirect(dev, buf, start, end);
 779
 780                         i += 3;
 781                 } while (i < sarea_priv->nbox);
 782         }
 783
 784         if (buf_priv->discard) {
 785                 buf_priv->age = dev_priv->sarea_priv->last_dispatch;
 786
 787                 /* Emit the vertex buffer age */
 788                 BEGIN_RING(2);
 789
 790                 OUT_RING(CCE_PACKET0(R128_LAST_DISPATCH_REG, 0));
 791                 OUT_RING(buf_priv->age);
 792
 793                 ADVANCE_RING();
 794
 795                 buf->pending = 1;
 796                 /* FIXME: Check dispatched field */
 797                 buf_priv->dispatched = 0;
 798         }
 799
 800         dev_priv->sarea_priv->last_dispatch++;
 801
 802         sarea_priv->dirty &= ~R128_UPLOAD_CLIPRECTS;
 803         sarea_priv->nbox = 0;
 804 }
 805
 806 static int r128_cce_dispatch_blit(DRMFILE filp,
 807                                   drm_device_t * dev, drm_r128_blit_t * blit)
 808 {
 809         drm_r128_private_t *dev_priv = dev->dev_private;
 810         drm_device_dma_t *dma = dev->dma;
 811         drm_buf_t *buf;
 812         drm_r128_buf_priv_t *buf_priv;
 813         u32 *data;
 814         int dword_shift, dwords;
 815         RING_LOCALS;
 816         DRM_DEBUG("\n");
 817
 818         /* The compiler won't optimize away a division by a variable,
 819          * even if the only legal values are powers of two.  Thus, we'll
 820          * use a shift instead.
 821          */
 822         switch (blit->format) {
 823         case R128_DATATYPE_ARGB8888:
 824                 dword_shift = 0;
 825                 break;
 826         case R128_DATATYPE_ARGB1555:
 827         case R128_DATATYPE_RGB565:
 828         case R128_DATATYPE_ARGB4444:
 829         case R128_DATATYPE_YVYU422:
 830         case R128_DATATYPE_VYUY422:
 831                 dword_shift = 1;
 832                 break;
 833         case R128_DATATYPE_CI8:
 834         case R128_DATATYPE_RGB8:
 835                 dword_shift = 2;
 836                 break;
 837         default:
 838                 DRM_ERROR("invalid blit format %d\n", blit->format);
 839                 return DRM_ERR(EINVAL);
 840         }
 841
 842         /* Flush the pixel cache, and mark the contents as Read Invalid.
 843          * This ensures no pixel data gets mixed up with the texture
 844          * data from the host data blit, otherwise part of the texture
 845          * image may be corrupted.
 846          */
 847         BEGIN_RING(2);
 848
 849         OUT_RING(CCE_PACKET0(R128_PC_GUI_CTLSTAT, 0));
 850         OUT_RING(R128_PC_RI_GUI | R128_PC_FLUSH_GUI);
 851
 852         ADVANCE_RING();
 853
 854         /* Dispatch the indirect buffer.
 855          */
 856         buf = dma->buflist[blit->idx];
 857         buf_priv = buf->dev_private;
 858
 859         if (buf->filp != filp) {
 860                 DRM_ERROR("process %d using buffer owned by %p\n",
 861                           DRM_CURRENTPID, buf->filp);
 862                 return DRM_ERR(EINVAL);
 863         }
 864         if (buf->pending) {
 865                 DRM_ERROR("sending pending buffer %d\n", blit->idx);
 866                 return DRM_ERR(EINVAL);
 867         }
 868
 869         buf_priv->discard = 1;
 870
 871         dwords = (blit->width * blit->height) >> dword_shift;
 872
 873         data = (u32 *) ((char *)dev->agp_buffer_map->handle + buf->offset);
 874
 875         data[0] = cpu_to_le32(CCE_PACKET3(R128_CNTL_HOSTDATA_BLT, dwords + 6));
 876         data[1] = cpu_to_le32((R128_GMC_DST_PITCH_OFFSET_CNTL |
 877                                R128_GMC_BRUSH_NONE |
 878                                (blit->format << 8) |
 879                                R128_GMC_SRC_DATATYPE_COLOR |
 880                                R128_ROP3_S |
 881                                R128_DP_SRC_SOURCE_HOST_DATA |
 882                                R128_GMC_CLR_CMP_CNTL_DIS |
 883                                R128_GMC_AUX_CLIP_DIS | R128_GMC_WR_MSK_DIS));
 884
 885         data[2] = cpu_to_le32((blit->pitch << 21) | (blit->offset >> 5));
 886         data[3] = cpu_to_le32(0xffffffff);
 887         data[4] = cpu_to_le32(0xffffffff);
 888         data[5] = cpu_to_le32((blit->y << 16) | blit->x);
 889         data[6] = cpu_to_le32((blit->height << 16) | blit->width);
 890         data[7] = cpu_to_le32(dwords);
 891
 892         buf->used = (dwords + 8) * sizeof(u32);
 893
 894         r128_cce_dispatch_indirect(dev, buf, 0, buf->used);
 895
 896         /* Flush the pixel cache after the blit completes.  This ensures
 897          * the texture data is written out to memory before rendering
 898          * continues.
 899          */
 900         BEGIN_RING(2);
 901
 902         OUT_RING(CCE_PACKET0(R128_PC_GUI_CTLSTAT, 0));
 903         OUT_RING(R128_PC_FLUSH_GUI);
 904
 905         ADVANCE_RING();
 906
 907         return 0;
 908 }
 909
 910 /* ================================================================
 911  * Tiled depth buffer management
 912  *
 913  * FIXME: These should all set the destination write mask for when we
 914  * have hardware stencil support.
 915  */
 916
 917 static int r128_cce_dispatch_write_span(drm_device_t * dev,
 918                                         drm_r128_depth_t * depth)
 919 {
 920         drm_r128_private_t *dev_priv = dev->dev_private;
 921         int count, x, y;
 922         u32 *buffer;
 923         u8 *mask;
 924         int i, buffer_size, mask_size;
 925         RING_LOCALS;
 926         DRM_DEBUG("\n");
 927
 928         count = depth->n;
 929         if (count > 4096 || count <= 0)
 930                 return DRM_ERR(EMSGSIZE);
 931
 932         if (DRM_COPY_FROM_USER(&x, depth->x, sizeof(x))) {
 933                 return DRM_ERR(EFAULT);
 934         }
 935         if (DRM_COPY_FROM_USER(&y, depth->y, sizeof(y))) {
 936                 return DRM_ERR(EFAULT);
 937         }
 938
 939         buffer_size = depth->n * sizeof(u32);
 940         buffer = drm_alloc(buffer_size, DRM_MEM_BUFS);
 941         if (buffer == NULL)
 942                 return DRM_ERR(ENOMEM);
 943         if (DRM_COPY_FROM_USER(buffer, depth->buffer, buffer_size)) {
 944                 drm_free(buffer, buffer_size, DRM_MEM_BUFS);
 945                 return DRM_ERR(EFAULT);
 946         }
 947
 948         mask_size = depth->n * sizeof(u8);
 949         if (depth->mask) {
 950                 mask = drm_alloc(mask_size, DRM_MEM_BUFS);
 951                 if (mask == NULL) {
 952                         drm_free(buffer, buffer_size, DRM_MEM_BUFS);
 953                         return DRM_ERR(ENOMEM);
 954                 }
 955                 if (DRM_COPY_FROM_USER(mask, depth->mask, mask_size)) {
 956                         drm_free(buffer, buffer_size, DRM_MEM_BUFS);
 957                         drm_free(mask, mask_size, DRM_MEM_BUFS);
 958                         return DRM_ERR(EFAULT);
 959                 }
 960
 961                 for (i = 0; i < count; i++, x++) {
 962                         if (mask[i]) {
 963                                 BEGIN_RING(6);
 964
 965                                 OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
 966                                 OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
 967                                          R128_GMC_BRUSH_SOLID_COLOR |
 968                                          (dev_priv->depth_fmt << 8) |
 969                                          R128_GMC_SRC_DATATYPE_COLOR |
 970                                          R128_ROP3_P |
 971                                          R128_GMC_CLR_CMP_CNTL_DIS |
 972                                          R128_GMC_WR_MSK_DIS);
 973
 974                                 OUT_RING(dev_priv->depth_pitch_offset_c);
 975                                 OUT_RING(buffer[i]);
 976
 977                                 OUT_RING((x << 16) | y);
 978                                 OUT_RING((1 << 16) | 1);
 979
 980                                 ADVANCE_RING();
 981                         }
 982                 }
 983
 984                 drm_free(mask, mask_size, DRM_MEM_BUFS);
 985         } else {
 986                 for (i = 0; i < count; i++, x++) {
 987                         BEGIN_RING(6);
 988
 989                         OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
 990                         OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
 991                                  R128_GMC_BRUSH_SOLID_COLOR |
 992                                  (dev_priv->depth_fmt << 8) |
 993                                  R128_GMC_SRC_DATATYPE_COLOR |
 994                                  R128_ROP3_P |
 995                                  R128_GMC_CLR_CMP_CNTL_DIS |
 996                                  R128_GMC_WR_MSK_DIS);
 997
 998                         OUT_RING(dev_priv->depth_pitch_offset_c);
 999                         OUT_RING(buffer[i]);
1000
1001                         OUT_RING((x << 16) | y);
1002                         OUT_RING((1 << 16) | 1);
1003
1004                         ADVANCE_RING();
1005                 }
1006         }
1007
1008         drm_free(buffer, buffer_size, DRM_MEM_BUFS);
1009
1010         return 0;
1011 }
1012
1013 static int r128_cce_dispatch_write_pixels(drm_device_t * dev,
1014                                           drm_r128_depth_t * depth)
1015 {
1016         drm_r128_private_t *dev_priv = dev->dev_private;
1017         int count, *x, *y;
1018         u32 *buffer;
1019         u8 *mask;
1020         int i, xbuf_size, ybuf_size, buffer_size, mask_size;
1021         RING_LOCALS;
1022         DRM_DEBUG("\n");
1023
1024         count = depth->n;
1025         if (count > 4096 || count <= 0)
1026                 return DRM_ERR(EMSGSIZE);
1027
1028         xbuf_size = count * sizeof(*x);
1029         ybuf_size = count * sizeof(*y);
1030         x = drm_alloc(xbuf_size, DRM_MEM_BUFS);
1031         if (x == NULL) {
1032                 return DRM_ERR(ENOMEM);
1033         }
1034         y = drm_alloc(ybuf_size, DRM_MEM_BUFS);
1035         if (y == NULL) {
1036                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1037                 return DRM_ERR(ENOMEM);
1038         }
1039         if (DRM_COPY_FROM_USER(x, depth->x, xbuf_size)) {
1040                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1041                 drm_free(y, ybuf_size, DRM_MEM_BUFS);
1042                 return DRM_ERR(EFAULT);
1043         }
1044         if (DRM_COPY_FROM_USER(y, depth->y, xbuf_size)) {
1045                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1046                 drm_free(y, ybuf_size, DRM_MEM_BUFS);
1047                 return DRM_ERR(EFAULT);
1048         }
1049
1050         buffer_size = depth->n * sizeof(u32);
1051         buffer = drm_alloc(buffer_size, DRM_MEM_BUFS);
1052         if (buffer == NULL) {
1053                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1054                 drm_free(y, ybuf_size, DRM_MEM_BUFS);
1055                 return DRM_ERR(ENOMEM);
1056         }
1057         if (DRM_COPY_FROM_USER(buffer, depth->buffer, buffer_size)) {
1058                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1059                 drm_free(y, ybuf_size, DRM_MEM_BUFS);
1060                 drm_free(buffer, buffer_size, DRM_MEM_BUFS);
1061                 return DRM_ERR(EFAULT);
1062         }
1063
1064         if (depth->mask) {
1065                 mask_size = depth->n * sizeof(u8);
1066                 mask = drm_alloc(mask_size, DRM_MEM_BUFS);
1067                 if (mask == NULL) {
1068                         drm_free(x, xbuf_size, DRM_MEM_BUFS);
1069                         drm_free(y, ybuf_size, DRM_MEM_BUFS);
1070                         drm_free(buffer, buffer_size, DRM_MEM_BUFS);
1071                         return DRM_ERR(ENOMEM);
1072                 }
1073                 if (DRM_COPY_FROM_USER(mask, depth->mask, mask_size)) {
1074                         drm_free(x, xbuf_size, DRM_MEM_BUFS);
1075                         drm_free(y, ybuf_size, DRM_MEM_BUFS);
1076                         drm_free(buffer, buffer_size, DRM_MEM_BUFS);
1077                         drm_free(mask, mask_size, DRM_MEM_BUFS);
1078                         return DRM_ERR(EFAULT);
1079                 }
1080
1081                 for (i = 0; i < count; i++) {
1082                         if (mask[i]) {
1083                                 BEGIN_RING(6);
1084
1085                                 OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
1086                                 OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
1087                                          R128_GMC_BRUSH_SOLID_COLOR |
1088                                          (dev_priv->depth_fmt << 8) |
1089                                          R128_GMC_SRC_DATATYPE_COLOR |
1090                                          R128_ROP3_P |
1091                                          R128_GMC_CLR_CMP_CNTL_DIS |
1092                                          R128_GMC_WR_MSK_DIS);
1093
1094                                 OUT_RING(dev_priv->depth_pitch_offset_c);
1095                                 OUT_RING(buffer[i]);
1096
1097                                 OUT_RING((x[i] << 16) | y[i]);
1098                                 OUT_RING((1 << 16) | 1);
1099
1100                                 ADVANCE_RING();
1101                         }
1102                 }
1103
1104                 drm_free(mask, mask_size, DRM_MEM_BUFS);
1105         } else {
1106                 for (i = 0; i < count; i++) {
1107                         BEGIN_RING(6);
1108
1109                         OUT_RING(CCE_PACKET3(R128_CNTL_PAINT_MULTI, 4));
1110                         OUT_RING(R128_GMC_DST_PITCH_OFFSET_CNTL |
1111                                  R128_GMC_BRUSH_SOLID_COLOR |
1112                                  (dev_priv->depth_fmt << 8) |
1113                                  R128_GMC_SRC_DATATYPE_COLOR |
1114                                  R128_ROP3_P |
1115                                  R128_GMC_CLR_CMP_CNTL_DIS |
1116                                  R128_GMC_WR_MSK_DIS);
1117
1118                         OUT_RING(dev_priv->depth_pitch_offset_c);
1119                         OUT_RING(buffer[i]);
1120
1121                         OUT_RING((x[i] << 16) | y[i]);
1122                         OUT_RING((1 << 16) | 1);
1123
1124                         ADVANCE_RING();
1125                 }
1126         }
1127
1128         drm_free(x, xbuf_size, DRM_MEM_BUFS);
1129         drm_free(y, ybuf_size, DRM_MEM_BUFS);
1130         drm_free(buffer, buffer_size, DRM_MEM_BUFS);
1131
1132         return 0;
1133 }
1134
1135 static int r128_cce_dispatch_read_span(drm_device_t * dev,
1136                                        drm_r128_depth_t * depth)
1137 {
1138         drm_r128_private_t *dev_priv = dev->dev_private;
1139         int count, x, y;
1140         RING_LOCALS;
1141         DRM_DEBUG("\n");
1142
1143         count = depth->n;
1144         if (count > 4096 || count <= 0)
1145                 return DRM_ERR(EMSGSIZE);
1146
1147         if (DRM_COPY_FROM_USER(&x, depth->x, sizeof(x))) {
1148                 return DRM_ERR(EFAULT);
1149         }
1150         if (DRM_COPY_FROM_USER(&y, depth->y, sizeof(y))) {
1151                 return DRM_ERR(EFAULT);
1152         }
1153
1154         BEGIN_RING(7);
1155
1156         OUT_RING(CCE_PACKET3(R128_CNTL_BITBLT_MULTI, 5));
1157         OUT_RING(R128_GMC_SRC_PITCH_OFFSET_CNTL |
1158                  R128_GMC_DST_PITCH_OFFSET_CNTL |
1159                  R128_GMC_BRUSH_NONE |
1160                  (dev_priv->depth_fmt << 8) |
1161                  R128_GMC_SRC_DATATYPE_COLOR |
1162                  R128_ROP3_S |
1163                  R128_DP_SRC_SOURCE_MEMORY |
1164                  R128_GMC_CLR_CMP_CNTL_DIS | R128_GMC_WR_MSK_DIS);
1165
1166         OUT_RING(dev_priv->depth_pitch_offset_c);
1167         OUT_RING(dev_priv->span_pitch_offset_c);
1168
1169         OUT_RING((x << 16) | y);
1170         OUT_RING((0 << 16) | 0);
1171         OUT_RING((count << 16) | 1);
1172
1173         ADVANCE_RING();
1174
1175         return 0;
1176 }
1177
1178 static int r128_cce_dispatch_read_pixels(drm_device_t * dev,
1179                                          drm_r128_depth_t * depth)
1180 {
1181         drm_r128_private_t *dev_priv = dev->dev_private;
1182         int count, *x, *y;
1183         int i, xbuf_size, ybuf_size;
1184         RING_LOCALS;
1185         DRM_DEBUG("%s\n", __FUNCTION__);
1186
1187         count = depth->n;
1188         if (count > 4096 || count <= 0)
1189                 return DRM_ERR(EMSGSIZE);
1190
1191         if (count > dev_priv->depth_pitch) {
1192                 count = dev_priv->depth_pitch;
1193         }
1194
1195         xbuf_size = count * sizeof(*x);
1196         ybuf_size = count * sizeof(*y);
1197         x = drm_alloc(xbuf_size, DRM_MEM_BUFS);
1198         if (x == NULL) {
1199                 return DRM_ERR(ENOMEM);
1200         }
1201         y = drm_alloc(ybuf_size, DRM_MEM_BUFS);
1202         if (y == NULL) {
1203                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1204                 return DRM_ERR(ENOMEM);
1205         }
1206         if (DRM_COPY_FROM_USER(x, depth->x, xbuf_size)) {
1207                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1208                 drm_free(y, ybuf_size, DRM_MEM_BUFS);
1209                 return DRM_ERR(EFAULT);
1210         }
1211         if (DRM_COPY_FROM_USER(y, depth->y, ybuf_size)) {
1212                 drm_free(x, xbuf_size, DRM_MEM_BUFS);
1213                 drm_free(y, ybuf_size, DRM_MEM_BUFS);
1214                 return DRM_ERR(EFAULT);
1215         }
1216
1217         for (i = 0; i < count; i++) {
1218                 BEGIN_RING(7);
1219
1220                 OUT_RING(CCE_PACKET3(R128_CNTL_BITBLT_MULTI, 5));
1221                 OUT_RING(R128_GMC_SRC_PITCH_OFFSET_CNTL |
1222                          R128_GMC_DST_PITCH_OFFSET_CNTL |
1223                          R128_GMC_BRUSH_NONE |
1224                          (dev_priv->depth_fmt << 8) |
1225                          R128_GMC_SRC_DATATYPE_COLOR |
1226                          R128_ROP3_S |
1227                          R128_DP_SRC_SOURCE_MEMORY |
1228                          R128_GMC_CLR_CMP_CNTL_DIS | R128_GMC_WR_MSK_DIS);
1229
1230                 OUT_RING(dev_priv->depth_pitch_offset_c);
1231                 OUT_RING(dev_priv->span_pitch_offset_c);
1232
1233                 OUT_RING((x[i] << 16) | y[i]);
1234                 OUT_RING((i << 16) | 0);
1235                 OUT_RING((1 << 16) | 1);
1236
1237                 ADVANCE_RING();
1238         }
1239
1240         drm_free(x, xbuf_size, DRM_MEM_BUFS);
1241         drm_free(y, ybuf_size, DRM_MEM_BUFS);
1242
1243         return 0;
1244 }
1245
1246 /* ================================================================
1247  * Polygon stipple
1248  */
1249
1250 static void r128_cce_dispatch_stipple(drm_device_t * dev, u32 * stipple)
1251 {
1252         drm_r128_private_t *dev_priv = dev->dev_private;
1253         int i;
1254         RING_LOCALS;
1255         DRM_DEBUG("%s\n", __FUNCTION__);
1256
1257         BEGIN_RING(33);
1258
1259         OUT_RING(CCE_PACKET0(R128_BRUSH_DATA0, 31));
1260         for (i = 0; i < 32; i++) {
1261                 OUT_RING(stipple[i]);
1262         }
1263
1264         ADVANCE_RING();
1265 }
1266
1267 /* ================================================================
1268  * IOCTL functions
1269  */
1270
1271 int r128_cce_clear(DRM_IOCTL_ARGS)
1272 {
1273         DRM_DEVICE;
1274         drm_r128_private_t *dev_priv = dev->dev_private;
1275         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
1276         drm_r128_clear_t clear;
1277         DRM_DEBUG("\n");
1278
1279         LOCK_TEST_WITH_RETURN(dev, filp);
1280
1281         DRM_COPY_FROM_USER_IOCTL(clear, (drm_r128_clear_t __user *) data,
1282                                  sizeof(clear));
1283
1284         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1285
1286         if (sarea_priv->nbox > R128_NR_SAREA_CLIPRECTS)
1287                 sarea_priv->nbox = R128_NR_SAREA_CLIPRECTS;
1288
1289         r128_cce_dispatch_clear(dev, &clear);
1290         COMMIT_RING();
1291
1292         /* Make sure we restore the 3D state next time.
1293          */
1294         dev_priv->sarea_priv->dirty |= R128_UPLOAD_CONTEXT | R128_UPLOAD_MASKS;
1295
1296         return 0;
1297 }
1298
1299 static int r128_do_init_pageflip(drm_device_t * dev)
1300 {
1301         drm_r128_private_t *dev_priv = dev->dev_private;
1302         DRM_DEBUG("\n");
1303
1304         dev_priv->crtc_offset = R128_READ(R128_CRTC_OFFSET);
1305         dev_priv->crtc_offset_cntl = R128_READ(R128_CRTC_OFFSET_CNTL);
1306
1307         R128_WRITE(R128_CRTC_OFFSET, dev_priv->front_offset);
1308         R128_WRITE(R128_CRTC_OFFSET_CNTL,
1309                    dev_priv->crtc_offset_cntl | R128_CRTC_OFFSET_FLIP_CNTL);
1310
1311         dev_priv->page_flipping = 1;
1312         dev_priv->current_page = 0;
1313         dev_priv->sarea_priv->pfCurrentPage = dev_priv->current_page;
1314
1315         return 0;
1316 }
1317
1318 int r128_do_cleanup_pageflip(drm_device_t * dev)
1319 {
1320         drm_r128_private_t *dev_priv = dev->dev_private;
1321         DRM_DEBUG("\n");
1322
1323         R128_WRITE(R128_CRTC_OFFSET, dev_priv->crtc_offset);
1324         R128_WRITE(R128_CRTC_OFFSET_CNTL, dev_priv->crtc_offset_cntl);
1325
1326         if (dev_priv->current_page != 0) {
1327                 r128_cce_dispatch_flip(dev);
1328                 COMMIT_RING();
1329         }
1330
1331         dev_priv->page_flipping = 0;
1332         return 0;
1333 }
1334
1335 /* Swapping and flipping are different operations, need different ioctls.
1336  * They can & should be intermixed to support multiple 3d windows.
1337  */
1338
1339 int r128_cce_flip(DRM_IOCTL_ARGS)
1340 {
1341         DRM_DEVICE;
1342         drm_r128_private_t *dev_priv = dev->dev_private;
1343         DRM_DEBUG("%s\n", __FUNCTION__);
1344
1345         LOCK_TEST_WITH_RETURN(dev, filp);
1346
1347         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1348
1349         if (!dev_priv->page_flipping)
1350                 r128_do_init_pageflip(dev);
1351
1352         r128_cce_dispatch_flip(dev);
1353
1354         COMMIT_RING();
1355         return 0;
1356 }
1357
1358 int r128_cce_swap(DRM_IOCTL_ARGS)
1359 {
1360         DRM_DEVICE;
1361         drm_r128_private_t *dev_priv = dev->dev_private;
1362         drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
1363         DRM_DEBUG("%s\n", __FUNCTION__);
1364
1365         LOCK_TEST_WITH_RETURN(dev, filp);
1366
1367         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1368
1369         if (sarea_priv->nbox > R128_NR_SAREA_CLIPRECTS)
1370                 sarea_priv->nbox = R128_NR_SAREA_CLIPRECTS;
1371
1372         r128_cce_dispatch_swap(dev);
1373         dev_priv->sarea_priv->dirty |= (R128_UPLOAD_CONTEXT |
1374                                         R128_UPLOAD_MASKS);
1375
1376         COMMIT_RING();
1377         return 0;
1378 }
1379
1380 int r128_cce_vertex(DRM_IOCTL_ARGS)
1381 {
1382         DRM_DEVICE;
1383         drm_r128_private_t *dev_priv = dev->dev_private;
1384         drm_device_dma_t *dma = dev->dma;
1385         drm_buf_t *buf;
1386         drm_r128_buf_priv_t *buf_priv;
1387         drm_r128_vertex_t vertex;
1388
1389         LOCK_TEST_WITH_RETURN(dev, filp);
1390
1391         if (!dev_priv) {
1392                 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
1393                 return DRM_ERR(EINVAL);
1394         }
1395
1396         DRM_COPY_FROM_USER_IOCTL(vertex, (drm_r128_vertex_t __user *) data,
1397                                  sizeof(vertex));
1398
1399         DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n",
1400                   DRM_CURRENTPID, vertex.idx, vertex.count, vertex.discard);
1401
1402         if (vertex.idx < 0 || vertex.idx >= dma->buf_count) {
1403                 DRM_ERROR("buffer index %d (of %d max)\n",
1404                           vertex.idx, dma->buf_count - 1);
1405                 return DRM_ERR(EINVAL);
1406         }
1407         if (vertex.prim < 0 ||
1408             vertex.prim > R128_CCE_VC_CNTL_PRIM_TYPE_TRI_TYPE2) {
1409                 DRM_ERROR("buffer prim %d\n", vertex.prim);
1410                 return DRM_ERR(EINVAL);
1411         }
1412
1413         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1414         VB_AGE_TEST_WITH_RETURN(dev_priv);
1415
1416         buf = dma->buflist[vertex.idx];
1417         buf_priv = buf->dev_private;
1418
1419         if (buf->filp != filp) {
1420                 DRM_ERROR("process %d using buffer owned by %p\n",
1421                           DRM_CURRENTPID, buf->filp);
1422                 return DRM_ERR(EINVAL);
1423         }
1424         if (buf->pending) {
1425                 DRM_ERROR("sending pending buffer %d\n", vertex.idx);
1426                 return DRM_ERR(EINVAL);
1427         }
1428
1429         buf->used = vertex.count;
1430         buf_priv->prim = vertex.prim;
1431         buf_priv->discard = vertex.discard;
1432
1433         r128_cce_dispatch_vertex(dev, buf);
1434
1435         COMMIT_RING();
1436         return 0;
1437 }
1438
1439 int r128_cce_indices(DRM_IOCTL_ARGS)
1440 {
1441         DRM_DEVICE;
1442         drm_r128_private_t *dev_priv = dev->dev_private;
1443         drm_device_dma_t *dma = dev->dma;
1444         drm_buf_t *buf;
1445         drm_r128_buf_priv_t *buf_priv;
1446         drm_r128_indices_t elts;
1447         int count;
1448
1449         LOCK_TEST_WITH_RETURN(dev, filp);
1450
1451         if (!dev_priv) {
1452                 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
1453                 return DRM_ERR(EINVAL);
1454         }
1455
1456         DRM_COPY_FROM_USER_IOCTL(elts, (drm_r128_indices_t __user *) data,
1457                                  sizeof(elts));
1458
1459         DRM_DEBUG("pid=%d buf=%d s=%d e=%d d=%d\n", DRM_CURRENTPID,
1460                   elts.idx, elts.start, elts.end, elts.discard);
1461
1462         if (elts.idx < 0 || elts.idx >= dma->buf_count) {
1463                 DRM_ERROR("buffer index %d (of %d max)\n",
1464                           elts.idx, dma->buf_count - 1);
1465                 return DRM_ERR(EINVAL);
1466         }
1467         if (elts.prim < 0 || elts.prim > R128_CCE_VC_CNTL_PRIM_TYPE_TRI_TYPE2) {
1468                 DRM_ERROR("buffer prim %d\n", elts.prim);
1469                 return DRM_ERR(EINVAL);
1470         }
1471
1472         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1473         VB_AGE_TEST_WITH_RETURN(dev_priv);
1474
1475         buf = dma->buflist[elts.idx];
1476         buf_priv = buf->dev_private;
1477
1478         if (buf->filp != filp) {
1479                 DRM_ERROR("process %d using buffer owned by %p\n",
1480                           DRM_CURRENTPID, buf->filp);
1481                 return DRM_ERR(EINVAL);
1482         }
1483         if (buf->pending) {
1484                 DRM_ERROR("sending pending buffer %d\n", elts.idx);
1485                 return DRM_ERR(EINVAL);
1486         }
1487
1488         count = (elts.end - elts.start) / sizeof(u16);
1489         elts.start -= R128_INDEX_PRIM_OFFSET;
1490
1491         if (elts.start & 0x7) {
1492                 DRM_ERROR("misaligned buffer 0x%x\n", elts.start);
1493                 return DRM_ERR(EINVAL);
1494         }
1495         if (elts.start < buf->used) {
1496                 DRM_ERROR("no header 0x%x - 0x%x\n", elts.start, buf->used);
1497                 return DRM_ERR(EINVAL);
1498         }
1499
1500         buf->used = elts.end;
1501         buf_priv->prim = elts.prim;
1502         buf_priv->discard = elts.discard;
1503
1504         r128_cce_dispatch_indices(dev, buf, elts.start, elts.end, count);
1505
1506         COMMIT_RING();
1507         return 0;
1508 }
1509
1510 int r128_cce_blit(DRM_IOCTL_ARGS)
1511 {
1512         DRM_DEVICE;
1513         drm_device_dma_t *dma = dev->dma;
1514         drm_r128_private_t *dev_priv = dev->dev_private;
1515         drm_r128_blit_t blit;
1516         int ret;
1517
1518         LOCK_TEST_WITH_RETURN(dev, filp);
1519
1520         DRM_COPY_FROM_USER_IOCTL(blit, (drm_r128_blit_t __user *) data,
1521                                  sizeof(blit));
1522
1523         DRM_DEBUG("pid=%d index=%d\n", DRM_CURRENTPID, blit.idx);
1524
1525         if (blit.idx < 0 || blit.idx >= dma->buf_count) {
1526                 DRM_ERROR("buffer index %d (of %d max)\n",
1527                           blit.idx, dma->buf_count - 1);
1528                 return DRM_ERR(EINVAL);
1529         }
1530
1531         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1532         VB_AGE_TEST_WITH_RETURN(dev_priv);
1533
1534         ret = r128_cce_dispatch_blit(filp, dev, &blit);
1535
1536         COMMIT_RING();
1537         return ret;
1538 }
1539
1540 int r128_cce_depth(DRM_IOCTL_ARGS)
1541 {
1542         DRM_DEVICE;
1543         drm_r128_private_t *dev_priv = dev->dev_private;
1544         drm_r128_depth_t depth;
1545         int ret;
1546
1547         LOCK_TEST_WITH_RETURN(dev, filp);
1548
1549         DRM_COPY_FROM_USER_IOCTL(depth, (drm_r128_depth_t __user *) data,
1550                                  sizeof(depth));
1551
1552         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1553
1554         ret = DRM_ERR(EINVAL);
1555         switch (depth.func) {
1556         case R128_WRITE_SPAN:
1557                 ret = r128_cce_dispatch_write_span(dev, &depth);
1558         case R128_WRITE_PIXELS:
1559                 ret = r128_cce_dispatch_write_pixels(dev, &depth);
1560         case R128_READ_SPAN:
1561                 ret = r128_cce_dispatch_read_span(dev, &depth);
1562         case R128_READ_PIXELS:
1563                 ret = r128_cce_dispatch_read_pixels(dev, &depth);
1564         }
1565
1566         COMMIT_RING();
1567         return ret;
1568 }
1569
1570 int r128_cce_stipple(DRM_IOCTL_ARGS)
1571 {
1572         DRM_DEVICE;
1573         drm_r128_private_t *dev_priv = dev->dev_private;
1574         drm_r128_stipple_t stipple;
1575         u32 mask[32];
1576
1577         LOCK_TEST_WITH_RETURN(dev, filp);
1578
1579         DRM_COPY_FROM_USER_IOCTL(stipple, (drm_r128_stipple_t __user *) data,
1580                                  sizeof(stipple));
1581
1582         if (DRM_COPY_FROM_USER(&mask, stipple.mask, 32 * sizeof(u32)))
1583                 return DRM_ERR(EFAULT);
1584
1585         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1586
1587         r128_cce_dispatch_stipple(dev, mask);
1588
1589         COMMIT_RING();
1590         return 0;
1591 }
1592
1593 int r128_cce_indirect(DRM_IOCTL_ARGS)
1594 {
1595         DRM_DEVICE;
1596         drm_r128_private_t *dev_priv = dev->dev_private;
1597         drm_device_dma_t *dma = dev->dma;
1598         drm_buf_t *buf;
1599         drm_r128_buf_priv_t *buf_priv;
1600         drm_r128_indirect_t indirect;
1601 #if 0
1602         RING_LOCALS;
1603 #endif
1604
1605         LOCK_TEST_WITH_RETURN(dev, filp);
1606
1607         if (!dev_priv) {
1608                 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
1609                 return DRM_ERR(EINVAL);
1610         }
1611
1612         DRM_COPY_FROM_USER_IOCTL(indirect, (drm_r128_indirect_t __user *) data,
1613                                  sizeof(indirect));
1614
1615         DRM_DEBUG("indirect: idx=%d s=%d e=%d d=%d\n",
1616                   indirect.idx, indirect.start, indirect.end, indirect.discard);
1617
1618         if (indirect.idx < 0 || indirect.idx >= dma->buf_count) {
1619                 DRM_ERROR("buffer index %d (of %d max)\n",
1620                           indirect.idx, dma->buf_count - 1);
1621                 return DRM_ERR(EINVAL);
1622         }
1623
1624         buf = dma->buflist[indirect.idx];
1625         buf_priv = buf->dev_private;
1626
1627         if (buf->filp != filp) {
1628                 DRM_ERROR("process %d using buffer owned by %p\n",
1629                           DRM_CURRENTPID, buf->filp);
1630                 return DRM_ERR(EINVAL);
1631         }
1632         if (buf->pending) {
1633                 DRM_ERROR("sending pending buffer %d\n", indirect.idx);
1634                 return DRM_ERR(EINVAL);
1635         }
1636
1637         if (indirect.start < buf->used) {
1638                 DRM_ERROR("reusing indirect: start=0x%x actual=0x%x\n",
1639                           indirect.start, buf->used);
1640                 return DRM_ERR(EINVAL);
1641         }
1642
1643         RING_SPACE_TEST_WITH_RETURN(dev_priv);
1644         VB_AGE_TEST_WITH_RETURN(dev_priv);
1645
1646         buf->used = indirect.end;
1647         buf_priv->discard = indirect.discard;
1648
1649 #if 0
1650         /* Wait for the 3D stream to idle before the indirect buffer
1651          * containing 2D acceleration commands is processed.
1652          */
1653         BEGIN_RING(2);
1654         RADEON_WAIT_UNTIL_3D_IDLE();
1655         ADVANCE_RING();
1656 #endif
1657
1658         /* Dispatch the indirect buffer full of commands from the
1659          * X server.  This is insecure and is thus only available to
1660          * privileged clients.
1661          */
1662         r128_cce_dispatch_indirect(dev, buf, indirect.start, indirect.end);
1663
1664         COMMIT_RING();
1665         return 0;
1666 }
1667
1668 int r128_getparam(DRM_IOCTL_ARGS)
1669 {
1670         DRM_DEVICE;
1671         drm_r128_private_t *dev_priv = dev->dev_private;
1672         drm_r128_getparam_t param;
1673         int value;
1674
1675         if (!dev_priv) {
1676                 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
1677                 return DRM_ERR(EINVAL);
1678         }
1679
1680         DRM_COPY_FROM_USER_IOCTL(param, (drm_r128_getparam_t __user *) data,
1681                                  sizeof(param));
1682
1683         DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
1684
1685         switch (param.param) {
1686         case R128_PARAM_IRQ_NR:
1687                 value = dev->irq;
1688                 break;
1689         default:
1690                 return DRM_ERR(EINVAL);
1691         }
1692
1693         if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
1694                 DRM_ERROR("copy_to_user\n");
1695                 return DRM_ERR(EFAULT);
1696         }
1697
1698         return 0;
1699 }
1700
1701 void r128_driver_prerelease(drm_device_t * dev, DRMFILE filp)
1702 {
1703         if (dev->dev_private) {
1704                 drm_r128_private_t *dev_priv = dev->dev_private;
1705                 if (dev_priv->page_flipping) {
1706                         r128_do_cleanup_pageflip(dev);
1707                 }
1708         }
1709 }
1710
1711 void r128_driver_pretakedown(drm_device_t * dev)
1712 {
1713         r128_do_cleanup_cce(dev);
1714 }