shared-core/radeon_cp.c

   1 /* radeon_cp.c -- CP support for Radeon -*- linux-c -*- */
   2 /*
   3  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   4  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   5  * Copyright 2007 Advanced Micro Devices, Inc.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  23  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  24  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  *
  27  * Authors:
  28  *    Kevin E. Martin <martin@valinux.com>
  29  *    Gareth Hughes <gareth@valinux.com>
  30  */
  31
  32 #include "drmP.h"
  33 #include "drm.h"
  34 #include "radeon_drm.h"
  35 #include "radeon_drv.h"
  36 #include "r300_reg.h"
  37
  38 #include "radeon_microcode.h"
  39 #define RADEON_FIFO_DEBUG       0
  40
  41 static int radeon_do_cleanup_cp(struct drm_device * dev);
  42
  43 static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  44 {
  45         u32 ret;
  46         RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
  47         ret = RADEON_READ(R520_MC_IND_DATA);
  48         RADEON_WRITE(R520_MC_IND_INDEX, 0);
  49         return ret;
  50 }
  51
  52 static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  53 {
  54         u32 ret;
  55         RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
  56         ret = RADEON_READ(RS480_NB_MC_DATA);
  57         RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
  58         return ret;
  59 }
  60
  61 static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  62 {
  63         u32 ret;
  64         RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
  65         ret = RADEON_READ(RS690_MC_DATA);
  66         RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
  67         return ret;
  68 }
  69
  70 static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  71 {
  72         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  73             return RS690_READ_MCIND(dev_priv, addr);
  74         else
  75             return RS480_READ_MCIND(dev_priv, addr);
  76 }
  77
  78 u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
  79 {
  80
  81         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
  82                 return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
  83         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  84                 return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
  85         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
  86                 return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
  87         else
  88                 return RADEON_READ(RADEON_MC_FB_LOCATION);
  89 }
  90
  91 static void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
  92 {
  93         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
  94                 R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
  95         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  96                 RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
  97         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
  98                 R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
  99         else
 100                 RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
 101 }
 102
 103 static void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc)
 104 {
 105         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 106                 R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
 107         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 108                 RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
 109         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 110                 R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
 111         else
 112                 RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
 113 }
 114
 115 static void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
 116 {
 117         u32 agp_base_hi = upper_32_bits(agp_base);
 118         u32 agp_base_lo = agp_base & 0xffffffff;
 119
 120         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
 121                 R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
 122                 R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
 123         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
 124                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
 125                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
 126         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
 127                 R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
 128                 R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
 129         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 130                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 131                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 132                 RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
 133         } else {
 134                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 135                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
 136                         RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
 137         }
 138 }
 139
 140 static int RADEON_READ_PLL(struct drm_device * dev, int addr)
 141 {
 142         drm_radeon_private_t *dev_priv = dev->dev_private;
 143
 144         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x1f);
 145         return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 146 }
 147
 148 static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
 149 {
 150         RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
 151         return RADEON_READ(RADEON_PCIE_DATA);
 152 }
 153
 154 #if RADEON_FIFO_DEBUG
 155 static void radeon_status(drm_radeon_private_t * dev_priv)
 156 {
 157         printk("%s:\n", __FUNCTION__);
 158         printk("RBBM_STATUS = 0x%08x\n",
 159                (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
 160         printk("CP_RB_RTPR = 0x%08x\n",
 161                (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
 162         printk("CP_RB_WTPR = 0x%08x\n",
 163                (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
 164         printk("AIC_CNTL = 0x%08x\n",
 165                (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
 166         printk("AIC_STAT = 0x%08x\n",
 167                (unsigned int)RADEON_READ(RADEON_AIC_STAT));
 168         printk("AIC_PT_BASE = 0x%08x\n",
 169                (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
 170         printk("TLB_ADDR = 0x%08x\n",
 171                (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
 172         printk("TLB_DATA = 0x%08x\n",
 173                (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
 174 }
 175 #endif
 176
 177 /* ================================================================
 178  * Engine, FIFO control
 179  */
 180
 181 static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
 182 {
 183         u32 tmp;
 184         int i;
 185
 186         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 187
 188         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
 189                 tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
 190                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 191                 RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
 192
 193                 for (i = 0; i < dev_priv->usec_timeout; i++) {
 194                         if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
 195                               & RADEON_RB3D_DC_BUSY)) {
 196                                 return 0;
 197                         }
 198                         DRM_UDELAY(1);
 199                 }
 200         } else {
 201                 /* 3D */
 202                 tmp = RADEON_READ(R300_RB3D_DSTCACHE_CTLSTAT);
 203                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 204                 RADEON_WRITE(R300_RB3D_DSTCACHE_CTLSTAT, tmp);
 205
 206                 /* 2D */
 207                 tmp = RADEON_READ(R300_DSTCACHE_CTLSTAT);
 208                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 209                 RADEON_WRITE(R300_DSTCACHE_CTLSTAT, tmp);
 210
 211                 for (i = 0; i < dev_priv->usec_timeout; i++) {
 212                         if (!(RADEON_READ(R300_DSTCACHE_CTLSTAT)
 213                           & RADEON_RB3D_DC_BUSY)) {
 214                                 return 0;
 215                         }
 216                         DRM_UDELAY(1);
 217                 }
 218         }
 219
 220 #if RADEON_FIFO_DEBUG
 221         DRM_ERROR("failed!\n");
 222         radeon_status(dev_priv);
 223 #endif
 224         return -EBUSY;
 225 }
 226
 227 static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
 228 {
 229         int i;
 230
 231         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 232
 233         for (i = 0; i < dev_priv->usec_timeout; i++) {
 234                 int slots = (RADEON_READ(RADEON_RBBM_STATUS)
 235                              & RADEON_RBBM_FIFOCNT_MASK);
 236                 if (slots >= entries)
 237                         return 0;
 238                 DRM_UDELAY(1);
 239         }
 240
 241 #if RADEON_FIFO_DEBUG
 242         DRM_ERROR("failed!\n");
 243         radeon_status(dev_priv);
 244 #endif
 245         return -EBUSY;
 246 }
 247
 248 static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
 249 {
 250         int i, ret;
 251
 252         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 253
 254         ret = radeon_do_wait_for_fifo(dev_priv, 64);
 255         if (ret)
 256                 return ret;
 257
 258         for (i = 0; i < dev_priv->usec_timeout; i++) {
 259                 if (!(RADEON_READ(RADEON_RBBM_STATUS)
 260                       & RADEON_RBBM_ACTIVE)) {
 261                         radeon_do_pixcache_flush(dev_priv);
 262                         return 0;
 263                 }
 264                 DRM_UDELAY(1);
 265         }
 266
 267 #if RADEON_FIFO_DEBUG
 268         DRM_ERROR("failed!\n");
 269         radeon_status(dev_priv);
 270 #endif
 271         return -EBUSY;
 272 }
 273
 274 static void radeon_init_pipes(drm_radeon_private_t * dev_priv)
 275 {
 276         uint32_t gb_tile_config, gb_pipe_sel = 0;
 277
 278         /* RS4xx/RS6xx/R4xx/R5xx */
 279         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
 280                 gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
 281                 dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
 282         } else {
 283                 /* R3xx */
 284                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 285                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350)) {
 286                         dev_priv->num_gb_pipes = 2;
 287                 } else {
 288                         /* R3Vxx */
 289                         dev_priv->num_gb_pipes = 1;
 290                 }
 291         }
 292         DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
 293
 294         gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
 295
 296         switch(dev_priv->num_gb_pipes) {
 297         case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
 298         case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
 299         case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
 300         default:
 301         case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
 302         }
 303
 304         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 305                 RADEON_WRITE_PLL(R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
 306                 RADEON_WRITE(R500_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
 307         }
 308         RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
 309         radeon_do_wait_for_idle(dev_priv);
 310         RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
 311         RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
 312                                                R300_DC_AUTOFLUSH_ENABLE |
 313                                                R300_DC_DC_DISABLE_IGNORE_PE));
 314
 315
 316 }
 317
 318 /* ================================================================
 319  * CP control, initialization
 320  */
 321
 322 /* Load the microcode for the CP */
 323 static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 324 {
 325         int i;
 326         DRM_DEBUG("\n");
 327
 328         radeon_do_wait_for_idle(dev_priv);
 329
 330         RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 331
 332         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
 333             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
 334             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
 335             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
 336             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
 337                 DRM_INFO("Loading R100 Microcode\n");
 338                 for (i = 0; i < 256; i++) {
 339                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 340                                      R100_cp_microcode[i][1]);
 341                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 342                                      R100_cp_microcode[i][0]);
 343                 }
 344         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
 345                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
 346                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
 347                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
 348                 DRM_INFO("Loading R200 Microcode\n");
 349                 for (i = 0; i < 256; i++) {
 350                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 351                                      R200_cp_microcode[i][1]);
 352                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 353                                      R200_cp_microcode[i][0]);
 354                 }
 355         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 356                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
 357                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
 358                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
 359                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 360                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 361                 DRM_INFO("Loading R300 Microcode\n");
 362                 for (i = 0; i < 256; i++) {
 363                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 364                                      R300_cp_microcode[i][1]);
 365                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 366                                      R300_cp_microcode[i][0]);
 367                 }
 368         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 369                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
 370                 DRM_INFO("Loading R400 Microcode\n");
 371                 for (i = 0; i < 256; i++) {
 372                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 373                                      R420_cp_microcode[i][1]);
 374                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 375                                      R420_cp_microcode[i][0]);
 376                 }
 377         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
 378                 DRM_INFO("Loading RS690 Microcode\n");
 379                 for (i = 0; i < 256; i++) {
 380                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 381                                      RS690_cp_microcode[i][1]);
 382                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 383                                      RS690_cp_microcode[i][0]);
 384                 }
 385         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
 386                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
 387                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
 388                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
 389                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
 390                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
 391                 DRM_INFO("Loading R500 Microcode\n");
 392                 for (i = 0; i < 256; i++) {
 393                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 394                                      R520_cp_microcode[i][1]);
 395                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 396                                      R520_cp_microcode[i][0]);
 397                 }
 398         }
 399 }
 400
 401 /* Flush any pending commands to the CP.  This should only be used just
 402  * prior to a wait for idle, as it informs the engine that the command
 403  * stream is ending.
 404  */
 405 static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
 406 {
 407         DRM_DEBUG("\n");
 408 #if 0
 409         u32 tmp;
 410
 411         tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
 412         RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
 413 #endif
 414 }
 415
 416 /* Wait for the CP to go idle.
 417  */
 418 int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
 419 {
 420         RING_LOCALS;
 421         DRM_DEBUG("\n");
 422
 423         BEGIN_RING(6);
 424
 425         RADEON_PURGE_CACHE();
 426         RADEON_PURGE_ZCACHE();
 427         RADEON_WAIT_UNTIL_IDLE();
 428
 429         ADVANCE_RING();
 430         COMMIT_RING();
 431
 432         return radeon_do_wait_for_idle(dev_priv);
 433 }
 434
 435 /* Start the Command Processor.
 436  */
 437 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
 438 {
 439         RING_LOCALS;
 440         DRM_DEBUG("\n");
 441
 442         radeon_do_wait_for_idle(dev_priv);
 443
 444         RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
 445
 446         dev_priv->cp_running = 1;
 447
 448         BEGIN_RING(6);
 449
 450         RADEON_PURGE_CACHE();
 451         RADEON_PURGE_ZCACHE();
 452         RADEON_WAIT_UNTIL_IDLE();
 453
 454         ADVANCE_RING();
 455         COMMIT_RING();
 456 }
 457
 458 /* Reset the Command Processor.  This will not flush any pending
 459  * commands, so you must wait for the CP command stream to complete
 460  * before calling this routine.
 461  */
 462 static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
 463 {
 464         u32 cur_read_ptr;
 465         DRM_DEBUG("\n");
 466
 467         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 468         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 469         SET_RING_HEAD(dev_priv, cur_read_ptr);
 470         dev_priv->ring.tail = cur_read_ptr;
 471 }
 472
 473 /* Stop the Command Processor.  This will not flush any pending
 474  * commands, so you must flush the command stream and wait for the CP
 475  * to go idle before calling this routine.
 476  */
 477 static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
 478 {
 479         DRM_DEBUG("\n");
 480
 481         RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
 482
 483         dev_priv->cp_running = 0;
 484 }
 485
 486 /* Reset the engine.  This will stop the CP if it is running.
 487  */
 488 static int radeon_do_engine_reset(struct drm_device * dev)
 489 {
 490         drm_radeon_private_t *dev_priv = dev->dev_private;
 491         u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
 492         DRM_DEBUG("\n");
 493
 494         radeon_do_pixcache_flush(dev_priv);
 495
 496         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 497                 /* may need something similar for newer chips */
 498                 clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 499                 mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
 500
 501                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
 502                                                     RADEON_FORCEON_MCLKA |
 503                                                     RADEON_FORCEON_MCLKB |
 504                                                     RADEON_FORCEON_YCLKA |
 505                                                     RADEON_FORCEON_YCLKB |
 506                                                     RADEON_FORCEON_MC |
 507                                                     RADEON_FORCEON_AIC));
 508         }
 509
 510         rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
 511
 512         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
 513                                               RADEON_SOFT_RESET_CP |
 514                                               RADEON_SOFT_RESET_HI |
 515                                               RADEON_SOFT_RESET_SE |
 516                                               RADEON_SOFT_RESET_RE |
 517                                               RADEON_SOFT_RESET_PP |
 518                                               RADEON_SOFT_RESET_E2 |
 519                                               RADEON_SOFT_RESET_RB));
 520         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 521         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
 522                                               ~(RADEON_SOFT_RESET_CP |
 523                                                 RADEON_SOFT_RESET_HI |
 524                                                 RADEON_SOFT_RESET_SE |
 525                                                 RADEON_SOFT_RESET_RE |
 526                                                 RADEON_SOFT_RESET_PP |
 527                                                 RADEON_SOFT_RESET_E2 |
 528                                                 RADEON_SOFT_RESET_RB)));
 529         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 530
 531         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 532                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
 533                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
 534                 RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
 535         }
 536
 537         /* setup the raster pipes */
 538         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
 539             radeon_init_pipes(dev_priv);
 540
 541         /* Reset the CP ring */
 542         radeon_do_cp_reset(dev_priv);
 543
 544         /* The CP is no longer running after an engine reset */
 545         dev_priv->cp_running = 0;
 546
 547         /* Reset any pending vertex, indirect buffers */
 548         radeon_freelist_reset(dev);
 549
 550         return 0;
 551 }
 552
 553 static void radeon_cp_init_ring_buffer(struct drm_device * dev,
 554                                        drm_radeon_private_t * dev_priv)
 555 {
 556         u32 ring_start, cur_read_ptr;
 557         u32 tmp;
 558
 559         /* Initialize the memory controller. With new memory map, the fb location
 560          * is not changed, it should have been properly initialized already. Part
 561          * of the problem is that the code below is bogus, assuming the GART is
 562          * always appended to the fb which is not necessarily the case
 563          */
 564         if (!dev_priv->new_memmap)
 565                 radeon_write_fb_location(dev_priv,
 566                              ((dev_priv->gart_vm_start - 1) & 0xffff0000)
 567                              | (dev_priv->fb_location >> 16));
 568
 569 #if __OS_HAS_AGP
 570         if (dev_priv->flags & RADEON_IS_AGP) {
 571                 radeon_write_agp_base(dev_priv, dev->agp->base);
 572
 573                 radeon_write_agp_location(dev_priv,
 574                              (((dev_priv->gart_vm_start - 1 +
 575                                 dev_priv->gart_size) & 0xffff0000) |
 576                               (dev_priv->gart_vm_start >> 16)));
 577
 578                 ring_start = (dev_priv->cp_ring->offset
 579                               - dev->agp->base
 580                               + dev_priv->gart_vm_start);
 581         } else
 582 #endif
 583                 ring_start = (dev_priv->cp_ring->offset
 584                               - (unsigned long)dev->sg->virtual
 585                               + dev_priv->gart_vm_start);
 586
 587         RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
 588
 589         /* Set the write pointer delay */
 590         RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
 591
 592         /* Initialize the ring buffer's read and write pointers */
 593         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 594         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 595         SET_RING_HEAD(dev_priv, cur_read_ptr);
 596         dev_priv->ring.tail = cur_read_ptr;
 597
 598 #if __OS_HAS_AGP
 599         if (dev_priv->flags & RADEON_IS_AGP) {
 600                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 601                              dev_priv->ring_rptr->offset
 602                              - dev->agp->base + dev_priv->gart_vm_start);
 603         } else
 604 #endif
 605         {
 606                 struct drm_sg_mem *entry = dev->sg;
 607                 unsigned long tmp_ofs, page_ofs;
 608
 609                 tmp_ofs = dev_priv->ring_rptr->offset -
 610                                 (unsigned long)dev->sg->virtual;
 611                 page_ofs = tmp_ofs >> PAGE_SHIFT;
 612
 613                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR, entry->busaddr[page_ofs]);
 614                 DRM_DEBUG("ring rptr: offset=0x%08lx handle=0x%08lx\n",
 615                           (unsigned long)entry->busaddr[page_ofs],
 616                           entry->handle + tmp_ofs);
 617         }
 618
 619         /* Set ring buffer size */
 620 #ifdef __BIG_ENDIAN
 621         RADEON_WRITE(RADEON_CP_RB_CNTL,
 622                      RADEON_BUF_SWAP_32BIT |
 623                      (dev_priv->ring.fetch_size_l2ow << 18) |
 624                      (dev_priv->ring.rptr_update_l2qw << 8) |
 625                      dev_priv->ring.size_l2qw);
 626 #else
 627         RADEON_WRITE(RADEON_CP_RB_CNTL,
 628                      (dev_priv->ring.fetch_size_l2ow << 18) |
 629                      (dev_priv->ring.rptr_update_l2qw << 8) |
 630                      dev_priv->ring.size_l2qw);
 631 #endif
 632
 633         /* Start with assuming that writeback doesn't work */
 634         dev_priv->writeback_works = 0;
 635
 636         /* Initialize the scratch register pointer.  This will cause
 637          * the scratch register values to be written out to memory
 638          * whenever they are updated.
 639          *
 640          * We simply put this behind the ring read pointer, this works
 641          * with PCI GART as well as (whatever kind of) AGP GART
 642          */
 643         RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
 644                      + RADEON_SCRATCH_REG_OFFSET);
 645
 646         dev_priv->scratch = ((__volatile__ u32 *)
 647                              dev_priv->ring_rptr->handle +
 648                              (RADEON_SCRATCH_REG_OFFSET / sizeof(u32)));
 649
 650         RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
 651
 652         /* Turn on bus mastering */
 653         tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
 654         RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 655
 656         dev_priv->sarea_priv->last_frame = dev_priv->scratch[0] = 0;
 657         RADEON_WRITE(RADEON_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame);
 658
 659         dev_priv->sarea_priv->last_dispatch = dev_priv->scratch[1] = 0;
 660         RADEON_WRITE(RADEON_LAST_DISPATCH_REG,
 661                      dev_priv->sarea_priv->last_dispatch);
 662
 663         dev_priv->sarea_priv->last_clear = dev_priv->scratch[2] = 0;
 664         RADEON_WRITE(RADEON_LAST_CLEAR_REG, dev_priv->sarea_priv->last_clear);
 665
 666         radeon_do_wait_for_idle(dev_priv);
 667
 668         /* Sync everything up */
 669         RADEON_WRITE(RADEON_ISYNC_CNTL,
 670                      (RADEON_ISYNC_ANY2D_IDLE3D |
 671                       RADEON_ISYNC_ANY3D_IDLE2D |
 672                       RADEON_ISYNC_WAIT_IDLEGUI |
 673                       RADEON_ISYNC_CPSCRATCH_IDLEGUI));
 674
 675 }
 676
 677 static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
 678 {
 679         u32 tmp;
 680
 681         /* Writeback doesn't seem to work everywhere, test it here and possibly
 682          * enable it if it appears to work
 683          */
 684         DRM_WRITE32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1), 0);
 685         RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
 686
 687         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 688                 if (DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1)) ==
 689                     0xdeadbeef)
 690                         break;
 691                 DRM_UDELAY(1);
 692         }
 693
 694         if (tmp < dev_priv->usec_timeout) {
 695                 dev_priv->writeback_works = 1;
 696                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 697         } else {
 698                 dev_priv->writeback_works = 0;
 699                 DRM_INFO("writeback test failed\n");
 700         }
 701         if (radeon_no_wb == 1) {
 702                 dev_priv->writeback_works = 0;
 703                 DRM_INFO("writeback forced off\n");
 704         }
 705
 706         if (!dev_priv->writeback_works) {
 707                 /* Disable writeback to avoid unnecessary bus master transfers */
 708                 RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) | RADEON_RB_NO_UPDATE);
 709                 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
 710         }
 711 }
 712
 713 /* Enable or disable IGP GART on the chip */
 714 static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
 715 {
 716         u32 temp;
 717
 718         if (on) {
 719                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 720                          dev_priv->gart_vm_start,
 721                          (long)dev_priv->gart_info.bus_addr,
 722                          dev_priv->gart_size);
 723
 724                 temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
 725
 726                 if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 727                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
 728                                                              RS690_BLOCK_GFX_D3_EN));
 729                 else
 730                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
 731
 732                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 733                                                                RS480_VA_SIZE_32MB));
 734
 735                 temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
 736                 IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
 737                                                         RS480_TLB_ENABLE |
 738                                                         RS480_GTW_LAC_EN |
 739                                                         RS480_1LEVEL_GART));
 740
 741                 temp = dev_priv->gart_info.bus_addr & 0xfffff000;
 742                 temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
 743                 IGP_WRITE_MCIND(RS480_GART_BASE, temp);
 744
 745                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
 746                 IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
 747                                                       RS480_REQ_TYPE_SNOOP_DIS));
 748
 749                 radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
 750
 751                 dev_priv->gart_size = 32*1024*1024;
 752                 temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) &
 753                         0xffff0000) | (dev_priv->gart_vm_start >> 16));
 754
 755                 radeon_write_agp_location(dev_priv, temp);
 756
 757                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
 758                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 759                                                                RS480_VA_SIZE_32MB));
 760
 761                 do {
 762                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 763                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 764                                 break;
 765                         DRM_UDELAY(1);
 766                 } while(1);
 767
 768                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
 769                                 RS480_GART_CACHE_INVALIDATE);
 770
 771                 do {
 772                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 773                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 774                                 break;
 775                         DRM_UDELAY(1);
 776                 } while(1);
 777
 778                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
 779         } else {
 780                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
 781         }
 782 }
 783
 784 static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 785 {
 786         u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
 787         if (on) {
 788
 789                 DRM_DEBUG("programming pcie %08X %08lX %08X\n",
 790                           dev_priv->gart_vm_start,
 791                           (long)dev_priv->gart_info.bus_addr,
 792                           dev_priv->gart_size);
 793                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
 794                                   dev_priv->gart_vm_start);
 795                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
 796                                   dev_priv->gart_info.bus_addr);
 797                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
 798                                   dev_priv->gart_vm_start);
 799                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
 800                                   dev_priv->gart_vm_start +
 801                                   dev_priv->gart_size - 1);
 802
 803                 radeon_write_agp_location(dev_priv, 0xffffffc0); /* ?? */
 804
 805                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 806                                   RADEON_PCIE_TX_GART_EN);
 807         } else {
 808                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 809                                   tmp & ~RADEON_PCIE_TX_GART_EN);
 810         }
 811 }
 812
 813 /* Enable or disable PCI GART on the chip */
 814 static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 815 {
 816         u32 tmp;
 817
 818         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 819             (dev_priv->flags & RADEON_IS_IGPGART)) {
 820                 radeon_set_igpgart(dev_priv, on);
 821                 return;
 822         }
 823
 824         if (dev_priv->flags & RADEON_IS_PCIE) {
 825                 radeon_set_pciegart(dev_priv, on);
 826                 return;
 827         }
 828
 829         tmp = RADEON_READ(RADEON_AIC_CNTL);
 830
 831         if (on) {
 832                 RADEON_WRITE(RADEON_AIC_CNTL,
 833                              tmp | RADEON_PCIGART_TRANSLATE_EN);
 834
 835                 /* set PCI GART page-table base address
 836                  */
 837                 RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
 838
 839                 /* set address range for PCI address translate
 840                  */
 841                 RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
 842                 RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
 843                              + dev_priv->gart_size - 1);
 844
 845                 /* Turn off AGP aperture -- is this required for PCI GART?
 846                  */
 847                 radeon_write_agp_location(dev_priv, 0xffffffc0);
 848                 RADEON_WRITE(RADEON_AGP_COMMAND, 0);    /* clear AGP_COMMAND */
 849         } else {
 850                 RADEON_WRITE(RADEON_AIC_CNTL,
 851                              tmp & ~RADEON_PCIGART_TRANSLATE_EN);
 852         }
 853 }
 854
 855 static int radeon_do_init_cp(struct drm_device * dev, drm_radeon_init_t * init)
 856 {
 857         drm_radeon_private_t *dev_priv = dev->dev_private;
 858
 859         DRM_DEBUG("\n");
 860
 861         /* if we require new memory map but we don't have it fail */
 862         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
 863                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
 864                 radeon_do_cleanup_cp(dev);
 865                 return -EINVAL;
 866         }
 867
 868         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP))
 869         {
 870                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
 871                 dev_priv->flags &= ~RADEON_IS_AGP;
 872         }
 873         else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
 874                  && !init->is_pci)
 875         {
 876                 DRM_DEBUG("Restoring AGP flag\n");
 877                 dev_priv->flags |= RADEON_IS_AGP;
 878         }
 879
 880         if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
 881                 DRM_ERROR("PCI GART memory not allocated!\n");
 882                 radeon_do_cleanup_cp(dev);
 883                 return -EINVAL;
 884         }
 885
 886         dev_priv->usec_timeout = init->usec_timeout;
 887         if (dev_priv->usec_timeout < 1 ||
 888             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
 889                 DRM_DEBUG("TIMEOUT problem!\n");
 890                 radeon_do_cleanup_cp(dev);
 891                 return -EINVAL;
 892         }
 893
 894         /* Enable vblank on CRTC1 for older X servers
 895          */
 896         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
 897
 898         switch(init->func) {
 899         case RADEON_INIT_R200_CP:
 900                 dev_priv->microcode_version = UCODE_R200;
 901                 break;
 902         case RADEON_INIT_R300_CP:
 903                 dev_priv->microcode_version = UCODE_R300;
 904                 break;
 905         default:
 906                 dev_priv->microcode_version = UCODE_R100;
 907         }
 908
 909         dev_priv->do_boxes = 0;
 910         dev_priv->cp_mode = init->cp_mode;
 911
 912         /* We don't support anything other than bus-mastering ring mode,
 913          * but the ring can be in either AGP or PCI space for the ring
 914          * read pointer.
 915          */
 916         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
 917             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
 918                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
 919                 radeon_do_cleanup_cp(dev);
 920                 return -EINVAL;
 921         }
 922
 923         switch (init->fb_bpp) {
 924         case 16:
 925                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
 926                 break;
 927         case 32:
 928         default:
 929                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
 930                 break;
 931         }
 932         dev_priv->front_offset = init->front_offset;
 933         dev_priv->front_pitch = init->front_pitch;
 934         dev_priv->back_offset = init->back_offset;
 935         dev_priv->back_pitch = init->back_pitch;
 936
 937         switch (init->depth_bpp) {
 938         case 16:
 939                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
 940                 break;
 941         case 32:
 942         default:
 943                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
 944                 break;
 945         }
 946         dev_priv->depth_offset = init->depth_offset;
 947         dev_priv->depth_pitch = init->depth_pitch;
 948
 949         /* Hardware state for depth clears.  Remove this if/when we no
 950          * longer clear the depth buffer with a 3D rectangle.  Hard-code
 951          * all values to prevent unwanted 3D state from slipping through
 952          * and screwing with the clear operation.
 953          */
 954         dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
 955                                            (dev_priv->color_fmt << 10) |
 956                                            (dev_priv->microcode_version ==
 957                                             UCODE_R100 ? RADEON_ZBLOCK16 : 0));
 958
 959         dev_priv->depth_clear.rb3d_zstencilcntl =
 960             (dev_priv->depth_fmt |
 961              RADEON_Z_TEST_ALWAYS |
 962              RADEON_STENCIL_TEST_ALWAYS |
 963              RADEON_STENCIL_S_FAIL_REPLACE |
 964              RADEON_STENCIL_ZPASS_REPLACE |
 965              RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
 966
 967         dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
 968                                          RADEON_BFACE_SOLID |
 969                                          RADEON_FFACE_SOLID |
 970                                          RADEON_FLAT_SHADE_VTX_LAST |
 971                                          RADEON_DIFFUSE_SHADE_FLAT |
 972                                          RADEON_ALPHA_SHADE_FLAT |
 973                                          RADEON_SPECULAR_SHADE_FLAT |
 974                                          RADEON_FOG_SHADE_FLAT |
 975                                          RADEON_VTX_PIX_CENTER_OGL |
 976                                          RADEON_ROUND_MODE_TRUNC |
 977                                          RADEON_ROUND_PREC_8TH_PIX);
 978
 979
 980         dev_priv->ring_offset = init->ring_offset;
 981         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
 982         dev_priv->buffers_offset = init->buffers_offset;
 983         dev_priv->gart_textures_offset = init->gart_textures_offset;
 984
 985         dev_priv->sarea = drm_getsarea(dev);
 986         if (!dev_priv->sarea) {
 987                 DRM_ERROR("could not find sarea!\n");
 988                 radeon_do_cleanup_cp(dev);
 989                 return -EINVAL;
 990         }
 991
 992         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
 993         if (!dev_priv->cp_ring) {
 994                 DRM_ERROR("could not find cp ring region!\n");
 995                 radeon_do_cleanup_cp(dev);
 996                 return -EINVAL;
 997         }
 998         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
 999         if (!dev_priv->ring_rptr) {
1000                 DRM_ERROR("could not find ring read pointer!\n");
1001                 radeon_do_cleanup_cp(dev);
1002                 return -EINVAL;
1003         }
1004         dev->agp_buffer_token = init->buffers_offset;
1005         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1006         if (!dev->agp_buffer_map) {
1007                 DRM_ERROR("could not find dma buffer region!\n");
1008                 radeon_do_cleanup_cp(dev);
1009                 return -EINVAL;
1010         }
1011
1012         if (init->gart_textures_offset) {
1013                 dev_priv->gart_textures =
1014                     drm_core_findmap(dev, init->gart_textures_offset);
1015                 if (!dev_priv->gart_textures) {
1016                         DRM_ERROR("could not find GART texture region!\n");
1017                         radeon_do_cleanup_cp(dev);
1018                         return -EINVAL;
1019                 }
1020         }
1021
1022         dev_priv->sarea_priv =
1023             (drm_radeon_sarea_t *) ((u8 *) dev_priv->sarea->handle +
1024                                     init->sarea_priv_offset);
1025
1026 #if __OS_HAS_AGP
1027         if (dev_priv->flags & RADEON_IS_AGP) {
1028                 drm_core_ioremap(dev_priv->cp_ring, dev);
1029                 drm_core_ioremap(dev_priv->ring_rptr, dev);
1030                 drm_core_ioremap(dev->agp_buffer_map, dev);
1031                 if (!dev_priv->cp_ring->handle ||
1032                     !dev_priv->ring_rptr->handle ||
1033                     !dev->agp_buffer_map->handle) {
1034                         DRM_ERROR("could not find ioremap agp regions!\n");
1035                         radeon_do_cleanup_cp(dev);
1036                         return -EINVAL;
1037                 }
1038         } else
1039 #endif
1040         {
1041                 dev_priv->cp_ring->handle = (void *)dev_priv->cp_ring->offset;
1042                 dev_priv->ring_rptr->handle =
1043                     (void *)dev_priv->ring_rptr->offset;
1044                 dev->agp_buffer_map->handle =
1045                     (void *)dev->agp_buffer_map->offset;
1046
1047                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
1048                           dev_priv->cp_ring->handle);
1049                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
1050                           dev_priv->ring_rptr->handle);
1051                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
1052                           dev->agp_buffer_map->handle);
1053         }
1054
1055         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
1056         dev_priv->fb_size =
1057                 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
1058                 - dev_priv->fb_location;
1059
1060         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1061                                         ((dev_priv->front_offset
1062                                           + dev_priv->fb_location) >> 10));
1063
1064         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1065                                        ((dev_priv->back_offset
1066                                          + dev_priv->fb_location) >> 10));
1067
1068         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1069                                         ((dev_priv->depth_offset
1070                                           + dev_priv->fb_location) >> 10));
1071
1072         dev_priv->gart_size = init->gart_size;
1073
1074         /* New let's set the memory map ... */
1075         if (dev_priv->new_memmap) {
1076                 u32 base = 0;
1077
1078                 DRM_INFO("Setting GART location based on new memory map\n");
1079
1080                 /* If using AGP, try to locate the AGP aperture at the same
1081                  * location in the card and on the bus, though we have to
1082                  * align it down.
1083                  */
1084 #if __OS_HAS_AGP
1085                 if (dev_priv->flags & RADEON_IS_AGP) {
1086                         base = dev->agp->base;
1087                         /* Check if valid */
1088                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
1089                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
1090                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
1091                                          dev->agp->base);
1092                                 base = 0;
1093                         }
1094                 }
1095 #endif
1096                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
1097                 if (base == 0) {
1098                         base = dev_priv->fb_location + dev_priv->fb_size;
1099                         if (base < dev_priv->fb_location ||
1100                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
1101                                 base = dev_priv->fb_location
1102                                         - dev_priv->gart_size;
1103                 }
1104                 dev_priv->gart_vm_start = base & 0xffc00000u;
1105                 if (dev_priv->gart_vm_start != base)
1106                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
1107                                  base, dev_priv->gart_vm_start);
1108         } else {
1109                 DRM_INFO("Setting GART location based on old memory map\n");
1110                 dev_priv->gart_vm_start = dev_priv->fb_location +
1111                         RADEON_READ(RADEON_CONFIG_APER_SIZE);
1112         }
1113
1114 #if __OS_HAS_AGP
1115         if (dev_priv->flags & RADEON_IS_AGP)
1116                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1117                                                  - dev->agp->base
1118                                                  + dev_priv->gart_vm_start);
1119         else
1120 #endif
1121                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1122                                         - (unsigned long)dev->sg->virtual
1123                                         + dev_priv->gart_vm_start);
1124
1125         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
1126         DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
1127         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
1128                   dev_priv->gart_buffers_offset);
1129
1130         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
1131         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
1132                               + init->ring_size / sizeof(u32));
1133         dev_priv->ring.size = init->ring_size;
1134         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
1135
1136         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
1137         dev_priv->ring.rptr_update_l2qw = drm_order( /* init->rptr_update */ 4096 / 8);
1138
1139         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
1140         dev_priv->ring.fetch_size_l2ow = drm_order( /* init->fetch_size */ 32 / 16);
1141
1142         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1143
1144         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
1145
1146 #if __OS_HAS_AGP
1147         if (dev_priv->flags & RADEON_IS_AGP) {
1148                 /* Turn off PCI GART */
1149                 radeon_set_pcigart(dev_priv, 0);
1150         } else
1151 #endif
1152         {
1153                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
1154                 /* if we have an offset set from userspace */
1155                 if (dev_priv->pcigart_offset_set) {
1156                         dev_priv->gart_info.bus_addr =
1157                             dev_priv->pcigart_offset + dev_priv->fb_location;
1158                         dev_priv->gart_info.mapping.offset =
1159                             dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
1160                         dev_priv->gart_info.mapping.size =
1161                             dev_priv->gart_info.table_size;
1162
1163                         drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
1164                         dev_priv->gart_info.addr =
1165                             dev_priv->gart_info.mapping.handle;
1166
1167                         if (dev_priv->flags & RADEON_IS_PCIE)
1168                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
1169                         else
1170                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1171                         dev_priv->gart_info.gart_table_location =
1172                             DRM_ATI_GART_FB;
1173
1174                         DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
1175                                   dev_priv->gart_info.addr,
1176                                   dev_priv->pcigart_offset);
1177                 } else {
1178                         if (dev_priv->flags & RADEON_IS_IGPGART)
1179                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
1180                         else
1181                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1182                         dev_priv->gart_info.gart_table_location =
1183                             DRM_ATI_GART_MAIN;
1184                         dev_priv->gart_info.addr = NULL;
1185                         dev_priv->gart_info.bus_addr = 0;
1186                         if (dev_priv->flags & RADEON_IS_PCIE) {
1187                                 DRM_ERROR
1188                                     ("Cannot use PCI Express without GART in FB memory\n");
1189                                 radeon_do_cleanup_cp(dev);
1190                                 return -EINVAL;
1191                         }
1192                 }
1193
1194                 if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
1195                         DRM_ERROR("failed to init PCI GART!\n");
1196                         radeon_do_cleanup_cp(dev);
1197                         return -ENOMEM;
1198                 }
1199
1200                 /* Turn on PCI GART */
1201                 radeon_set_pcigart(dev_priv, 1);
1202         }
1203
1204         radeon_cp_load_microcode(dev_priv);
1205         radeon_cp_init_ring_buffer(dev, dev_priv);
1206
1207         dev_priv->last_buf = 0;
1208
1209         radeon_do_engine_reset(dev);
1210         radeon_test_writeback(dev_priv);
1211
1212         return 0;
1213 }
1214
1215 static int radeon_do_cleanup_cp(struct drm_device * dev)
1216 {
1217         drm_radeon_private_t *dev_priv = dev->dev_private;
1218         DRM_DEBUG("\n");
1219
1220         /* Make sure interrupts are disabled here because the uninstall ioctl
1221          * may not have been called from userspace and after dev_private
1222          * is freed, it's too late.
1223          */
1224         if (dev->irq_enabled)
1225                 drm_irq_uninstall(dev);
1226
1227 #if __OS_HAS_AGP
1228         if (dev_priv->flags & RADEON_IS_AGP) {
1229                 if (dev_priv->cp_ring != NULL) {
1230                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1231                         dev_priv->cp_ring = NULL;
1232                 }
1233                 if (dev_priv->ring_rptr != NULL) {
1234                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1235                         dev_priv->ring_rptr = NULL;
1236                 }
1237                 if (dev->agp_buffer_map != NULL) {
1238                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1239                         dev->agp_buffer_map = NULL;
1240                 }
1241         } else
1242 #endif
1243         {
1244
1245                 if (dev_priv->gart_info.bus_addr) {
1246                         /* Turn off PCI GART */
1247                         radeon_set_pcigart(dev_priv, 0);
1248                         if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
1249                                 DRM_ERROR("failed to cleanup PCI GART!\n");
1250                 }
1251
1252                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
1253                 {
1254                         drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1255                         dev_priv->gart_info.addr = 0;
1256                 }
1257         }
1258         /* only clear to the start of flags */
1259         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1260
1261         return 0;
1262 }
1263
1264 /* This code will reinit the Radeon CP hardware after a resume from disc.
1265  * AFAIK, it would be very difficult to pickle the state at suspend time, so
1266  * here we make sure that all Radeon hardware initialisation is re-done without
1267  * affecting running applications.
1268  *
1269  * Charl P. Botha <http://cpbotha.net>
1270  */
1271 static int radeon_do_resume_cp(struct drm_device * dev)
1272 {
1273         drm_radeon_private_t *dev_priv = dev->dev_private;
1274
1275         if (!dev_priv) {
1276                 DRM_ERROR("Called with no initialization\n");
1277                 return -EINVAL;
1278         }
1279
1280         DRM_DEBUG("Starting radeon_do_resume_cp()\n");
1281
1282 #if __OS_HAS_AGP
1283         if (dev_priv->flags & RADEON_IS_AGP) {
1284                 /* Turn off PCI GART */
1285                 radeon_set_pcigart(dev_priv, 0);
1286         } else
1287 #endif
1288         {
1289                 /* Turn on PCI GART */
1290                 radeon_set_pcigart(dev_priv, 1);
1291         }
1292
1293         radeon_cp_load_microcode(dev_priv);
1294         radeon_cp_init_ring_buffer(dev, dev_priv);
1295
1296         radeon_do_engine_reset(dev);
1297         radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
1298
1299         DRM_DEBUG("radeon_do_resume_cp() complete\n");
1300
1301         return 0;
1302 }
1303
1304 int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
1305 {
1306         drm_radeon_init_t *init = data;
1307
1308         LOCK_TEST_WITH_RETURN(dev, file_priv);
1309
1310         if (init->func == RADEON_INIT_R300_CP)
1311                 r300_init_reg_flags(dev);
1312
1313         switch (init->func) {
1314         case RADEON_INIT_CP:
1315         case RADEON_INIT_R200_CP:
1316         case RADEON_INIT_R300_CP:
1317                 return radeon_do_init_cp(dev, init);
1318         case RADEON_CLEANUP_CP:
1319                 return radeon_do_cleanup_cp(dev);
1320         }
1321
1322         return -EINVAL;
1323 }
1324
1325 int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
1326 {
1327         drm_radeon_private_t *dev_priv = dev->dev_private;
1328         DRM_DEBUG("\n");
1329
1330         LOCK_TEST_WITH_RETURN(dev, file_priv);
1331
1332         if (dev_priv->cp_running) {
1333                 DRM_DEBUG("while CP running\n");
1334                 return 0;
1335         }
1336         if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
1337                 DRM_DEBUG("called with bogus CP mode (%d)\n",
1338                           dev_priv->cp_mode);
1339                 return 0;
1340         }
1341
1342         radeon_do_cp_start(dev_priv);
1343
1344         return 0;
1345 }
1346
1347 /* Stop the CP.  The engine must have been idled before calling this
1348  * routine.
1349  */
1350 int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
1351 {
1352         drm_radeon_private_t *dev_priv = dev->dev_private;
1353         drm_radeon_cp_stop_t *stop = data;
1354         int ret;
1355         DRM_DEBUG("\n");
1356
1357         LOCK_TEST_WITH_RETURN(dev, file_priv);
1358
1359         if (!dev_priv->cp_running)
1360                 return 0;
1361
1362         /* Flush any pending CP commands.  This ensures any outstanding
1363          * commands are exectuted by the engine before we turn it off.
1364          */
1365         if (stop->flush) {
1366                 radeon_do_cp_flush(dev_priv);
1367         }
1368
1369         /* If we fail to make the engine go idle, we return an error
1370          * code so that the DRM ioctl wrapper can try again.
1371          */
1372         if (stop->idle) {
1373                 ret = radeon_do_cp_idle(dev_priv);
1374                 if (ret)
1375                         return ret;
1376         }
1377
1378         /* Finally, we can turn off the CP.  If the engine isn't idle,
1379          * we will get some dropped triangles as they won't be fully
1380          * rendered before the CP is shut down.
1381          */
1382         radeon_do_cp_stop(dev_priv);
1383
1384         /* Reset the engine */
1385         radeon_do_engine_reset(dev);
1386
1387         return 0;
1388 }
1389
1390 void radeon_do_release(struct drm_device * dev)
1391 {
1392         drm_radeon_private_t *dev_priv = dev->dev_private;
1393         int i, ret;
1394
1395         if (dev_priv) {
1396                 if (dev_priv->cp_running) {
1397                         /* Stop the cp */
1398                         while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
1399                                 DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1400 #ifdef __linux__
1401                                 schedule();
1402 #else
1403 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
1404                                 mtx_sleep(&ret, &dev->dev_lock, PZERO, "rdnrel",
1405                                        1);
1406 #else
1407                                 tsleep(&ret, PZERO, "rdnrel", 1);
1408 #endif
1409 #endif
1410                         }
1411                         radeon_do_cp_stop(dev_priv);
1412                         radeon_do_engine_reset(dev);
1413                 }
1414
1415                 /* Disable *all* interrupts */
1416                 if (dev_priv->mmio)     /* remove this after permanent addmaps */
1417                         RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
1418
1419                 if (dev_priv->mmio) {   /* remove all surfaces */
1420                         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1421                                 RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
1422                                 RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
1423                                              16 * i, 0);
1424                                 RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
1425                                              16 * i, 0);
1426                         }
1427                 }
1428
1429                 /* Free memory heap structures */
1430                 radeon_mem_takedown(&(dev_priv->gart_heap));
1431                 radeon_mem_takedown(&(dev_priv->fb_heap));
1432
1433                 /* deallocate kernel resources */
1434                 radeon_do_cleanup_cp(dev);
1435         }
1436 }
1437
1438 /* Just reset the CP ring.  Called as part of an X Server engine reset.
1439  */
1440 int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1441 {
1442         drm_radeon_private_t *dev_priv = dev->dev_private;
1443         DRM_DEBUG("\n");
1444
1445         LOCK_TEST_WITH_RETURN(dev, file_priv);
1446
1447         if (!dev_priv) {
1448                 DRM_DEBUG("called before init done\n");
1449                 return -EINVAL;
1450         }
1451
1452         radeon_do_cp_reset(dev_priv);
1453
1454         /* The CP is no longer running after an engine reset */
1455         dev_priv->cp_running = 0;
1456
1457         return 0;
1458 }
1459
1460 int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
1461 {
1462         drm_radeon_private_t *dev_priv = dev->dev_private;
1463         DRM_DEBUG("\n");
1464
1465         LOCK_TEST_WITH_RETURN(dev, file_priv);
1466
1467         return radeon_do_cp_idle(dev_priv);
1468 }
1469
1470 /* Added by Charl P. Botha to call radeon_do_resume_cp().
1471  */
1472 int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
1473 {
1474
1475         return radeon_do_resume_cp(dev);
1476 }
1477
1478 int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1479 {
1480         DRM_DEBUG("\n");
1481
1482         LOCK_TEST_WITH_RETURN(dev, file_priv);
1483
1484         return radeon_do_engine_reset(dev);
1485 }
1486
1487 /* ================================================================
1488  * Fullscreen mode
1489  */
1490
1491 /* KW: Deprecated to say the least:
1492  */
1493 int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
1494 {
1495         return 0;
1496 }
1497
1498 /* ================================================================
1499  * Freelist management
1500  */
1501
1502 /* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
1503  *   bufs until freelist code is used.  Note this hides a problem with
1504  *   the scratch register * (used to keep track of last buffer
1505  *   completed) being written to before * the last buffer has actually
1506  *   completed rendering.
1507  *
1508  * KW:  It's also a good way to find free buffers quickly.
1509  *
1510  * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
1511  * sleep.  However, bugs in older versions of radeon_accel.c mean that
1512  * we essentially have to do this, else old clients will break.
1513  *
1514  * However, it does leave open a potential deadlock where all the
1515  * buffers are held by other clients, which can't release them because
1516  * they can't get the lock.
1517  */
1518
1519 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1520 {
1521         struct drm_device_dma *dma = dev->dma;
1522         drm_radeon_private_t *dev_priv = dev->dev_private;
1523         drm_radeon_buf_priv_t *buf_priv;
1524         struct drm_buf *buf;
1525         int i, t;
1526         int start;
1527
1528         if (++dev_priv->last_buf >= dma->buf_count)
1529                 dev_priv->last_buf = 0;
1530
1531         start = dev_priv->last_buf;
1532
1533         for (t = 0; t < dev_priv->usec_timeout; t++) {
1534                 u32 done_age = GET_SCRATCH(1);
1535                 DRM_DEBUG("done_age = %d\n", done_age);
1536                 for (i = start; i < dma->buf_count; i++) {
1537                         buf = dma->buflist[i];
1538                         buf_priv = buf->dev_private;
1539                         if (buf->file_priv == NULL || (buf->pending &&
1540                                                        buf_priv->age <=
1541                                                        done_age)) {
1542                                 dev_priv->stats.requested_bufs++;
1543                                 buf->pending = 0;
1544                                 return buf;
1545                         }
1546                         start = 0;
1547                 }
1548
1549                 if (t) {
1550                         DRM_UDELAY(1);
1551                         dev_priv->stats.freelist_loops++;
1552                 }
1553         }
1554
1555         DRM_DEBUG("returning NULL!\n");
1556         return NULL;
1557 }
1558
1559 #if 0
1560 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1561 {
1562         struct drm_device_dma *dma = dev->dma;
1563         drm_radeon_private_t *dev_priv = dev->dev_private;
1564         drm_radeon_buf_priv_t *buf_priv;
1565         struct drm_buf *buf;
1566         int i, t;
1567         int start;
1568         u32 done_age = DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1));
1569
1570         if (++dev_priv->last_buf >= dma->buf_count)
1571                 dev_priv->last_buf = 0;
1572
1573         start = dev_priv->last_buf;
1574         dev_priv->stats.freelist_loops++;
1575
1576         for (t = 0; t < 2; t++) {
1577                 for (i = start; i < dma->buf_count; i++) {
1578                         buf = dma->buflist[i];
1579                         buf_priv = buf->dev_private;
1580                         if (buf->file_priv == 0 || (buf->pending &&
1581                                                     buf_priv->age <=
1582                                                     done_age)) {
1583                                 dev_priv->stats.requested_bufs++;
1584                                 buf->pending = 0;
1585                                 return buf;
1586                         }
1587                 }
1588                 start = 0;
1589         }
1590
1591         return NULL;
1592 }
1593 #endif
1594
1595 void radeon_freelist_reset(struct drm_device * dev)
1596 {
1597         struct drm_device_dma *dma = dev->dma;
1598         drm_radeon_private_t *dev_priv = dev->dev_private;
1599         int i;
1600
1601         dev_priv->last_buf = 0;
1602         for (i = 0; i < dma->buf_count; i++) {
1603                 struct drm_buf *buf = dma->buflist[i];
1604                 drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1605                 buf_priv->age = 0;
1606         }
1607 }
1608
1609 /* ================================================================
1610  * CP command submission
1611  */
1612
1613 int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
1614 {
1615         drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
1616         int i;
1617         u32 last_head = GET_RING_HEAD(dev_priv);
1618
1619         for (i = 0; i < dev_priv->usec_timeout; i++) {
1620                 u32 head = GET_RING_HEAD(dev_priv);
1621
1622                 ring->space = (head - ring->tail) * sizeof(u32);
1623                 if (ring->space <= 0)
1624                         ring->space += ring->size;
1625                 if (ring->space > n)
1626                         return 0;
1627
1628                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
1629
1630                 if (head != last_head)
1631                         i = 0;
1632                 last_head = head;
1633
1634                 DRM_UDELAY(1);
1635         }
1636
1637         /* FIXME: This return value is ignored in the BEGIN_RING macro! */
1638 #if RADEON_FIFO_DEBUG
1639         radeon_status(dev_priv);
1640         DRM_ERROR("failed!\n");
1641 #endif
1642         return -EBUSY;
1643 }
1644
1645 static int radeon_cp_get_buffers(struct drm_device *dev,
1646                                  struct drm_file *file_priv,
1647                                  struct drm_dma * d)
1648 {
1649         int i;
1650         struct drm_buf *buf;
1651
1652         for (i = d->granted_count; i < d->request_count; i++) {
1653                 buf = radeon_freelist_get(dev);
1654                 if (!buf)
1655                         return -EBUSY;  /* NOTE: broken client */
1656
1657                 buf->file_priv = file_priv;
1658
1659                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
1660                                      sizeof(buf->idx)))
1661                         return -EFAULT;
1662                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
1663                                      sizeof(buf->total)))
1664                         return -EFAULT;
1665
1666                 d->granted_count++;
1667         }
1668         return 0;
1669 }
1670
1671 int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
1672 {
1673         struct drm_device_dma *dma = dev->dma;
1674         int ret = 0;
1675         struct drm_dma *d = data;
1676
1677         LOCK_TEST_WITH_RETURN(dev, file_priv);
1678
1679         /* Please don't send us buffers.
1680          */
1681         if (d->send_count != 0) {
1682                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
1683                           DRM_CURRENTPID, d->send_count);
1684                 return -EINVAL;
1685         }
1686
1687         /* We'll send you buffers.
1688          */
1689         if (d->request_count < 0 || d->request_count > dma->buf_count) {
1690                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
1691                           DRM_CURRENTPID, d->request_count, dma->buf_count);
1692                 return -EINVAL;
1693         }
1694
1695         d->granted_count = 0;
1696
1697         if (d->request_count) {
1698                 ret = radeon_cp_get_buffers(dev, file_priv, d);
1699         }
1700
1701         return ret;
1702 }
1703
1704 int radeon_driver_load(struct drm_device *dev, unsigned long flags)
1705 {
1706         drm_radeon_private_t *dev_priv;
1707         int ret = 0;
1708
1709         dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
1710         if (dev_priv == NULL)
1711                 return -ENOMEM;
1712
1713         memset(dev_priv, 0, sizeof(drm_radeon_private_t));
1714         dev->dev_private = (void *)dev_priv;
1715         dev_priv->flags = flags;
1716
1717         switch (flags & RADEON_FAMILY_MASK) {
1718         case CHIP_R100:
1719         case CHIP_RV200:
1720         case CHIP_R200:
1721         case CHIP_R300:
1722         case CHIP_R350:
1723         case CHIP_R420:
1724         case CHIP_RV410:
1725         case CHIP_RV515:
1726         case CHIP_R520:
1727         case CHIP_RV570:
1728         case CHIP_R580:
1729                 dev_priv->flags |= RADEON_HAS_HIERZ;
1730                 break;
1731         default:
1732                 /* all other chips have no hierarchical z buffer */
1733                 break;
1734         }
1735
1736         if (drm_device_is_agp(dev))
1737                 dev_priv->flags |= RADEON_IS_AGP;
1738         else if (drm_device_is_pcie(dev))
1739                 dev_priv->flags |= RADEON_IS_PCIE;
1740         else
1741                 dev_priv->flags |= RADEON_IS_PCI;
1742
1743         DRM_DEBUG("%s card detected\n",
1744                   ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
1745         return ret;
1746 }
1747
1748 /* Create mappings for registers and framebuffer so userland doesn't necessarily
1749  * have to find them.
1750  */
1751 int radeon_driver_firstopen(struct drm_device *dev)
1752 {
1753         int ret;
1754         drm_local_map_t *map;
1755         drm_radeon_private_t *dev_priv = dev->dev_private;
1756
1757         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
1758
1759         ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
1760                          drm_get_resource_len(dev, 2), _DRM_REGISTERS,
1761                          _DRM_READ_ONLY, &dev_priv->mmio);
1762         if (ret != 0)
1763                 return ret;
1764
1765         dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
1766         ret = drm_addmap(dev, dev_priv->fb_aper_offset,
1767                          drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
1768                          _DRM_WRITE_COMBINING, &map);
1769         if (ret != 0)
1770                 return ret;
1771
1772         return 0;
1773 }
1774
1775 int radeon_driver_unload(struct drm_device *dev)
1776 {
1777         drm_radeon_private_t *dev_priv = dev->dev_private;
1778
1779         DRM_DEBUG("\n");
1780         drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
1781
1782         dev->dev_private = NULL;
1783         return 0;
1784 }