shared-core/radeon_cp.c

   1 /* radeon_cp.c -- CP support for Radeon -*- linux-c -*- */
   2 /*
   3  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   4  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   5  * Copyright 2007 Advanced Micro Devices, Inc.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  23  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  24  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  *
  27  * Authors:
  28  *    Kevin E. Martin <martin@valinux.com>
  29  *    Gareth Hughes <gareth@valinux.com>
  30  */
  31
  32 #include "drmP.h"
  33 #include "drm.h"
  34 #include "drm_sarea.h"
  35 #include "radeon_drm.h"
  36 #include "radeon_drv.h"
  37 #include "r300_reg.h"
  38
  39 #include "radeon_microcode.h"
  40 #define RADEON_FIFO_DEBUG       0
  41
  42 static int radeon_do_cleanup_cp(struct drm_device * dev);
  43 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv);
  44
  45 static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  46 {
  47         u32 ret;
  48         RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
  49         ret = RADEON_READ(R520_MC_IND_DATA);
  50         RADEON_WRITE(R520_MC_IND_INDEX, 0);
  51         return ret;
  52 }
  53
  54 static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  55 {
  56         u32 ret;
  57         RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
  58         ret = RADEON_READ(RS480_NB_MC_DATA);
  59         RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
  60         return ret;
  61 }
  62
  63 static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  64 {
  65         u32 ret;
  66         RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
  67         ret = RADEON_READ(RS690_MC_DATA);
  68         RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
  69         return ret;
  70 }
  71
  72 static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  73 {
  74         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  75             return RS690_READ_MCIND(dev_priv, addr);
  76         else
  77             return RS480_READ_MCIND(dev_priv, addr);
  78 }
  79
  80 u32 radeon_read_mc_reg(drm_radeon_private_t *dev_priv, int addr)
  81 {
  82         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  83                 return IGP_READ_MCIND(dev_priv, addr);
  84         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515)
  85                 return R500_READ_MCIND(dev_priv, addr);
  86         return 0;
  87 }
  88
  89 void radeon_write_mc_reg(drm_radeon_private_t *dev_priv, u32 addr, u32 val)
  90 {
  91         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  92                 IGP_WRITE_MCIND(addr, val);
  93         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515)
  94                 R500_WRITE_MCIND(addr, val);
  95 }
  96
  97 u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
  98 {
  99
 100         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 101                 return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
 102         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 103                 return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
 104         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 105                 return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
 106         else
 107                 return RADEON_READ(RADEON_MC_FB_LOCATION);
 108 }
 109
 110 static void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
 111 {
 112         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 113                 R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
 114         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 115                 RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
 116         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 117                 R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
 118         else
 119                 RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
 120 }
 121
 122 static void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc)
 123 {
 124         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 125                 R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
 126         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 127                 RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
 128         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 129                 R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
 130         else
 131                 RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
 132 }
 133
 134 static void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
 135 {
 136         u32 agp_base_hi = upper_32_bits(agp_base);
 137         u32 agp_base_lo = agp_base & 0xffffffff;
 138
 139         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
 140                 R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
 141                 R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
 142         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
 143                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
 144                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
 145         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
 146                 R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
 147                 R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
 148         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 149                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 150                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 151                 RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
 152         } else {
 153                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 154                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
 155                         RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
 156         }
 157 }
 158
 159
 160 void radeon_pll_errata_after_index(struct drm_radeon_private *dev_priv)
 161 {
 162         if (!(dev_priv->pll_errata & CHIP_ERRATA_PLL_DUMMYREADS))
 163                 return;
 164
 165         (void)RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 166         (void)RADEON_READ(RADEON_CRTC_GEN_CNTL);
 167 }
 168
 169 void radeon_pll_errata_after_data(struct drm_radeon_private *dev_priv)
 170 {
 171         /* This workarounds is necessary on RV100, RS100 and RS200 chips
 172          * or the chip could hang on a subsequent access
 173          */
 174         if (dev_priv->pll_errata & CHIP_ERRATA_PLL_DELAY)
 175                 udelay(5000);
 176
 177         /* This function is required to workaround a hardware bug in some (all?)
 178          * revisions of the R300.  This workaround should be called after every
 179          * CLOCK_CNTL_INDEX register access.  If not, register reads afterward
 180          * may not be correct.
 181          */
 182         if (dev_priv->pll_errata & CHIP_ERRATA_R300_CG) {
 183                 uint32_t save, tmp;
 184
 185                 save = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 186                 tmp = save & ~(0x3f | RADEON_PLL_WR_EN);
 187                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, tmp);
 188                 tmp = RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 189                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, save);
 190         }
 191 }
 192
 193 int RADEON_READ_PLL(struct drm_radeon_private *dev_priv, int addr)
 194 {
 195         uint32_t data;
 196
 197         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x3f);
 198         radeon_pll_errata_after_index(dev_priv);
 199         data = RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 200         radeon_pll_errata_after_data(dev_priv);
 201         return data;
 202 }
 203
 204 void RADEON_WRITE_PLL(struct drm_radeon_private *dev_priv, int addr, uint32_t data)
 205 {
 206         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, ((addr & 0x3f) | RADEON_PLL_WR_EN));
 207         radeon_pll_errata_after_index(dev_priv);
 208         RADEON_WRITE(RADEON_CLOCK_CNTL_DATA, data);
 209         radeon_pll_errata_after_data(dev_priv);
 210 }
 211
 212 static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
 213 {
 214         RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
 215         return RADEON_READ(RADEON_PCIE_DATA);
 216 }
 217
 218 /* ATOM accessor methods */
 219 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 220 {
 221         uint32_t ret = radeon_read_mc_reg(info->dev->dev_private, reg);
 222
 223         //      DRM_DEBUG("(%x) = %x\n", reg, ret);
 224         return ret;
 225 }
 226
 227 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 228 {
 229   //    DRM_DEBUG("(%x,  %x)\n", reg, val);
 230         radeon_write_mc_reg(info->dev->dev_private, reg, val);
 231 }
 232
 233 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 234 {
 235         drm_radeon_private_t *dev_priv = info->dev->dev_private;
 236
 237         //      DRM_DEBUG("(%x,  %x)\n", reg*4, val);
 238         RADEON_WRITE(reg*4, val);
 239 }
 240
 241 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 242 {
 243         uint32_t ret;
 244         drm_radeon_private_t *dev_priv = info->dev->dev_private;
 245
 246         ret = RADEON_READ(reg*4);
 247         //      DRM_DEBUG("(%x) = %x\n", reg*4, ret);
 248         return ret;
 249 }
 250
 251 #if RADEON_FIFO_DEBUG
 252 static void radeon_status(drm_radeon_private_t * dev_priv)
 253 {
 254         printk("%s:\n", __FUNCTION__);
 255         printk("RBBM_STATUS = 0x%08x\n",
 256                (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
 257         printk("CP_RB_RTPR = 0x%08x\n",
 258                (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
 259         printk("CP_RB_WTPR = 0x%08x\n",
 260                (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
 261         printk("AIC_CNTL = 0x%08x\n",
 262                (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
 263         printk("AIC_STAT = 0x%08x\n",
 264                (unsigned int)RADEON_READ(RADEON_AIC_STAT));
 265         printk("AIC_PT_BASE = 0x%08x\n",
 266                (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
 267         printk("TLB_ADDR = 0x%08x\n",
 268                (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
 269         printk("TLB_DATA = 0x%08x\n",
 270                (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
 271 }
 272 #endif
 273
 274 /* ================================================================
 275  * Engine, FIFO control
 276  */
 277
 278 static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
 279 {
 280         u32 tmp;
 281         int i;
 282
 283         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 284
 285         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
 286                 tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
 287                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 288                 RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
 289
 290                 for (i = 0; i < dev_priv->usec_timeout; i++) {
 291                         if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
 292                               & RADEON_RB3D_DC_BUSY)) {
 293                                 return 0;
 294                         }
 295                         DRM_UDELAY(1);
 296                 }
 297         } else {
 298                 /* don't flush or purge cache here or lockup */
 299                 return 0;
 300         }
 301
 302 #if RADEON_FIFO_DEBUG
 303         DRM_ERROR("failed!\n");
 304         radeon_status(dev_priv);
 305 #endif
 306         return -EBUSY;
 307 }
 308
 309 static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
 310 {
 311         int i;
 312
 313         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 314
 315         for (i = 0; i < dev_priv->usec_timeout; i++) {
 316                 int slots = (RADEON_READ(RADEON_RBBM_STATUS)
 317                              & RADEON_RBBM_FIFOCNT_MASK);
 318                 if (slots >= entries)
 319                         return 0;
 320                 DRM_UDELAY(1);
 321         }
 322         DRM_INFO("wait for fifo failed status : 0x%08X 0x%08X\n",
 323                  RADEON_READ(RADEON_RBBM_STATUS),
 324                  RADEON_READ(R300_VAP_CNTL_STATUS));
 325
 326 #if RADEON_FIFO_DEBUG
 327         DRM_ERROR("failed!\n");
 328         radeon_status(dev_priv);
 329 #endif
 330         return -EBUSY;
 331 }
 332
 333 static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
 334 {
 335         int i, ret;
 336
 337         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 338
 339         ret = radeon_do_wait_for_fifo(dev_priv, 64);
 340         if (ret)
 341                 return ret;
 342
 343         for (i = 0; i < dev_priv->usec_timeout; i++) {
 344                 if (!(RADEON_READ(RADEON_RBBM_STATUS)
 345                       & RADEON_RBBM_ACTIVE)) {
 346                         radeon_do_pixcache_flush(dev_priv);
 347                         return 0;
 348                 }
 349                 DRM_UDELAY(1);
 350         }
 351         DRM_INFO("wait idle failed status : 0x%08X 0x%08X\n",
 352                  RADEON_READ(RADEON_RBBM_STATUS),
 353                  RADEON_READ(R300_VAP_CNTL_STATUS));
 354
 355 #if RADEON_FIFO_DEBUG
 356         DRM_ERROR("failed!\n");
 357         radeon_status(dev_priv);
 358 #endif
 359         return -EBUSY;
 360 }
 361
 362 static void radeon_init_pipes(drm_radeon_private_t * dev_priv)
 363 {
 364         uint32_t gb_tile_config, gb_pipe_sel = 0;
 365
 366         /* RS4xx/RS6xx/R4xx/R5xx */
 367         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
 368                 gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
 369                 dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
 370         } else {
 371                 /* R3xx */
 372                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 373                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350)) {
 374                         dev_priv->num_gb_pipes = 2;
 375                 } else {
 376                         /* R3Vxx */
 377                         dev_priv->num_gb_pipes = 1;
 378                 }
 379         }
 380         DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
 381
 382         gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
 383
 384         switch(dev_priv->num_gb_pipes) {
 385         case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
 386         case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
 387         case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
 388         default:
 389         case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
 390         }
 391
 392         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 393                 RADEON_WRITE_PLL(dev_priv, R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
 394                 RADEON_WRITE(R500_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
 395         }
 396         RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
 397         radeon_do_wait_for_idle(dev_priv);
 398         RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
 399         RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
 400                                                R300_DC_AUTOFLUSH_ENABLE |
 401                                                R300_DC_DC_DISABLE_IGNORE_PE));
 402
 403
 404 }
 405
 406 /* ================================================================
 407  * CP control, initialization
 408  */
 409
 410 /* Load the microcode for the CP */
 411 static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 412 {
 413         int i;
 414         DRM_DEBUG("\n");
 415
 416         radeon_do_wait_for_idle(dev_priv);
 417
 418         RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 419
 420         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
 421             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
 422             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
 423             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
 424             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
 425                 DRM_INFO("Loading R100 Microcode\n");
 426                 for (i = 0; i < 256; i++) {
 427                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 428                                      R100_cp_microcode[i][1]);
 429                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 430                                      R100_cp_microcode[i][0]);
 431                 }
 432         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
 433                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
 434                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
 435                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
 436                 DRM_INFO("Loading R200 Microcode\n");
 437                 for (i = 0; i < 256; i++) {
 438                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 439                                      R200_cp_microcode[i][1]);
 440                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 441                                      R200_cp_microcode[i][0]);
 442                 }
 443         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 444                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
 445                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
 446                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
 447                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 448                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 449                 DRM_INFO("Loading R300 Microcode\n");
 450                 for (i = 0; i < 256; i++) {
 451                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 452                                      R300_cp_microcode[i][1]);
 453                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 454                                      R300_cp_microcode[i][0]);
 455                 }
 456         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 457                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
 458                 DRM_INFO("Loading R400 Microcode\n");
 459                 for (i = 0; i < 256; i++) {
 460                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 461                                      R420_cp_microcode[i][1]);
 462                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 463                                      R420_cp_microcode[i][0]);
 464                 }
 465         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
 466                 DRM_INFO("Loading RS690 Microcode\n");
 467                 for (i = 0; i < 256; i++) {
 468                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 469                                      RS690_cp_microcode[i][1]);
 470                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 471                                      RS690_cp_microcode[i][0]);
 472                 }
 473         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
 474                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
 475                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
 476                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
 477                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
 478                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
 479                 DRM_INFO("Loading R500 Microcode\n");
 480                 for (i = 0; i < 256; i++) {
 481                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 482                                      R520_cp_microcode[i][1]);
 483                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 484                                      R520_cp_microcode[i][0]);
 485                 }
 486         }
 487 }
 488
 489 /* Flush any pending commands to the CP.  This should only be used just
 490  * prior to a wait for idle, as it informs the engine that the command
 491  * stream is ending.
 492  */
 493 static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
 494 {
 495         DRM_DEBUG("\n");
 496 #if 0
 497         u32 tmp;
 498
 499         tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
 500         RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
 501 #endif
 502 }
 503
 504 /* Wait for the CP to go idle.
 505  */
 506 int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
 507 {
 508         RING_LOCALS;
 509         DRM_DEBUG("\n");
 510
 511         BEGIN_RING(6);
 512
 513         RADEON_PURGE_CACHE();
 514         RADEON_PURGE_ZCACHE();
 515         RADEON_WAIT_UNTIL_IDLE();
 516
 517         ADVANCE_RING();
 518         COMMIT_RING();
 519
 520         return radeon_do_wait_for_idle(dev_priv);
 521 }
 522
 523 /* Start the Command Processor.
 524  */
 525 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
 526 {
 527         RING_LOCALS;
 528         DRM_DEBUG("\n");
 529
 530         radeon_do_wait_for_idle(dev_priv);
 531
 532         RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
 533
 534         dev_priv->cp_running = 1;
 535
 536         BEGIN_RING(8);
 537         /* isync can only be written through cp on r5xx write it here */
 538         OUT_RING(CP_PACKET0(RADEON_ISYNC_CNTL, 0));
 539         OUT_RING(RADEON_ISYNC_ANY2D_IDLE3D |
 540                  RADEON_ISYNC_ANY3D_IDLE2D |
 541                  RADEON_ISYNC_WAIT_IDLEGUI |
 542                  RADEON_ISYNC_CPSCRATCH_IDLEGUI);
 543         RADEON_PURGE_CACHE();
 544         RADEON_PURGE_ZCACHE();
 545         RADEON_WAIT_UNTIL_IDLE();
 546         ADVANCE_RING();
 547         COMMIT_RING();
 548
 549         dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 550 }
 551
 552 /* Reset the Command Processor.  This will not flush any pending
 553  * commands, so you must wait for the CP command stream to complete
 554  * before calling this routine.
 555  */
 556 static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
 557 {
 558         u32 cur_read_ptr;
 559         DRM_DEBUG("\n");
 560
 561         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 562         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 563         SET_RING_HEAD(dev_priv, cur_read_ptr);
 564         dev_priv->ring.tail = cur_read_ptr;
 565 }
 566
 567 /* Stop the Command Processor.  This will not flush any pending
 568  * commands, so you must flush the command stream and wait for the CP
 569  * to go idle before calling this routine.
 570  */
 571 static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
 572 {
 573         DRM_DEBUG("\n");
 574
 575         RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
 576
 577         dev_priv->cp_running = 0;
 578 }
 579
 580 /* Reset the engine.  This will stop the CP if it is running.
 581  */
 582 static int radeon_do_engine_reset(struct drm_device * dev)
 583 {
 584         drm_radeon_private_t *dev_priv = dev->dev_private;
 585         u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
 586         DRM_DEBUG("\n");
 587
 588         radeon_do_pixcache_flush(dev_priv);
 589
 590         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 591                 /* may need something similar for newer chips */
 592                 clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 593                 mclk_cntl = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
 594
 595                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, (mclk_cntl |
 596                                                               RADEON_FORCEON_MCLKA |
 597                                                               RADEON_FORCEON_MCLKB |
 598                                                               RADEON_FORCEON_YCLKA |
 599                                                               RADEON_FORCEON_YCLKB |
 600                                                               RADEON_FORCEON_MC |
 601                                                               RADEON_FORCEON_AIC));
 602         }
 603
 604         rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
 605
 606         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
 607                                               RADEON_SOFT_RESET_CP |
 608                                               RADEON_SOFT_RESET_HI |
 609                                               RADEON_SOFT_RESET_SE |
 610                                               RADEON_SOFT_RESET_RE |
 611                                               RADEON_SOFT_RESET_PP |
 612                                               RADEON_SOFT_RESET_E2 |
 613                                               RADEON_SOFT_RESET_RB));
 614         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 615         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
 616                                               ~(RADEON_SOFT_RESET_CP |
 617                                                 RADEON_SOFT_RESET_HI |
 618                                                 RADEON_SOFT_RESET_SE |
 619                                                 RADEON_SOFT_RESET_RE |
 620                                                 RADEON_SOFT_RESET_PP |
 621                                                 RADEON_SOFT_RESET_E2 |
 622                                                 RADEON_SOFT_RESET_RB)));
 623         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 624
 625         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 626                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, mclk_cntl);
 627                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
 628                 RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
 629         }
 630
 631         /* setup the raster pipes */
 632         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
 633             radeon_init_pipes(dev_priv);
 634
 635         /* Reset the CP ring */
 636         radeon_do_cp_reset(dev_priv);
 637
 638         /* The CP is no longer running after an engine reset */
 639         dev_priv->cp_running = 0;
 640
 641         /* Reset any pending vertex, indirect buffers */
 642         if (dev->dma)
 643                 radeon_freelist_reset(dev);
 644
 645         return 0;
 646 }
 647
 648 static void radeon_cp_init_ring_buffer(struct drm_device * dev,
 649                                        drm_radeon_private_t * dev_priv)
 650 {
 651         u32 ring_start, cur_read_ptr;
 652         u32 tmp;
 653
 654         /* Initialize the memory controller. With new memory map, the fb location
 655          * is not changed, it should have been properly initialized already. Part
 656          * of the problem is that the code below is bogus, assuming the GART is
 657          * always appended to the fb which is not necessarily the case
 658          */
 659         if (!dev_priv->new_memmap)
 660                 radeon_write_fb_location(dev_priv,
 661                                          ((dev_priv->gart_vm_start - 1) & 0xffff0000)
 662                                          | (dev_priv->fb_location >> 16));
 663
 664         if (dev_priv->mm.ring) {
 665                 ring_start = dev_priv->mm.ring->offset +
 666                         dev_priv->gart_vm_start;
 667         } else
 668 #if __OS_HAS_AGP
 669         if (dev_priv->flags & RADEON_IS_AGP) {
 670                 radeon_write_agp_base(dev_priv, dev->agp->base);
 671
 672                 radeon_write_agp_location(dev_priv,
 673                              (((dev_priv->gart_vm_start - 1 +
 674                                 dev_priv->gart_size) & 0xffff0000) |
 675                               (dev_priv->gart_vm_start >> 16)));
 676
 677                 ring_start = (dev_priv->cp_ring->offset
 678                               - dev->agp->base
 679                               + dev_priv->gart_vm_start);
 680         } else
 681 #endif
 682                 ring_start = (dev_priv->cp_ring->offset
 683                               - (unsigned long)dev->sg->virtual
 684                               + dev_priv->gart_vm_start);
 685
 686         RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
 687
 688         /* Set the write pointer delay */
 689         RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
 690
 691         /* Initialize the ring buffer's read and write pointers */
 692         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 693         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 694         SET_RING_HEAD(dev_priv, cur_read_ptr);
 695         dev_priv->ring.tail = cur_read_ptr;
 696
 697
 698         if (dev_priv->mm.ring_read_ptr) {
 699                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 700                              dev_priv->mm.ring_read_ptr->offset +
 701                              dev_priv->gart_vm_start);
 702         } else
 703 #if __OS_HAS_AGP
 704         if (dev_priv->flags & RADEON_IS_AGP) {
 705                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 706                              dev_priv->ring_rptr->offset
 707                              - dev->agp->base + dev_priv->gart_vm_start);
 708         } else
 709 #endif
 710         {
 711                 struct drm_sg_mem *entry = dev->sg;
 712                 unsigned long tmp_ofs, page_ofs;
 713
 714                 tmp_ofs = dev_priv->ring_rptr->offset -
 715                                 (unsigned long)dev->sg->virtual;
 716                 page_ofs = tmp_ofs >> PAGE_SHIFT;
 717
 718                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR, entry->busaddr[page_ofs]);
 719                 DRM_DEBUG("ring rptr: offset=0x%08lx handle=0x%08lx\n",
 720                           (unsigned long)entry->busaddr[page_ofs],
 721                           entry->handle + tmp_ofs);
 722         }
 723
 724         /* Set ring buffer size */
 725 #ifdef __BIG_ENDIAN
 726         RADEON_WRITE(RADEON_CP_RB_CNTL,
 727                      RADEON_BUF_SWAP_32BIT |
 728                      (dev_priv->ring.fetch_size_l2ow << 18) |
 729                      (dev_priv->ring.rptr_update_l2qw << 8) |
 730                      dev_priv->ring.size_l2qw);
 731 #else
 732         RADEON_WRITE(RADEON_CP_RB_CNTL,
 733                      (dev_priv->ring.fetch_size_l2ow << 18) |
 734                      (dev_priv->ring.rptr_update_l2qw << 8) |
 735                      dev_priv->ring.size_l2qw);
 736 #endif
 737
 738         /* Start with assuming that writeback doesn't work */
 739         dev_priv->writeback_works = 0;
 740
 741         /* Initialize the scratch register pointer.  This will cause
 742          * the scratch register values to be written out to memory
 743          * whenever they are updated.
 744          *
 745          * We simply put this behind the ring read pointer, this works
 746          * with PCI GART as well as (whatever kind of) AGP GART
 747          */
 748         RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
 749                      + RADEON_SCRATCH_REG_OFFSET);
 750
 751         if (dev_priv->mm.ring_read_ptr)
 752                 dev_priv->scratch = ((__volatile__ u32 *)
 753                                      dev_priv->mm.ring_read_ptr_map.virtual +
 754                                      (RADEON_SCRATCH_REG_OFFSET / sizeof(u32)));
 755         else
 756                 dev_priv->scratch = ((__volatile__ u32 *)
 757                                      dev_priv->ring_rptr->handle +
 758                                      (RADEON_SCRATCH_REG_OFFSET / sizeof(u32)));
 759
 760         RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
 761
 762         /* Turn on bus mastering */
 763         tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
 764         RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 765
 766         dev_priv->scratch[0] = 0;
 767         RADEON_WRITE(RADEON_LAST_FRAME_REG, 0);
 768
 769         dev_priv->scratch[1] = 0;
 770         RADEON_WRITE(RADEON_LAST_DISPATCH_REG, 0);
 771
 772         dev_priv->scratch[2] = 0;
 773         RADEON_WRITE(RADEON_LAST_CLEAR_REG, 0);
 774
 775         radeon_do_wait_for_idle(dev_priv);
 776
 777         /* Sync everything up */
 778         RADEON_WRITE(RADEON_ISYNC_CNTL,
 779                      (RADEON_ISYNC_ANY2D_IDLE3D |
 780                       RADEON_ISYNC_ANY3D_IDLE2D |
 781                       RADEON_ISYNC_WAIT_IDLEGUI |
 782                       RADEON_ISYNC_CPSCRATCH_IDLEGUI));
 783
 784 }
 785
 786 static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
 787 {
 788         u32 tmp;
 789         void *ring_read_ptr;
 790
 791         if (dev_priv->mm.ring_read_ptr)
 792                 ring_read_ptr = dev_priv->mm.ring_read_ptr_map.virtual;
 793         else
 794                 ring_read_ptr = dev_priv->ring_rptr->handle;
 795
 796         /* Writeback doesn't seem to work everywhere, test it here and possibly
 797          * enable it if it appears to work
 798          */
 799         writel(0, ring_read_ptr + RADEON_SCRATCHOFF(1));
 800         RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
 801
 802         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 803                 if (readl(ring_read_ptr + RADEON_SCRATCHOFF(1)) ==
 804                     0xdeadbeef)
 805                         break;
 806                 DRM_UDELAY(1);
 807         }
 808
 809         if (tmp < dev_priv->usec_timeout) {
 810                 dev_priv->writeback_works = 1;
 811                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 812         } else {
 813                 dev_priv->writeback_works = 0;
 814                 DRM_INFO("writeback test failed\n");
 815         }
 816         if (radeon_no_wb == 1) {
 817                 dev_priv->writeback_works = 0;
 818                 DRM_INFO("writeback forced off\n");
 819         }
 820
 821         if (!dev_priv->writeback_works) {
 822                 /* Disable writeback to avoid unnecessary bus master transfers */
 823                 RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) | RADEON_RB_NO_UPDATE);
 824                 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
 825         }
 826 }
 827
 828 /* Enable or disable IGP GART on the chip */
 829 static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
 830 {
 831         u32 temp;
 832
 833         if (on) {
 834                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 835                          dev_priv->gart_vm_start,
 836                          (long)dev_priv->gart_info.bus_addr,
 837                          dev_priv->gart_size);
 838
 839                 temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
 840
 841                 if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 842                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
 843                                                              RS690_BLOCK_GFX_D3_EN));
 844                 else
 845                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
 846
 847                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 848                                                                RS480_VA_SIZE_32MB));
 849
 850                 temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
 851                 IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
 852                                                         RS480_TLB_ENABLE |
 853                                                         RS480_GTW_LAC_EN |
 854                                                         RS480_1LEVEL_GART));
 855
 856                 temp = dev_priv->gart_info.bus_addr & 0xfffff000;
 857                 temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
 858                 IGP_WRITE_MCIND(RS480_GART_BASE, temp);
 859
 860                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
 861                 IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
 862                                                       RS480_REQ_TYPE_SNOOP_DIS));
 863
 864                 radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
 865
 866                 dev_priv->gart_size = 32*1024*1024;
 867                 temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) &
 868                         0xffff0000) | (dev_priv->gart_vm_start >> 16));
 869
 870                 radeon_write_agp_location(dev_priv, temp);
 871
 872                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
 873                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 874                                                                RS480_VA_SIZE_32MB));
 875
 876                 do {
 877                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 878                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 879                                 break;
 880                         DRM_UDELAY(1);
 881                 } while(1);
 882
 883                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
 884                                 RS480_GART_CACHE_INVALIDATE);
 885
 886                 do {
 887                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 888                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 889                                 break;
 890                         DRM_UDELAY(1);
 891                 } while(1);
 892
 893                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
 894         } else {
 895                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
 896         }
 897 }
 898
 899 static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 900 {
 901         u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
 902         if (on) {
 903
 904                 DRM_DEBUG("programming pcie %08X %08lX %08X\n",
 905                           dev_priv->gart_vm_start,
 906                           (long)dev_priv->gart_info.bus_addr,
 907                           dev_priv->gart_size);
 908                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
 909                                   dev_priv->gart_vm_start);
 910                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
 911                                   dev_priv->gart_info.bus_addr);
 912                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
 913                                   dev_priv->gart_vm_start);
 914                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
 915                                   dev_priv->gart_vm_start +
 916                                   dev_priv->gart_size - 1);
 917
 918                 radeon_write_agp_location(dev_priv, 0xffffffc0); /* ?? */
 919
 920                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 921                                   RADEON_PCIE_TX_GART_EN);
 922         } else {
 923                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 924                                   tmp & ~RADEON_PCIE_TX_GART_EN);
 925         }
 926 }
 927
 928 /* Enable or disable PCI GART on the chip */
 929 void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 930 {
 931         u32 tmp;
 932
 933         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 934             (dev_priv->flags & RADEON_IS_IGPGART)) {
 935                 radeon_set_igpgart(dev_priv, on);
 936                 return;
 937         }
 938
 939         if (dev_priv->flags & RADEON_IS_PCIE) {
 940                 radeon_set_pciegart(dev_priv, on);
 941                 return;
 942         }
 943
 944         tmp = RADEON_READ(RADEON_AIC_CNTL);
 945
 946         if (on) {
 947                 RADEON_WRITE(RADEON_AIC_CNTL,
 948                              tmp | RADEON_PCIGART_TRANSLATE_EN);
 949
 950                 /* set PCI GART page-table base address
 951                  */
 952                 RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
 953
 954                 /* set address range for PCI address translate
 955                  */
 956                 RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
 957                 RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
 958                              + dev_priv->gart_size - 1);
 959
 960                 /* Turn off AGP aperture -- is this required for PCI GART?
 961                  */
 962                 radeon_write_agp_location(dev_priv, 0xffffffc0);
 963                 RADEON_WRITE(RADEON_AGP_COMMAND, 0);    /* clear AGP_COMMAND */
 964         } else {
 965                 RADEON_WRITE(RADEON_AIC_CNTL,
 966                              tmp & ~RADEON_PCIGART_TRANSLATE_EN);
 967         }
 968 }
 969
 970 static int radeon_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
 971                              struct drm_file *file_priv)
 972 {
 973         drm_radeon_private_t *dev_priv = dev->dev_private;
 974         struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
 975
 976         DRM_DEBUG("\n");
 977
 978         /* if we require new memory map but we don't have it fail */
 979         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
 980                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
 981                 radeon_do_cleanup_cp(dev);
 982                 return -EINVAL;
 983         }
 984
 985         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP))
 986         {
 987                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
 988                 dev_priv->flags &= ~RADEON_IS_AGP;
 989         }
 990         else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
 991                  && !init->is_pci)
 992         {
 993                 DRM_DEBUG("Restoring AGP flag\n");
 994                 dev_priv->flags |= RADEON_IS_AGP;
 995         }
 996
 997         if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
 998                 DRM_ERROR("PCI GART memory not allocated!\n");
 999                 radeon_do_cleanup_cp(dev);
1000                 return -EINVAL;
1001         }
1002
1003         dev_priv->usec_timeout = init->usec_timeout;
1004         if (dev_priv->usec_timeout < 1 ||
1005             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1006                 DRM_DEBUG("TIMEOUT problem!\n");
1007                 radeon_do_cleanup_cp(dev);
1008                 return -EINVAL;
1009         }
1010
1011         /* Enable vblank on CRTC1 for older X servers
1012          */
1013         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
1014
1015         dev_priv->do_boxes = 0;
1016         dev_priv->cp_mode = init->cp_mode;
1017
1018         /* We don't support anything other than bus-mastering ring mode,
1019          * but the ring can be in either AGP or PCI space for the ring
1020          * read pointer.
1021          */
1022         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
1023             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
1024                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
1025                 radeon_do_cleanup_cp(dev);
1026                 return -EINVAL;
1027         }
1028
1029         switch (init->fb_bpp) {
1030         case 16:
1031                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
1032                 break;
1033         case 32:
1034         default:
1035                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1036                 break;
1037         }
1038         dev_priv->front_offset = init->front_offset;
1039         dev_priv->front_pitch = init->front_pitch;
1040         dev_priv->back_offset = init->back_offset;
1041         dev_priv->back_pitch = init->back_pitch;
1042
1043         switch (init->depth_bpp) {
1044         case 16:
1045                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
1046                 break;
1047         case 32:
1048         default:
1049                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
1050                 break;
1051         }
1052         dev_priv->depth_offset = init->depth_offset;
1053         dev_priv->depth_pitch = init->depth_pitch;
1054
1055         /* Hardware state for depth clears.  Remove this if/when we no
1056          * longer clear the depth buffer with a 3D rectangle.  Hard-code
1057          * all values to prevent unwanted 3D state from slipping through
1058          * and screwing with the clear operation.
1059          */
1060         dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
1061                                            (dev_priv->color_fmt << 10) |
1062                                            (dev_priv->chip_family < CHIP_R200 ? RADEON_ZBLOCK16 : 0));
1063
1064         dev_priv->depth_clear.rb3d_zstencilcntl =
1065             (dev_priv->depth_fmt |
1066              RADEON_Z_TEST_ALWAYS |
1067              RADEON_STENCIL_TEST_ALWAYS |
1068              RADEON_STENCIL_S_FAIL_REPLACE |
1069              RADEON_STENCIL_ZPASS_REPLACE |
1070              RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
1071
1072         dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
1073                                          RADEON_BFACE_SOLID |
1074                                          RADEON_FFACE_SOLID |
1075                                          RADEON_FLAT_SHADE_VTX_LAST |
1076                                          RADEON_DIFFUSE_SHADE_FLAT |
1077                                          RADEON_ALPHA_SHADE_FLAT |
1078                                          RADEON_SPECULAR_SHADE_FLAT |
1079                                          RADEON_FOG_SHADE_FLAT |
1080                                          RADEON_VTX_PIX_CENTER_OGL |
1081                                          RADEON_ROUND_MODE_TRUNC |
1082                                          RADEON_ROUND_PREC_8TH_PIX);
1083
1084
1085         dev_priv->ring_offset = init->ring_offset;
1086         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
1087         dev_priv->buffers_offset = init->buffers_offset;
1088         dev_priv->gart_textures_offset = init->gart_textures_offset;
1089
1090         master_priv->sarea = drm_getsarea(dev);
1091         if (!master_priv->sarea) {
1092                 DRM_ERROR("could not find sarea!\n");
1093                 radeon_do_cleanup_cp(dev);
1094                 return -EINVAL;
1095         }
1096
1097         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
1098         if (!dev_priv->cp_ring) {
1099                 DRM_ERROR("could not find cp ring region!\n");
1100                 radeon_do_cleanup_cp(dev);
1101                 return -EINVAL;
1102         }
1103         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
1104         if (!dev_priv->ring_rptr) {
1105                 DRM_ERROR("could not find ring read pointer!\n");
1106                 radeon_do_cleanup_cp(dev);
1107                 return -EINVAL;
1108         }
1109         dev->agp_buffer_token = init->buffers_offset;
1110         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1111         if (!dev->agp_buffer_map) {
1112                 DRM_ERROR("could not find dma buffer region!\n");
1113                 radeon_do_cleanup_cp(dev);
1114                 return -EINVAL;
1115         }
1116
1117         if (init->gart_textures_offset) {
1118                 dev_priv->gart_textures =
1119                     drm_core_findmap(dev, init->gart_textures_offset);
1120                 if (!dev_priv->gart_textures) {
1121                         DRM_ERROR("could not find GART texture region!\n");
1122                         radeon_do_cleanup_cp(dev);
1123                         return -EINVAL;
1124                 }
1125         }
1126
1127 #if __OS_HAS_AGP
1128         if (dev_priv->flags & RADEON_IS_AGP) {
1129                 drm_core_ioremap(dev_priv->cp_ring, dev);
1130                 drm_core_ioremap(dev_priv->ring_rptr, dev);
1131                 drm_core_ioremap(dev->agp_buffer_map, dev);
1132                 if (!dev_priv->cp_ring->handle ||
1133                     !dev_priv->ring_rptr->handle ||
1134                     !dev->agp_buffer_map->handle) {
1135                         DRM_ERROR("could not find ioremap agp regions!\n");
1136                         radeon_do_cleanup_cp(dev);
1137                         return -EINVAL;
1138                 }
1139         } else
1140 #endif
1141         {
1142                 dev_priv->cp_ring->handle = (void *)dev_priv->cp_ring->offset;
1143                 dev_priv->ring_rptr->handle =
1144                     (void *)dev_priv->ring_rptr->offset;
1145                 dev->agp_buffer_map->handle =
1146                     (void *)dev->agp_buffer_map->offset;
1147
1148                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
1149                           dev_priv->cp_ring->handle);
1150                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
1151                           dev_priv->ring_rptr->handle);
1152                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
1153                           dev->agp_buffer_map->handle);
1154         }
1155
1156         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
1157         dev_priv->fb_size =
1158                 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
1159                 - dev_priv->fb_location;
1160
1161         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1162                                         ((dev_priv->front_offset
1163                                           + dev_priv->fb_location) >> 10));
1164
1165         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1166                                        ((dev_priv->back_offset
1167                                          + dev_priv->fb_location) >> 10));
1168
1169         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1170                                         ((dev_priv->depth_offset
1171                                           + dev_priv->fb_location) >> 10));
1172
1173         dev_priv->gart_size = init->gart_size;
1174
1175         /* New let's set the memory map ... */
1176         if (dev_priv->new_memmap) {
1177                 u32 base = 0;
1178
1179                 DRM_INFO("Setting GART location based on new memory map\n");
1180
1181                 /* If using AGP, try to locate the AGP aperture at the same
1182                  * location in the card and on the bus, though we have to
1183                  * align it down.
1184                  */
1185 #if __OS_HAS_AGP
1186                 if (dev_priv->flags & RADEON_IS_AGP) {
1187                         base = dev->agp->base;
1188                         /* Check if valid */
1189                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
1190                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
1191                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
1192                                          dev->agp->base);
1193                                 base = 0;
1194                         }
1195                 }
1196 #endif
1197                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
1198                 if (base == 0) {
1199                         base = dev_priv->fb_location + dev_priv->fb_size;
1200                         if (base < dev_priv->fb_location ||
1201                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
1202                                 base = dev_priv->fb_location
1203                                         - dev_priv->gart_size;
1204                 }
1205                 dev_priv->gart_vm_start = base & 0xffc00000u;
1206                 if (dev_priv->gart_vm_start != base)
1207                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
1208                                  base, dev_priv->gart_vm_start);
1209         } else {
1210                 DRM_INFO("Setting GART location based on old memory map\n");
1211                 dev_priv->gart_vm_start = dev_priv->fb_location +
1212                         RADEON_READ(RADEON_CONFIG_APER_SIZE);
1213         }
1214
1215 #if __OS_HAS_AGP
1216         if (dev_priv->flags & RADEON_IS_AGP)
1217                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1218                                                  - dev->agp->base
1219                                                  + dev_priv->gart_vm_start);
1220         else
1221 #endif
1222                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1223                                         - (unsigned long)dev->sg->virtual
1224                                         + dev_priv->gart_vm_start);
1225
1226         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
1227         DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
1228         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
1229                   dev_priv->gart_buffers_offset);
1230
1231         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
1232         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
1233                               + init->ring_size / sizeof(u32));
1234         dev_priv->ring.size = init->ring_size;
1235         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
1236
1237         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
1238         dev_priv->ring.rptr_update_l2qw = drm_order( /* init->rptr_update */ 4096 / 8);
1239
1240         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
1241         dev_priv->ring.fetch_size_l2ow = drm_order( /* init->fetch_size */ 32 / 16);
1242
1243         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1244
1245         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
1246
1247 #if __OS_HAS_AGP
1248         if (dev_priv->flags & RADEON_IS_AGP) {
1249                 /* Turn off PCI GART */
1250                 radeon_set_pcigart(dev_priv, 0);
1251         } else
1252 #endif
1253         {
1254                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
1255                 /* if we have an offset set from userspace */
1256                 if (dev_priv->pcigart_offset_set) {
1257                         /* if it came from userspace - remap it */
1258                         if (dev_priv->pcigart_offset_set == 1) {
1259                                 dev_priv->gart_info.bus_addr =
1260                                         dev_priv->pcigart_offset + dev_priv->fb_location;
1261                                 dev_priv->gart_info.mapping.offset =
1262                                         dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
1263                                 dev_priv->gart_info.mapping.size =
1264                                         dev_priv->gart_info.table_size;
1265
1266                                 /* this is done by the mm now */
1267                                 drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
1268                                 dev_priv->gart_info.addr =
1269                                         dev_priv->gart_info.mapping.handle;
1270
1271                                 memset(dev_priv->gart_info.addr, 0, dev_priv->gart_info.table_size);
1272                                 if (dev_priv->flags & RADEON_IS_PCIE)
1273                                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
1274                                 else
1275                                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1276                                 dev_priv->gart_info.gart_table_location =
1277                                         DRM_ATI_GART_FB;
1278
1279                                 DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
1280                                           dev_priv->gart_info.addr,
1281                                           dev_priv->pcigart_offset);
1282                         }
1283                 } else {
1284
1285                         if (dev_priv->flags & RADEON_IS_PCIE) {
1286                                 DRM_ERROR
1287                                     ("Cannot use PCI Express without GART in FB memory\n");
1288                                 radeon_do_cleanup_cp(dev);
1289                                 return -EINVAL;
1290                         }
1291                         if (dev_priv->flags & RADEON_IS_IGPGART)
1292                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
1293                         else
1294                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1295                         dev_priv->gart_info.gart_table_location =
1296                             DRM_ATI_GART_MAIN;
1297                         dev_priv->gart_info.addr = NULL;
1298                         dev_priv->gart_info.bus_addr = 0;
1299
1300                 }
1301
1302                 if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
1303                         DRM_ERROR("failed to init PCI GART!\n");
1304                         radeon_do_cleanup_cp(dev);
1305                         return -ENOMEM;
1306                 }
1307
1308                 /* Turn on PCI GART */
1309                 radeon_set_pcigart(dev_priv, 1);
1310         }
1311
1312         radeon_cp_load_microcode(dev_priv);
1313         radeon_cp_init_ring_buffer(dev, dev_priv);
1314
1315         dev_priv->last_buf = 0;
1316
1317         radeon_do_engine_reset(dev);
1318         radeon_test_writeback(dev_priv);
1319
1320         return 0;
1321 }
1322
1323 static int radeon_do_cleanup_cp(struct drm_device * dev)
1324 {
1325         drm_radeon_private_t *dev_priv = dev->dev_private;
1326         DRM_DEBUG("\n");
1327
1328         /* Make sure interrupts are disabled here because the uninstall ioctl
1329          * may not have been called from userspace and after dev_private
1330          * is freed, it's too late.
1331          */
1332         if (dev->irq_enabled)
1333                 drm_irq_uninstall(dev);
1334
1335 #if __OS_HAS_AGP
1336         if (dev_priv->flags & RADEON_IS_AGP) {
1337                 if (dev_priv->cp_ring != NULL) {
1338                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1339                         dev_priv->cp_ring = NULL;
1340                 }
1341                 if (dev_priv->ring_rptr != NULL) {
1342                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1343                         dev_priv->ring_rptr = NULL;
1344                 }
1345                 if (dev->agp_buffer_map != NULL) {
1346                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1347                         dev->agp_buffer_map = NULL;
1348                 }
1349         } else
1350 #endif
1351         {
1352
1353                 if (dev_priv->gart_info.bus_addr) {
1354                         /* Turn off PCI GART */
1355                         radeon_set_pcigart(dev_priv, 0);
1356                         if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
1357                                 DRM_ERROR("failed to cleanup PCI GART!\n");
1358                 }
1359
1360                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
1361                 {
1362                         if (dev_priv->pcigart_offset_set == 1) {
1363                                 drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1364                                 dev_priv->gart_info.addr = NULL;
1365                         }
1366                 }
1367         }
1368         /* only clear to the start of flags */
1369         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1370
1371         return 0;
1372 }
1373
1374 /* This code will reinit the Radeon CP hardware after a resume from disc.
1375  * AFAIK, it would be very difficult to pickle the state at suspend time, so
1376  * here we make sure that all Radeon hardware initialisation is re-done without
1377  * affecting running applications.
1378  *
1379  * Charl P. Botha <http://cpbotha.net>
1380  */
1381 static int radeon_do_resume_cp(struct drm_device * dev)
1382 {
1383         drm_radeon_private_t *dev_priv = dev->dev_private;
1384
1385         if (!dev_priv) {
1386                 DRM_ERROR("Called with no initialization\n");
1387                 return -EINVAL;
1388         }
1389
1390         DRM_DEBUG("Starting radeon_do_resume_cp()\n");
1391
1392 #if __OS_HAS_AGP
1393         if (dev_priv->flags & RADEON_IS_AGP) {
1394                 /* Turn off PCI GART */
1395                 radeon_set_pcigart(dev_priv, 0);
1396         } else
1397 #endif
1398         {
1399                 /* Turn on PCI GART */
1400                 radeon_set_pcigart(dev_priv, 1);
1401         }
1402
1403         radeon_cp_load_microcode(dev_priv);
1404         radeon_cp_init_ring_buffer(dev, dev_priv);
1405
1406         radeon_do_engine_reset(dev);
1407         radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
1408
1409         DRM_DEBUG("radeon_do_resume_cp() complete\n");
1410
1411         return 0;
1412 }
1413
1414 int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
1415 {
1416         drm_radeon_init_t *init = data;
1417
1418         /* on a modesetting driver ignore this stuff */
1419         if (drm_core_check_feature(dev, DRIVER_MODESET))
1420                 return 0;
1421
1422         LOCK_TEST_WITH_RETURN(dev, file_priv);
1423
1424         if (init->func == RADEON_INIT_R300_CP)
1425                 r300_init_reg_flags(dev);
1426
1427         switch (init->func) {
1428         case RADEON_INIT_CP:
1429         case RADEON_INIT_R200_CP:
1430         case RADEON_INIT_R300_CP:
1431                 return radeon_do_init_cp(dev, init, file_priv);
1432         case RADEON_CLEANUP_CP:
1433                 return radeon_do_cleanup_cp(dev);
1434         }
1435
1436         return -EINVAL;
1437 }
1438
1439 int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
1440 {
1441         drm_radeon_private_t *dev_priv = dev->dev_private;
1442         DRM_DEBUG("\n");
1443
1444         if (drm_core_check_feature(dev, DRIVER_MODESET))
1445                 return 0;
1446
1447         LOCK_TEST_WITH_RETURN(dev, file_priv);
1448
1449         if (dev_priv->cp_running) {
1450                 DRM_DEBUG("while CP running\n");
1451                 return 0;
1452         }
1453         if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
1454                 DRM_DEBUG("called with bogus CP mode (%d)\n",
1455                           dev_priv->cp_mode);
1456                 return 0;
1457         }
1458
1459         radeon_do_cp_start(dev_priv);
1460
1461         return 0;
1462 }
1463
1464 /* Stop the CP.  The engine must have been idled before calling this
1465  * routine.
1466  */
1467 int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
1468 {
1469         drm_radeon_private_t *dev_priv = dev->dev_private;
1470         drm_radeon_cp_stop_t *stop = data;
1471         int ret;
1472         DRM_DEBUG("\n");
1473
1474         if (drm_core_check_feature(dev, DRIVER_MODESET))
1475                 return 0;
1476
1477         LOCK_TEST_WITH_RETURN(dev, file_priv);
1478
1479         if (!dev_priv->cp_running)
1480                 return 0;
1481
1482         /* Flush any pending CP commands.  This ensures any outstanding
1483          * commands are exectuted by the engine before we turn it off.
1484          */
1485         if (stop->flush) {
1486                 radeon_do_cp_flush(dev_priv);
1487         }
1488
1489         /* If we fail to make the engine go idle, we return an error
1490          * code so that the DRM ioctl wrapper can try again.
1491          */
1492         if (stop->idle) {
1493                 ret = radeon_do_cp_idle(dev_priv);
1494                 if (ret)
1495                         return ret;
1496         }
1497
1498         /* Finally, we can turn off the CP.  If the engine isn't idle,
1499          * we will get some dropped triangles as they won't be fully
1500          * rendered before the CP is shut down.
1501          */
1502         radeon_do_cp_stop(dev_priv);
1503
1504         /* Reset the engine */
1505         radeon_do_engine_reset(dev);
1506
1507         return 0;
1508 }
1509
1510 void radeon_do_release(struct drm_device * dev)
1511 {
1512         drm_radeon_private_t *dev_priv = dev->dev_private;
1513         int i, ret;
1514
1515         if (drm_core_check_feature(dev, DRIVER_MODESET))
1516                 return;
1517
1518         if (dev_priv) {
1519                 if (dev_priv->cp_running) {
1520                         /* Stop the cp */
1521                         while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
1522                                 DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1523 #ifdef __linux__
1524                                 schedule();
1525 #else
1526 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
1527                                 mtx_sleep(&ret, &dev->dev_lock, PZERO, "rdnrel",
1528                                        1);
1529 #else
1530                                 tsleep(&ret, PZERO, "rdnrel", 1);
1531 #endif
1532 #endif
1533                         }
1534                         radeon_do_cp_stop(dev_priv);
1535                         radeon_do_engine_reset(dev);
1536                 }
1537
1538                 /* Disable *all* interrupts */
1539                 if (dev_priv->mmio)     /* remove this after permanent addmaps */
1540                         RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
1541
1542                 if (dev_priv->mmio) {   /* remove all surfaces */
1543                         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1544                                 RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
1545                                 RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
1546                                              16 * i, 0);
1547                                 RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
1548                                              16 * i, 0);
1549                         }
1550                 }
1551
1552                 /* Free memory heap structures */
1553                 radeon_mem_takedown(&(dev_priv->gart_heap));
1554                 radeon_mem_takedown(&(dev_priv->fb_heap));
1555
1556
1557                 radeon_gem_mm_fini(dev);
1558
1559                 /* deallocate kernel resources */
1560                 radeon_do_cleanup_cp(dev);
1561         }
1562 }
1563
1564 /* Just reset the CP ring.  Called as part of an X Server engine reset.
1565  */
1566 int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1567 {
1568         drm_radeon_private_t *dev_priv = dev->dev_private;
1569         DRM_DEBUG("\n");
1570
1571         if (drm_core_check_feature(dev, DRIVER_MODESET))
1572                 return 0;
1573
1574         LOCK_TEST_WITH_RETURN(dev, file_priv);
1575
1576         if (!dev_priv) {
1577                 DRM_DEBUG("called before init done\n");
1578                 return -EINVAL;
1579         }
1580
1581         radeon_do_cp_reset(dev_priv);
1582
1583         /* The CP is no longer running after an engine reset */
1584         dev_priv->cp_running = 0;
1585
1586         return 0;
1587 }
1588
1589 int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
1590 {
1591         drm_radeon_private_t *dev_priv = dev->dev_private;
1592         DRM_DEBUG("\n");
1593
1594
1595         if (!drm_core_check_feature(dev, DRIVER_MODESET))
1596                 LOCK_TEST_WITH_RETURN(dev, file_priv);
1597
1598         return radeon_do_cp_idle(dev_priv);
1599 }
1600
1601 /* Added by Charl P. Botha to call radeon_do_resume_cp().
1602  */
1603 int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
1604 {
1605
1606         if (drm_core_check_feature(dev, DRIVER_MODESET))
1607                 return 0;
1608
1609         return radeon_do_resume_cp(dev);
1610 }
1611
1612 int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1613 {
1614         DRM_DEBUG("\n");
1615
1616         if (drm_core_check_feature(dev, DRIVER_MODESET))
1617                 return 0;
1618
1619         LOCK_TEST_WITH_RETURN(dev, file_priv);
1620
1621         return radeon_do_engine_reset(dev);
1622 }
1623
1624 /* ================================================================
1625  * Fullscreen mode
1626  */
1627
1628 /* KW: Deprecated to say the least:
1629  */
1630 int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
1631 {
1632         return 0;
1633 }
1634
1635 /* ================================================================
1636  * Freelist management
1637  */
1638
1639 /* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
1640  *   bufs until freelist code is used.  Note this hides a problem with
1641  *   the scratch register * (used to keep track of last buffer
1642  *   completed) being written to before * the last buffer has actually
1643  *   completed rendering.
1644  *
1645  * KW:  It's also a good way to find free buffers quickly.
1646  *
1647  * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
1648  * sleep.  However, bugs in older versions of radeon_accel.c mean that
1649  * we essentially have to do this, else old clients will break.
1650  *
1651  * However, it does leave open a potential deadlock where all the
1652  * buffers are held by other clients, which can't release them because
1653  * they can't get the lock.
1654  */
1655
1656 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1657 {
1658         struct drm_device_dma *dma = dev->dma;
1659         drm_radeon_private_t *dev_priv = dev->dev_private;
1660         drm_radeon_buf_priv_t *buf_priv;
1661         struct drm_buf *buf;
1662         int i, t;
1663         int start;
1664
1665         if (++dev_priv->last_buf >= dma->buf_count)
1666                 dev_priv->last_buf = 0;
1667
1668         start = dev_priv->last_buf;
1669
1670         for (t = 0; t < dev_priv->usec_timeout; t++) {
1671                 u32 done_age = GET_SCRATCH(1);
1672                 DRM_DEBUG("done_age = %d\n", done_age);
1673                 for (i = start; i < dma->buf_count; i++) {
1674                         buf = dma->buflist[i];
1675                         buf_priv = buf->dev_private;
1676                         if (buf->file_priv == NULL || (buf->pending &&
1677                                                        buf_priv->age <=
1678                                                        done_age)) {
1679                                 dev_priv->stats.requested_bufs++;
1680                                 buf->pending = 0;
1681                                 return buf;
1682                         }
1683                         start = 0;
1684                 }
1685
1686                 if (t) {
1687                         DRM_UDELAY(1);
1688                         dev_priv->stats.freelist_loops++;
1689                 }
1690         }
1691
1692         DRM_DEBUG("returning NULL!\n");
1693         return NULL;
1694 }
1695
1696 #if 0
1697 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1698 {
1699         struct drm_device_dma *dma = dev->dma;
1700         drm_radeon_private_t *dev_priv = dev->dev_private;
1701         drm_radeon_buf_priv_t *buf_priv;
1702         struct drm_buf *buf;
1703         int i, t;
1704         int start;
1705         u32 done_age = DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1));
1706
1707         if (++dev_priv->last_buf >= dma->buf_count)
1708                 dev_priv->last_buf = 0;
1709
1710         start = dev_priv->last_buf;
1711         dev_priv->stats.freelist_loops++;
1712
1713         for (t = 0; t < 2; t++) {
1714                 for (i = start; i < dma->buf_count; i++) {
1715                         buf = dma->buflist[i];
1716                         buf_priv = buf->dev_private;
1717                         if (buf->file_priv == 0 || (buf->pending &&
1718                                                     buf_priv->age <=
1719                                                     done_age)) {
1720                                 dev_priv->stats.requested_bufs++;
1721                                 buf->pending = 0;
1722                                 return buf;
1723                         }
1724                 }
1725                 start = 0;
1726         }
1727
1728         return NULL;
1729 }
1730 #endif
1731
1732 void radeon_freelist_reset(struct drm_device * dev)
1733 {
1734         struct drm_device_dma *dma = dev->dma;
1735         drm_radeon_private_t *dev_priv = dev->dev_private;
1736         int i;
1737
1738         dev_priv->last_buf = 0;
1739         for (i = 0; i < dma->buf_count; i++) {
1740                 struct drm_buf *buf = dma->buflist[i];
1741                 drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1742                 buf_priv->age = 0;
1743         }
1744 }
1745
1746 /* ================================================================
1747  * CP command submission
1748  */
1749
1750 int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
1751 {
1752         drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
1753         int i;
1754         u32 last_head = GET_RING_HEAD(dev_priv);
1755
1756         for (i = 0; i < dev_priv->usec_timeout; i++) {
1757                 u32 head = GET_RING_HEAD(dev_priv);
1758
1759                 ring->space = (head - ring->tail) * sizeof(u32);
1760                 if (ring->space <= 0)
1761                         ring->space += ring->size;
1762                 if (ring->space > n)
1763                         return 0;
1764
1765                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
1766
1767                 if (head != last_head)
1768                         i = 0;
1769                 last_head = head;
1770
1771                 DRM_UDELAY(1);
1772         }
1773
1774         /* FIXME: This return value is ignored in the BEGIN_RING macro! */
1775 #if RADEON_FIFO_DEBUG
1776         radeon_status(dev_priv);
1777         DRM_ERROR("failed!\n");
1778 #endif
1779         return -EBUSY;
1780 }
1781
1782 static int radeon_cp_get_buffers(struct drm_device *dev,
1783                                  struct drm_file *file_priv,
1784                                  struct drm_dma * d)
1785 {
1786         int i;
1787         struct drm_buf *buf;
1788
1789         for (i = d->granted_count; i < d->request_count; i++) {
1790                 buf = radeon_freelist_get(dev);
1791                 if (!buf)
1792                         return -EBUSY;  /* NOTE: broken client */
1793
1794                 buf->file_priv = file_priv;
1795
1796                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
1797                                      sizeof(buf->idx)))
1798                         return -EFAULT;
1799                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
1800                                      sizeof(buf->total)))
1801                         return -EFAULT;
1802
1803                 d->granted_count++;
1804         }
1805         return 0;
1806 }
1807
1808 int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
1809 {
1810         struct drm_device_dma *dma = dev->dma;
1811         int ret = 0;
1812         struct drm_dma *d = data;
1813
1814         LOCK_TEST_WITH_RETURN(dev, file_priv);
1815
1816         /* Please don't send us buffers.
1817          */
1818         if (d->send_count != 0) {
1819                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
1820                           DRM_CURRENTPID, d->send_count);
1821                 return -EINVAL;
1822         }
1823
1824         /* We'll send you buffers.
1825          */
1826         if (d->request_count < 0 || d->request_count > dma->buf_count) {
1827                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
1828                           DRM_CURRENTPID, d->request_count, dma->buf_count);
1829                 return -EINVAL;
1830         }
1831
1832         d->granted_count = 0;
1833
1834         if (d->request_count) {
1835                 ret = radeon_cp_get_buffers(dev, file_priv, d);
1836         }
1837
1838         return ret;
1839 }
1840
1841 static void radeon_get_vram_type(struct drm_device *dev)
1842 {
1843         struct drm_radeon_private *dev_priv = dev->dev_private;
1844         uint32_t tmp;
1845
1846         if (dev_priv->flags & RADEON_IS_IGP || (dev_priv->chip_family >= CHIP_R300))
1847                 dev_priv->is_ddr = true;
1848         else if (RADEON_READ(RADEON_MEM_SDRAM_MODE_REG) & RADEON_MEM_CFG_TYPE_DDR)
1849                 dev_priv->is_ddr = true;
1850         else
1851                 dev_priv->is_ddr = false;
1852
1853         if ((dev_priv->chip_family >= CHIP_R600) &&
1854             (dev_priv->chip_family <= CHIP_RV635)) {
1855                 int chansize;
1856
1857                 tmp = RADEON_READ(R600_RAMCFG);
1858                 if (tmp & R600_CHANSIZE_OVERRIDE)
1859                         chansize = 16;
1860                 else if (tmp & R600_CHANSIZE)
1861                         chansize = 64;
1862                 else
1863                         chansize = 32;
1864
1865                 if (dev_priv->chip_family == CHIP_R600)
1866                         dev_priv->ram_width = 8 * chansize;
1867                 else if (dev_priv->chip_family == CHIP_RV670)
1868                         dev_priv->ram_width = 4 * chansize;
1869                 else if ((dev_priv->chip_family == CHIP_RV610) ||
1870                          (dev_priv->chip_family == CHIP_RV620))
1871                         dev_priv->ram_width = chansize;
1872                 else if ((dev_priv->chip_family == CHIP_RV630) ||
1873                          (dev_priv->chip_family == CHIP_RV635))
1874                         dev_priv->ram_width = 2 * chansize;
1875         } else if (dev_priv->chip_family == CHIP_RV515) {
1876                 tmp = radeon_read_mc_reg(dev_priv, RV515_MC_CNTL);
1877                 tmp &= RV515_MEM_NUM_CHANNELS_MASK;
1878                 switch (tmp) {
1879                 case 0: dev_priv->ram_width = 64; break;
1880                 case 1: dev_priv->ram_width = 128; break;
1881                 default: dev_priv->ram_width = 128; break;
1882                 }
1883         } else if ((dev_priv->chip_family >= CHIP_R520) &&
1884                    (dev_priv->chip_family <= CHIP_RV570)) {
1885                 tmp = radeon_read_mc_reg(dev_priv, R520_MC_CNTL0);
1886                 switch ((tmp & R520_MEM_NUM_CHANNELS_MASK) >> R520_MEM_NUM_CHANNELS_SHIFT) {
1887                 case 0: dev_priv->ram_width = 32; break;
1888                 case 1: dev_priv->ram_width = 64; break;
1889                 case 2: dev_priv->ram_width = 128; break;
1890                 case 3: dev_priv->ram_width = 256; break;
1891                 default: dev_priv->ram_width = 128; break;
1892                 }
1893         } else if ((dev_priv->chip_family == CHIP_RV100) ||
1894                    (dev_priv->chip_family == CHIP_RS100) ||
1895                    (dev_priv->chip_family == CHIP_RS200)) {
1896                 tmp = RADEON_READ(RADEON_MEM_CNTL);
1897                 if (tmp & RV100_HALF_MODE)
1898                         dev_priv->ram_width = 32;
1899                 else
1900                         dev_priv->ram_width = 64;
1901
1902                 if (dev_priv->flags & RADEON_SINGLE_CRTC) {
1903                         dev_priv->ram_width /= 4;
1904                         dev_priv->is_ddr = true;
1905                 }
1906         } else if (dev_priv->chip_family <= CHIP_RV280) {
1907                 tmp = RADEON_READ(RADEON_MEM_CNTL);
1908                 if (tmp & RADEON_MEM_NUM_CHANNELS_MASK)
1909                         dev_priv->ram_width = 128;
1910                 else
1911                         dev_priv->ram_width = 64;
1912         } else {
1913                 /* newer IGPs */
1914                 dev_priv->ram_width = 128;
1915         }
1916         DRM_DEBUG("RAM width %d bits %cDR\n", dev_priv->ram_width, dev_priv->is_ddr ? 'D' : 'S');
1917 }
1918
1919 static void radeon_force_some_clocks(struct drm_device *dev)
1920 {
1921         struct drm_radeon_private *dev_priv = dev->dev_private;
1922         uint32_t tmp;
1923
1924         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
1925         tmp |= RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_VIP;
1926         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
1927 }
1928
1929 static void radeon_set_dynamic_clock(struct drm_device *dev, int mode)
1930 {
1931         struct drm_radeon_private *dev_priv = dev->dev_private;
1932         uint32_t tmp;
1933
1934         switch(mode) {
1935         case 0:
1936                 if (dev_priv->flags & RADEON_SINGLE_CRTC) {
1937                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
1938                         tmp |= (RADEON_SCLK_FORCE_CP   | RADEON_SCLK_FORCE_HDP |
1939                                 RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP |
1940                                 RADEON_SCLK_FORCE_E2   | RADEON_SCLK_FORCE_SE  |
1941                                 RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
1942                                 RADEON_SCLK_FORCE_RE   | RADEON_SCLK_FORCE_PB  |
1943                                 RADEON_SCLK_FORCE_TAM  | RADEON_SCLK_FORCE_TDM |
1944                                 RADEON_SCLK_FORCE_RB);
1945                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
1946                 } else if (dev_priv->chip_family == CHIP_RV350) {
1947                         /* for RV350/M10, no delays are required. */
1948                         tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
1949                         tmp |= (R300_SCLK_FORCE_TCL |
1950                                 R300_SCLK_FORCE_GA |
1951                                 R300_SCLK_FORCE_CBA);
1952                         RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
1953
1954                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
1955                         tmp &= ~(RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP      |
1956                                  RADEON_SCLK_FORCE_HDP   | RADEON_SCLK_FORCE_DISP1   |
1957                                  RADEON_SCLK_FORCE_TOP   | RADEON_SCLK_FORCE_E2      |
1958                                  R300_SCLK_FORCE_VAP     | RADEON_SCLK_FORCE_IDCT    |
1959                                  RADEON_SCLK_FORCE_VIP   | R300_SCLK_FORCE_SR        |
1960                                  R300_SCLK_FORCE_PX      | R300_SCLK_FORCE_TX        |
1961                                  R300_SCLK_FORCE_US      | RADEON_SCLK_FORCE_TV_SCLK |
1962                                  R300_SCLK_FORCE_SU      | RADEON_SCLK_FORCE_OV0);
1963                         tmp |=  RADEON_DYN_STOP_LAT_MASK;
1964                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
1965
1966                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
1967                         tmp &= ~RADEON_SCLK_MORE_FORCEON;
1968                         tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
1969                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
1970
1971                         tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
1972                         tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
1973                                 RADEON_PIXCLK_DAC_ALWAYS_ONb);
1974                         RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
1975
1976                         tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
1977                         tmp |= (RADEON_PIX2CLK_ALWAYS_ONb         |
1978                                 RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
1979                                 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
1980                                 R300_DVOCLK_ALWAYS_ONb            |
1981                                 RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
1982                                 RADEON_PIXCLK_GV_ALWAYS_ONb       |
1983                                 R300_PIXCLK_DVO_ALWAYS_ONb        |
1984                                 RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
1985                                 RADEON_PIXCLK_TMDS_ALWAYS_ONb     |
1986                                 R300_PIXCLK_TRANS_ALWAYS_ONb      |
1987                                 R300_PIXCLK_TVO_ALWAYS_ONb        |
1988                                 R300_P2G2CLK_ALWAYS_ONb           |
1989                                 R300_P2G2CLK_ALWAYS_ONb);
1990                         RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
1991                 } else {
1992                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
1993                         tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
1994                         tmp |= RADEON_SCLK_FORCE_SE;
1995
1996                         if ( dev_priv->flags & RADEON_SINGLE_CRTC ) {
1997                                 tmp |= ( RADEON_SCLK_FORCE_RB    |
1998                                          RADEON_SCLK_FORCE_TDM   |
1999                                          RADEON_SCLK_FORCE_TAM   |
2000                                          RADEON_SCLK_FORCE_PB    |
2001                                          RADEON_SCLK_FORCE_RE    |
2002                                          RADEON_SCLK_FORCE_VIP   |
2003                                          RADEON_SCLK_FORCE_IDCT  |
2004                                          RADEON_SCLK_FORCE_TOP   |
2005                                          RADEON_SCLK_FORCE_DISP1 |
2006                                          RADEON_SCLK_FORCE_DISP2 |
2007                                          RADEON_SCLK_FORCE_HDP    );
2008                         } else if ((dev_priv->chip_family == CHIP_R300) ||
2009                                    (dev_priv->chip_family == CHIP_R350)) {
2010                                 tmp |= ( RADEON_SCLK_FORCE_HDP   |
2011                                          RADEON_SCLK_FORCE_DISP1 |
2012                                          RADEON_SCLK_FORCE_DISP2 |
2013                                          RADEON_SCLK_FORCE_TOP   |
2014                                          RADEON_SCLK_FORCE_IDCT  |
2015                                          RADEON_SCLK_FORCE_VIP);
2016                         }
2017
2018                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2019
2020                         udelay(16000);
2021
2022                         if ((dev_priv->chip_family == CHIP_R300) ||
2023                             (dev_priv->chip_family == CHIP_R350)) {
2024                                 tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
2025                                 tmp |= ( R300_SCLK_FORCE_TCL |
2026                                          R300_SCLK_FORCE_GA  |
2027                                          R300_SCLK_FORCE_CBA);
2028                                 RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
2029                                 udelay(16000);
2030                         }
2031
2032                         if (dev_priv->flags & RADEON_IS_IGP) {
2033                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
2034                                 tmp &= ~(RADEON_FORCEON_MCLKA |
2035                                          RADEON_FORCEON_YCLKA);
2036                                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, tmp);
2037                                 udelay(16000);
2038                         }
2039
2040                         if ((dev_priv->chip_family == CHIP_RV200) ||
2041                             (dev_priv->chip_family == CHIP_RV250) ||
2042                             (dev_priv->chip_family == CHIP_RV280)) {
2043                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
2044                                 tmp |= RADEON_SCLK_MORE_FORCEON;
2045                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
2046                                 udelay(16000);
2047                         }
2048
2049                         tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
2050                         tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb         |
2051                                  RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
2052                                  RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
2053                                  RADEON_PIXCLK_GV_ALWAYS_ONb       |
2054                                  RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
2055                                  RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
2056                                  RADEON_PIXCLK_TMDS_ALWAYS_ONb);
2057
2058                         RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
2059                         udelay(16000);
2060
2061                         tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
2062                         tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb  |
2063                                  RADEON_PIXCLK_DAC_ALWAYS_ONb);
2064                         RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
2065                 }
2066                 DRM_DEBUG("Dynamic Clock Scaling Disabled\n");
2067                 break;
2068         case 1:
2069                 if (dev_priv->flags & RADEON_SINGLE_CRTC) {
2070                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2071                         if ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) >
2072                             RADEON_CFG_ATI_REV_A13) {
2073                                 tmp &= ~(RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_RB);
2074                         }
2075                         tmp &= ~(RADEON_SCLK_FORCE_HDP  | RADEON_SCLK_FORCE_DISP1 |
2076                                  RADEON_SCLK_FORCE_TOP  | RADEON_SCLK_FORCE_SE   |
2077                                  RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE   |
2078                                  RADEON_SCLK_FORCE_PB   | RADEON_SCLK_FORCE_TAM  |
2079                                  RADEON_SCLK_FORCE_TDM);
2080                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2081                 } else if ((dev_priv->chip_family == CHIP_R300) ||
2082                            (dev_priv->chip_family == CHIP_R350) ||
2083                            (dev_priv->chip_family == CHIP_RV350)) {
2084                         if (dev_priv->chip_family == CHIP_RV350) {
2085                                 tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
2086                                 tmp &= ~(R300_SCLK_FORCE_TCL |
2087                                          R300_SCLK_FORCE_GA  |
2088                                          R300_SCLK_FORCE_CBA);
2089                                 tmp |=  (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
2090                                          R300_SCLK_GA_MAX_DYN_STOP_LAT  |
2091                                          R300_SCLK_CBA_MAX_DYN_STOP_LAT);
2092                                 RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
2093
2094                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2095                                 tmp &= ~(RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP      |
2096                                          RADEON_SCLK_FORCE_HDP   | RADEON_SCLK_FORCE_DISP1   |
2097                                          RADEON_SCLK_FORCE_TOP   | RADEON_SCLK_FORCE_E2      |
2098                                          R300_SCLK_FORCE_VAP     | RADEON_SCLK_FORCE_IDCT    |
2099                                          RADEON_SCLK_FORCE_VIP   | R300_SCLK_FORCE_SR        |
2100                                          R300_SCLK_FORCE_PX      | R300_SCLK_FORCE_TX        |
2101                                          R300_SCLK_FORCE_US      | RADEON_SCLK_FORCE_TV_SCLK |
2102                                          R300_SCLK_FORCE_SU      | RADEON_SCLK_FORCE_OV0);
2103                                 tmp |=  RADEON_DYN_STOP_LAT_MASK;
2104                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2105
2106                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
2107                                 tmp &= ~RADEON_SCLK_MORE_FORCEON;
2108                                 tmp |=  RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
2109                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
2110
2111                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
2112                                 tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
2113                                         RADEON_PIXCLK_DAC_ALWAYS_ONb);
2114                                 RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
2115
2116                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
2117                                 tmp |= (RADEON_PIX2CLK_ALWAYS_ONb         |
2118                                         RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
2119                                         RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
2120                                         R300_DVOCLK_ALWAYS_ONb            |
2121                                         RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
2122                                         RADEON_PIXCLK_GV_ALWAYS_ONb       |
2123                                         R300_PIXCLK_DVO_ALWAYS_ONb        |
2124                                         RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
2125                                         RADEON_PIXCLK_TMDS_ALWAYS_ONb     |
2126                                         R300_PIXCLK_TRANS_ALWAYS_ONb      |
2127                                         R300_PIXCLK_TVO_ALWAYS_ONb        |
2128                                         R300_P2G2CLK_ALWAYS_ONb           |
2129                                         R300_P2G2CLK_ALWAYS_ONb);
2130                                 RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
2131
2132                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_MISC);
2133                                 tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
2134                                         RADEON_IO_MCLK_DYN_ENABLE);
2135                                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_MISC, tmp);
2136
2137                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
2138                                 tmp |= (RADEON_FORCEON_MCLKA |
2139                                         RADEON_FORCEON_MCLKB);
2140
2141                                 tmp &= ~(RADEON_FORCEON_YCLKA  |
2142                                          RADEON_FORCEON_YCLKB  |
2143                                          RADEON_FORCEON_MC);
2144
2145                                 /* Some releases of vbios have set DISABLE_MC_MCLKA
2146                                    and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
2147                                    bits will cause H/W hang when reading video memory with dynamic clocking
2148                                    enabled. */
2149                                 if ((tmp & R300_DISABLE_MC_MCLKA) &&
2150                                     (tmp & R300_DISABLE_MC_MCLKB)) {
2151                                         /* If both bits are set, then check the active channels */
2152                                         tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
2153                                         if (dev_priv->ram_width == 64) {
2154                                                 if (RADEON_READ(RADEON_MEM_CNTL) & R300_MEM_USE_CD_CH_ONLY)
2155                                                         tmp &= ~R300_DISABLE_MC_MCLKB;
2156                                                 else
2157                                                         tmp &= ~R300_DISABLE_MC_MCLKA;
2158                                         } else {
2159                                                 tmp &= ~(R300_DISABLE_MC_MCLKA |
2160                                                          R300_DISABLE_MC_MCLKB);
2161                                         }
2162                                 }
2163
2164                                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, tmp);
2165                         } else {
2166                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2167                                 tmp &= ~(R300_SCLK_FORCE_VAP);
2168                                 tmp |= RADEON_SCLK_FORCE_CP;
2169                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2170                                 udelay(15000);
2171
2172                                 tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
2173                                 tmp &= ~(R300_SCLK_FORCE_TCL |
2174                                          R300_SCLK_FORCE_GA  |
2175                                          R300_SCLK_FORCE_CBA);
2176                                 RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
2177                         }
2178                 } else {
2179                         tmp = RADEON_READ_PLL(dev_priv, RADEON_CLK_PWRMGT_CNTL);
2180                         tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK     |
2181                                  RADEON_DISP_DYN_STOP_LAT_MASK   |
2182                                  RADEON_DYN_STOP_MODE_MASK);
2183
2184                         tmp |= (RADEON_ENGIN_DYNCLK_MODE |
2185                                 (0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
2186                         RADEON_WRITE_PLL(dev_priv, RADEON_CLK_PWRMGT_CNTL, tmp);
2187                         udelay(15000);
2188
2189                         tmp = RADEON_READ_PLL(dev_priv, RADEON_CLK_PIN_CNTL);
2190                         tmp |= RADEON_SCLK_DYN_START_CNTL;
2191                         RADEON_WRITE_PLL(dev_priv, RADEON_CLK_PIN_CNTL, tmp);
2192                         udelay(15000);
2193
2194                         /* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
2195                            to lockup randomly, leave them as set by BIOS.
2196                         */
2197                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2198                         /*tmp &= RADEON_SCLK_SRC_SEL_MASK;*/
2199                         tmp &= ~RADEON_SCLK_FORCEON_MASK;
2200
2201                         /*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300*/
2202                         if (((dev_priv->chip_family == CHIP_RV250) &&
2203                              ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
2204                               RADEON_CFG_ATI_REV_A13)) ||
2205                             ((dev_priv->chip_family == CHIP_RV100) &&
2206                              ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <=
2207                               RADEON_CFG_ATI_REV_A13))){
2208                                 tmp |= RADEON_SCLK_FORCE_CP;
2209                                 tmp |= RADEON_SCLK_FORCE_VIP;
2210                         }
2211
2212                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2213
2214                         if ((dev_priv->chip_family == CHIP_RV200) ||
2215                             (dev_priv->chip_family == CHIP_RV250) ||
2216                             (dev_priv->chip_family == CHIP_RV280)) {
2217                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
2218                                 tmp &= ~RADEON_SCLK_MORE_FORCEON;
2219
2220                                 /* RV200::A11 A12 RV250::A11 A12 */
2221                                 if (((dev_priv->chip_family == CHIP_RV200) ||
2222                                      (dev_priv->chip_family == CHIP_RV250)) &&
2223                                     ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
2224                                      RADEON_CFG_ATI_REV_A13)) {
2225                                         tmp |= RADEON_SCLK_MORE_FORCEON;
2226                                 }
2227                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
2228                                 udelay(15000);
2229                         }
2230
2231                         /* RV200::A11 A12, RV250::A11 A12 */
2232                         if (((dev_priv->chip_family == CHIP_RV200) ||
2233                              (dev_priv->chip_family == CHIP_RV250)) &&
2234                             ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
2235                              RADEON_CFG_ATI_REV_A13)) {
2236                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_PLL_PWRMGT_CNTL);
2237                                 tmp |= RADEON_TCL_BYPASS_DISABLE;
2238                                 RADEON_WRITE_PLL(dev_priv, RADEON_PLL_PWRMGT_CNTL, tmp);
2239                         }
2240                         udelay(15000);
2241
2242                         /*enable dynamic mode for display clocks (PIXCLK and PIX2CLK)*/
2243                         tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
2244                         tmp |=  (RADEON_PIX2CLK_ALWAYS_ONb         |
2245                                  RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
2246                                  RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
2247                                  RADEON_PIXCLK_GV_ALWAYS_ONb       |
2248                                  RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
2249                                  RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
2250                                  RADEON_PIXCLK_TMDS_ALWAYS_ONb);
2251
2252                         RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
2253                         udelay(15000);
2254
2255                         tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
2256                         tmp |= (RADEON_PIXCLK_ALWAYS_ONb  |
2257                                 RADEON_PIXCLK_DAC_ALWAYS_ONb);
2258
2259                         RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
2260                         udelay(15000);
2261                 }
2262                 DRM_DEBUG("Dynamic Clock Scaling Enabled\n");
2263                 break;
2264         default:
2265                 break;
2266         }
2267
2268 }
2269
2270 int radeon_modeset_cp_init(struct drm_device *dev)
2271 {
2272         drm_radeon_private_t *dev_priv = dev->dev_private;
2273
2274         /* allocate a ring and ring rptr bits from GART space */
2275         /* these are allocated in GEM files */
2276
2277         dev_priv->usec_timeout = RADEON_DEFAULT_CP_TIMEOUT;
2278         dev_priv->ring.size = RADEON_DEFAULT_RING_SIZE;
2279         dev_priv->cp_mode = RADEON_CSQ_PRIBM_INDBM;
2280
2281         dev_priv->ring.start = (u32 *)(void *)(unsigned long)dev_priv->mm.ring_map.virtual;
2282         dev_priv->ring.end = (u32 *)(void *)(unsigned long)dev_priv->mm.ring_map.virtual +
2283                 dev_priv->ring.size / sizeof(u32);
2284         dev_priv->ring.size_l2qw = drm_order(dev_priv->ring.size / 8);
2285         dev_priv->ring.rptr_update = 4096;
2286         dev_priv->ring.rptr_update_l2qw = drm_order(4096 / 8);
2287         dev_priv->ring.fetch_size = 32;
2288         dev_priv->ring.fetch_size_l2ow = drm_order(32 / 16);
2289         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
2290         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
2291
2292         dev_priv->new_memmap = 1;
2293
2294         radeon_cp_load_microcode(dev_priv);
2295
2296         DRM_DEBUG("ring offset is %x %x\n", dev_priv->mm.ring->offset, dev_priv->mm.ring_read_ptr->offset);
2297
2298         radeon_cp_init_ring_buffer(dev, dev_priv);
2299
2300         radeon_do_engine_reset(dev);
2301         radeon_test_writeback(dev_priv);
2302
2303         radeon_do_cp_start(dev_priv);
2304         return 0;
2305 }
2306
2307 static bool radeon_get_bios(struct drm_device *dev)
2308 {
2309         drm_radeon_private_t *dev_priv = dev->dev_private;
2310         u8 __iomem *bios;
2311         size_t size;
2312         uint16_t tmp;
2313
2314         bios = pci_map_rom(dev->pdev, &size);
2315         if (!bios)
2316                 return -1;
2317
2318         dev_priv->bios = kmalloc(size, GFP_KERNEL);
2319         if (!dev_priv->bios) {
2320                 pci_unmap_rom(dev->pdev, bios);
2321                 return -1;
2322         }
2323
2324         memcpy(dev_priv->bios, bios, size);
2325
2326         pci_unmap_rom(dev->pdev, bios);
2327
2328         if (dev_priv->bios[0] != 0x55 || dev_priv->bios[1] != 0xaa)
2329                 goto free_bios;
2330
2331         dev_priv->bios_header_start = radeon_bios16(dev_priv, 0x48);
2332
2333         if (!dev_priv->bios_header_start)
2334                 goto free_bios;
2335
2336         tmp = dev_priv->bios_header_start + 4;
2337
2338         if (!memcmp(dev_priv->bios + tmp, "ATOM", 4) ||
2339             !memcmp(dev_priv->bios + tmp, "MOTA", 4))
2340                 dev_priv->is_atom_bios = true;
2341         else
2342                 dev_priv->is_atom_bios = false;
2343
2344         DRM_DEBUG("%sBIOS detected\n", dev_priv->is_atom_bios ? "ATOM" : "COM");
2345         return true;
2346 free_bios:
2347         kfree(dev_priv->bios);
2348         dev_priv->bios = NULL;
2349         return false;
2350 }
2351
2352 int radeon_modeset_preinit(struct drm_device *dev)
2353 {
2354         drm_radeon_private_t *dev_priv = dev->dev_private;
2355         static struct card_info card;
2356         int ret;
2357
2358         card.dev = dev;
2359         card.reg_read = cail_reg_read;
2360         card.reg_write = cail_reg_write;
2361         card.mc_read = cail_mc_read;
2362         card.mc_write = cail_mc_write;
2363
2364         ret = radeon_get_bios(dev);
2365         if (!ret)
2366                 return -1;
2367
2368         if (dev_priv->is_atom_bios) {
2369                 dev_priv->mode_info.atom_context = atom_parse(&card, dev_priv->bios);
2370                 radeon_get_clock_info(dev);
2371         }
2372         return 0;
2373 }
2374
2375
2376 int radeon_driver_load(struct drm_device *dev, unsigned long flags)
2377 {
2378         drm_radeon_private_t *dev_priv;
2379         int ret = 0;
2380
2381         dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
2382         if (dev_priv == NULL)
2383                 return -ENOMEM;
2384
2385         memset(dev_priv, 0, sizeof(drm_radeon_private_t));
2386         dev->dev_private = (void *)dev_priv;
2387         dev_priv->flags = flags;
2388
2389         switch (flags & RADEON_FAMILY_MASK) {
2390         case CHIP_R100:
2391         case CHIP_RV200:
2392         case CHIP_R200:
2393         case CHIP_R300:
2394         case CHIP_R350:
2395         case CHIP_R420:
2396         case CHIP_RV410:
2397         case CHIP_RV515:
2398         case CHIP_R520:
2399         case CHIP_RV570:
2400         case CHIP_R580:
2401                 dev_priv->flags |= RADEON_HAS_HIERZ;
2402                 break;
2403         default:
2404                 /* all other chips have no hierarchical z buffer */
2405                 break;
2406         }
2407
2408         dev_priv->chip_family = flags & RADEON_FAMILY_MASK;
2409         if (drm_device_is_agp(dev))
2410                 dev_priv->flags |= RADEON_IS_AGP;
2411         else if (drm_device_is_pcie(dev))
2412                 dev_priv->flags |= RADEON_IS_PCIE;
2413         else
2414                 dev_priv->flags |= RADEON_IS_PCI;
2415
2416
2417
2418         DRM_DEBUG("%s card detected\n",
2419                   ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
2420
2421         ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
2422                          drm_get_resource_len(dev, 2), _DRM_REGISTERS,
2423                          _DRM_DRIVER | _DRM_READ_ONLY, &dev_priv->mmio);
2424         if (ret != 0)
2425                 return ret;
2426
2427         if (drm_core_check_feature(dev, DRIVER_MODESET))
2428                 radeon_modeset_preinit(dev);
2429
2430
2431         radeon_get_vram_type(dev);
2432
2433         dev_priv->pll_errata = 0;
2434
2435         if (dev_priv->chip_family == CHIP_R300 &&
2436             (RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) == RADEON_CFG_ATI_REV_A11)
2437                 dev_priv->pll_errata |= CHIP_ERRATA_R300_CG;
2438
2439         if (dev_priv->chip_family == CHIP_RV200 ||
2440             dev_priv->chip_family == CHIP_RS200)
2441                 dev_priv->pll_errata |= CHIP_ERRATA_PLL_DUMMYREADS;
2442
2443
2444         if (dev_priv->chip_family == CHIP_RV100 ||
2445             dev_priv->chip_family == CHIP_RS100 ||
2446             dev_priv->chip_family == CHIP_RS200)
2447                 dev_priv->pll_errata |= CHIP_ERRATA_PLL_DELAY;
2448
2449
2450         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2451                 if ((dev_priv->flags & RADEON_IS_MOBILITY) && !radeon_is_avivo(dev_priv)) {
2452                         radeon_set_dynamic_clock(dev, radeon_dynclks);
2453                 } else if (radeon_is_avivo(dev_priv)) {
2454                         if (radeon_dynclks) {
2455                                 radeon_atom_static_pwrmgt_setup(dev, 1);
2456                                 radeon_atom_dyn_clk_setup(dev, 1);
2457                         }
2458                 }
2459                 radeon_force_some_clocks(dev);
2460         }
2461
2462         /* init memory manager - start with all of VRAM and a 32MB GART aperture for now */
2463         dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
2464
2465         drm_bo_driver_init(dev);
2466
2467         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2468
2469                 dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
2470                 dev_priv->fb_size =
2471                         ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
2472                         - dev_priv->fb_location;
2473                 radeon_gem_mm_init(dev);
2474                 radeon_modeset_init(dev);
2475
2476                 radeon_modeset_cp_init(dev);
2477                 dev->devname = kstrdup(DRIVER_NAME, GFP_KERNEL);
2478
2479                 drm_irq_install(dev);
2480         }
2481
2482
2483         return ret;
2484 }
2485
2486
2487 int radeon_master_create(struct drm_device *dev, struct drm_master *master)
2488 {
2489         struct drm_radeon_master_private *master_priv;
2490         unsigned long sareapage;
2491         int ret;
2492
2493         master_priv = drm_calloc(1, sizeof(*master_priv), DRM_MEM_DRIVER);
2494         if (!master_priv)
2495                 return -ENOMEM;
2496
2497         /* prebuild the SAREA */
2498         sareapage = max(SAREA_MAX, PAGE_SIZE);
2499         ret = drm_addmap(dev, 0, sareapage, _DRM_SHM, _DRM_CONTAINS_LOCK|_DRM_DRIVER,
2500                          &master_priv->sarea);
2501         if (ret) {
2502                 DRM_ERROR("SAREA setup failed\n");
2503                 return ret;
2504         }
2505         master_priv->sarea_priv = master_priv->sarea->handle + sizeof(struct drm_sarea);
2506         master_priv->sarea_priv->pfCurrentPage = 0;
2507
2508         master->driver_priv = master_priv;
2509         return 0;
2510 }
2511
2512 void radeon_master_destroy(struct drm_device *dev, struct drm_master *master)
2513 {
2514         struct drm_radeon_master_private *master_priv = master->driver_priv;
2515         struct drm_radeon_private *dev_priv = dev->dev_private;
2516
2517         if (!master_priv)
2518                 return;
2519
2520         if (master_priv->sarea_priv &&
2521             master_priv->sarea_priv->pfCurrentPage != 0)
2522                 radeon_cp_dispatch_flip(dev, master);
2523
2524         master_priv->sarea_priv = NULL;
2525         if (master_priv->sarea)
2526                 drm_rmmap(dev, master_priv->sarea);
2527
2528         drm_free(master_priv, sizeof(*master_priv), DRM_MEM_DRIVER);
2529
2530         master->driver_priv = NULL;
2531 }
2532 /* Create mappings for registers and framebuffer so userland doesn't necessarily
2533  * have to find them.
2534  */
2535 int radeon_driver_firstopen(struct drm_device *dev)
2536 {
2537         int ret;
2538         drm_local_map_t *map;
2539         drm_radeon_private_t *dev_priv = dev->dev_private;
2540
2541         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
2542
2543         if (!drm_core_check_feature(dev, DRIVER_MODESET))
2544                 radeon_gem_mm_init(dev);
2545
2546         ret = drm_addmap(dev, dev_priv->fb_aper_offset,
2547                          drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
2548                          _DRM_WRITE_COMBINING, &map);
2549         if (ret != 0)
2550                 return ret;
2551
2552         return 0;
2553 }
2554
2555 int radeon_driver_unload(struct drm_device *dev)
2556 {
2557         drm_radeon_private_t *dev_priv = dev->dev_private;
2558
2559         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2560                 drm_irq_uninstall(dev);
2561                 radeon_modeset_cleanup(dev);
2562                 radeon_gem_mm_fini(dev);
2563         }
2564
2565         drm_bo_driver_finish(dev);
2566         drm_rmmap(dev, dev_priv->mmio);
2567
2568         DRM_DEBUG("\n");
2569         drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
2570
2571         dev->dev_private = NULL;
2572         return 0;
2573 }
2574
2575 void radeon_gart_flush(struct drm_device *dev)
2576 {
2577         drm_radeon_private_t *dev_priv = dev->dev_private;
2578
2579         if (dev_priv->flags & RADEON_IS_IGPGART) {
2580                 IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
2581                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, RS480_GART_CACHE_INVALIDATE);
2582                 IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
2583                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
2584         } else if (dev_priv->flags & RADEON_IS_PCIE) {
2585                 u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
2586                 tmp |= RADEON_PCIE_TX_GART_INVALIDATE_TLB;
2587                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp);
2588                 tmp &= ~RADEON_PCIE_TX_GART_INVALIDATE_TLB;
2589                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp);
2590         } else {
2591
2592
2593         }
2594
2595 }