shared-core/radeon_cp.c

   1 /* radeon_cp.c -- CP support for Radeon -*- linux-c -*- */
   2 /*
   3  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   4  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   5  * Copyright 2007 Advanced Micro Devices, Inc.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  23  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  24  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  *
  27  * Authors:
  28  *    Kevin E. Martin <martin@valinux.com>
  29  *    Gareth Hughes <gareth@valinux.com>
  30  */
  31
  32 #include "drmP.h"
  33 #include "drm.h"
  34 #include "drm_sarea.h"
  35 #include "radeon_drm.h"
  36 #include "radeon_drv.h"
  37 #include "r300_reg.h"
  38
  39 #include "radeon_microcode.h"
  40 #define RADEON_FIFO_DEBUG       0
  41
  42 static int radeon_do_cleanup_cp(struct drm_device * dev);
  43 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv);
  44
  45 static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  46 {
  47         u32 ret;
  48         RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
  49         ret = RADEON_READ(R520_MC_IND_DATA);
  50         RADEON_WRITE(R520_MC_IND_INDEX, 0);
  51         return ret;
  52 }
  53
  54 static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  55 {
  56         u32 ret;
  57         RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
  58         ret = RADEON_READ(RS480_NB_MC_DATA);
  59         RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
  60         return ret;
  61 }
  62
  63 static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  64 {
  65         u32 ret;
  66         RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
  67         ret = RADEON_READ(RS690_MC_DATA);
  68         RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
  69         return ret;
  70 }
  71
  72 static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  73 {
  74         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  75             return RS690_READ_MCIND(dev_priv, addr);
  76         else
  77             return RS480_READ_MCIND(dev_priv, addr);
  78 }
  79
  80 u32 radeon_read_mc_reg(drm_radeon_private_t *dev_priv, int addr)
  81 {
  82         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  83                 return IGP_READ_MCIND(dev_priv, addr);
  84         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515)
  85                 return R500_READ_MCIND(dev_priv, addr);
  86         return 0;
  87 }
  88
  89 void radeon_write_mc_reg(drm_radeon_private_t *dev_priv, u32 addr, u32 val)
  90 {
  91         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
  92                 IGP_WRITE_MCIND(addr, val);
  93         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515)
  94                 R500_WRITE_MCIND(addr, val);
  95 }
  96
  97 u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
  98 {
  99
 100         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 101                 return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
 102         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 103                 return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
 104         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 105                 return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
 106         else
 107                 return RADEON_READ(RADEON_MC_FB_LOCATION);
 108 }
 109
 110 void radeon_read_agp_location(drm_radeon_private_t *dev_priv, u32 *agp_lo, u32 *agp_hi)
 111 {
 112         if (dev_priv->chip_family == CHIP_RV770) {
 113
 114         } else if (dev_priv->chip_family == CHIP_R600) {
 115                 *agp_lo = RADEON_READ(R600_MC_VM_AGP_BOT);
 116                 *agp_hi = RADEON_READ(R600_MC_VM_AGP_TOP);
 117         } else if (dev_priv->chip_family == CHIP_RV515) {
 118                 *agp_lo = radeon_read_mc_reg(dev_priv, RV515_MC_FB_LOCATION);
 119                 *agp_hi = 0;
 120         } else if (dev_priv->chip_family == CHIP_RS600) {
 121                 *agp_lo = 0;
 122                 *agp_hi = 0;
 123         } else if (dev_priv->chip_family == CHIP_RS690 ||
 124                    dev_priv->chip_family == CHIP_RS740) {
 125                 *agp_lo = radeon_read_mc_reg(dev_priv, RS690_MC_AGP_LOCATION);
 126                 *agp_hi = 0;
 127         } else if (dev_priv->chip_family >= CHIP_R520) {
 128                 *agp_lo = radeon_read_mc_reg(dev_priv, R520_MC_AGP_LOCATION);
 129                 *agp_hi = 0;
 130         } else {
 131                 *agp_lo = RADEON_READ(RADEON_MC_FB_LOCATION);
 132                 *agp_hi = 0;
 133         }
 134 }
 135
 136 void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
 137 {
 138         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 139                 R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
 140         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 141                 RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
 142         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 143                 R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
 144         else
 145                 RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
 146 }
 147
 148 static void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc, u32 agp_loc_hi)
 149 {
 150         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 151                 R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
 152         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 153                 RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
 154         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 155                 R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
 156         else
 157                 RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
 158 }
 159
 160 static void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
 161 {
 162         u32 agp_base_hi = upper_32_bits(agp_base);
 163         u32 agp_base_lo = agp_base & 0xffffffff;
 164
 165         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
 166                 R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
 167                 R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
 168         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
 169                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
 170                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
 171         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
 172                 R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
 173                 R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
 174         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 175                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 176                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 177                 RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
 178         } else {
 179                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 180                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
 181                         RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
 182         }
 183 }
 184
 185
 186 void radeon_pll_errata_after_index(struct drm_radeon_private *dev_priv)
 187 {
 188         if (!(dev_priv->pll_errata & CHIP_ERRATA_PLL_DUMMYREADS))
 189                 return;
 190
 191         (void)RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 192         (void)RADEON_READ(RADEON_CRTC_GEN_CNTL);
 193 }
 194
 195 void radeon_pll_errata_after_data(struct drm_radeon_private *dev_priv)
 196 {
 197         /* This workarounds is necessary on RV100, RS100 and RS200 chips
 198          * or the chip could hang on a subsequent access
 199          */
 200         if (dev_priv->pll_errata & CHIP_ERRATA_PLL_DELAY)
 201                 udelay(5000);
 202
 203         /* This function is required to workaround a hardware bug in some (all?)
 204          * revisions of the R300.  This workaround should be called after every
 205          * CLOCK_CNTL_INDEX register access.  If not, register reads afterward
 206          * may not be correct.
 207          */
 208         if (dev_priv->pll_errata & CHIP_ERRATA_R300_CG) {
 209                 uint32_t save, tmp;
 210
 211                 save = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 212                 tmp = save & ~(0x3f | RADEON_PLL_WR_EN);
 213                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, tmp);
 214                 tmp = RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 215                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, save);
 216         }
 217 }
 218
 219 u32 RADEON_READ_PLL(struct drm_radeon_private *dev_priv, int addr)
 220 {
 221         uint32_t data;
 222
 223         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x3f);
 224         radeon_pll_errata_after_index(dev_priv);
 225         data = RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 226         radeon_pll_errata_after_data(dev_priv);
 227         return data;
 228 }
 229
 230 void RADEON_WRITE_PLL(struct drm_radeon_private *dev_priv, int addr, uint32_t data)
 231 {
 232         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, ((addr & 0x3f) | RADEON_PLL_WR_EN));
 233         radeon_pll_errata_after_index(dev_priv);
 234         RADEON_WRITE(RADEON_CLOCK_CNTL_DATA, data);
 235         radeon_pll_errata_after_data(dev_priv);
 236 }
 237
 238 static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
 239 {
 240         RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
 241         return RADEON_READ(RADEON_PCIE_DATA);
 242 }
 243
 244 /* ATOM accessor methods */
 245 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 246 {
 247         uint32_t ret = radeon_read_mc_reg(info->dev->dev_private, reg);
 248
 249         //      DRM_DEBUG("(%x) = %x\n", reg, ret);
 250         return ret;
 251 }
 252
 253 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 254 {
 255   //    DRM_DEBUG("(%x,  %x)\n", reg, val);
 256         radeon_write_mc_reg(info->dev->dev_private, reg, val);
 257 }
 258
 259 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 260 {
 261         drm_radeon_private_t *dev_priv = info->dev->dev_private;
 262
 263         //      DRM_DEBUG("(%x,  %x)\n", reg*4, val);
 264         RADEON_WRITE(reg*4, val);
 265 }
 266
 267 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 268 {
 269         uint32_t ret;
 270         drm_radeon_private_t *dev_priv = info->dev->dev_private;
 271
 272         ret = RADEON_READ(reg*4);
 273         //      DRM_DEBUG("(%x) = %x\n", reg*4, ret);
 274         return ret;
 275 }
 276
 277 #if RADEON_FIFO_DEBUG
 278 static void radeon_status(drm_radeon_private_t * dev_priv)
 279 {
 280         printk("%s:\n", __FUNCTION__);
 281         printk("RBBM_STATUS = 0x%08x\n",
 282                (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
 283         printk("CP_RB_RTPR = 0x%08x\n",
 284                (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
 285         printk("CP_RB_WTPR = 0x%08x\n",
 286                (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
 287         printk("AIC_CNTL = 0x%08x\n",
 288                (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
 289         printk("AIC_STAT = 0x%08x\n",
 290                (unsigned int)RADEON_READ(RADEON_AIC_STAT));
 291         printk("AIC_PT_BASE = 0x%08x\n",
 292                (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
 293         printk("TLB_ADDR = 0x%08x\n",
 294                (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
 295         printk("TLB_DATA = 0x%08x\n",
 296                (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
 297 }
 298 #endif
 299
 300 /* ================================================================
 301  * Engine, FIFO control
 302  */
 303
 304 static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
 305 {
 306         u32 tmp;
 307         int i;
 308
 309         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 310
 311         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
 312                 tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
 313                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 314                 RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
 315
 316                 for (i = 0; i < dev_priv->usec_timeout; i++) {
 317                         if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
 318                               & RADEON_RB3D_DC_BUSY)) {
 319                                 return 0;
 320                         }
 321                         DRM_UDELAY(1);
 322                 }
 323         } else {
 324                 /* don't flush or purge cache here or lockup */
 325                 return 0;
 326         }
 327
 328 #if RADEON_FIFO_DEBUG
 329         DRM_ERROR("failed!\n");
 330         radeon_status(dev_priv);
 331 #endif
 332         return -EBUSY;
 333 }
 334
 335 static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
 336 {
 337         int i;
 338
 339         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 340
 341         for (i = 0; i < dev_priv->usec_timeout; i++) {
 342                 int slots = (RADEON_READ(RADEON_RBBM_STATUS)
 343                              & RADEON_RBBM_FIFOCNT_MASK);
 344                 if (slots >= entries)
 345                         return 0;
 346                 DRM_UDELAY(1);
 347         }
 348         DRM_INFO("wait for fifo failed status : 0x%08X 0x%08X\n",
 349                  RADEON_READ(RADEON_RBBM_STATUS),
 350                  RADEON_READ(R300_VAP_CNTL_STATUS));
 351
 352 #if RADEON_FIFO_DEBUG
 353         DRM_ERROR("failed!\n");
 354         radeon_status(dev_priv);
 355 #endif
 356         return -EBUSY;
 357 }
 358
 359 static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
 360 {
 361         int i, ret;
 362
 363         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 364
 365         ret = radeon_do_wait_for_fifo(dev_priv, 64);
 366         if (ret)
 367                 return ret;
 368
 369         for (i = 0; i < dev_priv->usec_timeout; i++) {
 370                 if (!(RADEON_READ(RADEON_RBBM_STATUS)
 371                       & RADEON_RBBM_ACTIVE)) {
 372                         radeon_do_pixcache_flush(dev_priv);
 373                         return 0;
 374                 }
 375                 DRM_UDELAY(1);
 376         }
 377         DRM_INFO("wait idle failed status : 0x%08X 0x%08X\n",
 378                  RADEON_READ(RADEON_RBBM_STATUS),
 379                  RADEON_READ(R300_VAP_CNTL_STATUS));
 380
 381 #if RADEON_FIFO_DEBUG
 382         DRM_ERROR("failed!\n");
 383         radeon_status(dev_priv);
 384 #endif
 385         return -EBUSY;
 386 }
 387
 388 static void radeon_init_pipes(drm_radeon_private_t * dev_priv)
 389 {
 390         uint32_t gb_tile_config, gb_pipe_sel = 0;
 391
 392         /* RS4xx/RS6xx/R4xx/R5xx */
 393         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
 394                 gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
 395                 dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
 396         } else {
 397                 /* R3xx */
 398                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 399                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350)) {
 400                         dev_priv->num_gb_pipes = 2;
 401                 } else {
 402                         /* R3Vxx */
 403                         dev_priv->num_gb_pipes = 1;
 404                 }
 405         }
 406         DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
 407
 408         gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
 409
 410         switch(dev_priv->num_gb_pipes) {
 411         case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
 412         case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
 413         case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
 414         default:
 415         case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
 416         }
 417
 418         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 419                 RADEON_WRITE_PLL(dev_priv, R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
 420                 RADEON_WRITE(R500_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
 421         }
 422         RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
 423         radeon_do_wait_for_idle(dev_priv);
 424         RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
 425         RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
 426                                                R300_DC_AUTOFLUSH_ENABLE |
 427                                                R300_DC_DC_DISABLE_IGNORE_PE));
 428
 429
 430 }
 431
 432 /* ================================================================
 433  * CP control, initialization
 434  */
 435
 436 /* Load the microcode for the CP */
 437 static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 438 {
 439         int i;
 440         DRM_DEBUG("\n");
 441
 442         radeon_do_wait_for_idle(dev_priv);
 443
 444         RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 445
 446         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
 447             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
 448             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
 449             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
 450             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
 451                 DRM_INFO("Loading R100 Microcode\n");
 452                 for (i = 0; i < 256; i++) {
 453                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 454                                      R100_cp_microcode[i][1]);
 455                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 456                                      R100_cp_microcode[i][0]);
 457                 }
 458         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
 459                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
 460                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
 461                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
 462                 DRM_INFO("Loading R200 Microcode\n");
 463                 for (i = 0; i < 256; i++) {
 464                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 465                                      R200_cp_microcode[i][1]);
 466                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 467                                      R200_cp_microcode[i][0]);
 468                 }
 469         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 470                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
 471                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
 472                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
 473                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 474                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 475                 DRM_INFO("Loading R300 Microcode\n");
 476                 for (i = 0; i < 256; i++) {
 477                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 478                                      R300_cp_microcode[i][1]);
 479                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 480                                      R300_cp_microcode[i][0]);
 481                 }
 482         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 483                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
 484                 DRM_INFO("Loading R400 Microcode\n");
 485                 for (i = 0; i < 256; i++) {
 486                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 487                                      R420_cp_microcode[i][1]);
 488                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 489                                      R420_cp_microcode[i][0]);
 490                 }
 491         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
 492                 DRM_INFO("Loading RS690 Microcode\n");
 493                 for (i = 0; i < 256; i++) {
 494                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 495                                      RS690_cp_microcode[i][1]);
 496                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 497                                      RS690_cp_microcode[i][0]);
 498                 }
 499         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
 500                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
 501                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
 502                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
 503                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
 504                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
 505                 DRM_INFO("Loading R500 Microcode\n");
 506                 for (i = 0; i < 256; i++) {
 507                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 508                                      R520_cp_microcode[i][1]);
 509                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 510                                      R520_cp_microcode[i][0]);
 511                 }
 512         }
 513 }
 514
 515 /* Flush any pending commands to the CP.  This should only be used just
 516  * prior to a wait for idle, as it informs the engine that the command
 517  * stream is ending.
 518  */
 519 static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
 520 {
 521         DRM_DEBUG("\n");
 522 #if 0
 523         u32 tmp;
 524
 525         tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
 526         RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
 527 #endif
 528 }
 529
 530 /* Wait for the CP to go idle.
 531  */
 532 int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
 533 {
 534         RING_LOCALS;
 535         DRM_DEBUG("\n");
 536
 537         BEGIN_RING(6);
 538
 539         RADEON_PURGE_CACHE();
 540         RADEON_PURGE_ZCACHE();
 541         RADEON_WAIT_UNTIL_IDLE();
 542
 543         ADVANCE_RING();
 544         COMMIT_RING();
 545
 546         return radeon_do_wait_for_idle(dev_priv);
 547 }
 548
 549 /* Start the Command Processor.
 550  */
 551 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
 552 {
 553         RING_LOCALS;
 554         DRM_DEBUG("\n");
 555
 556         radeon_do_wait_for_idle(dev_priv);
 557
 558         RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
 559
 560         dev_priv->cp_running = 1;
 561
 562         BEGIN_RING(8);
 563         /* isync can only be written through cp on r5xx write it here */
 564         OUT_RING(CP_PACKET0(RADEON_ISYNC_CNTL, 0));
 565         OUT_RING(RADEON_ISYNC_ANY2D_IDLE3D |
 566                  RADEON_ISYNC_ANY3D_IDLE2D |
 567                  RADEON_ISYNC_WAIT_IDLEGUI |
 568                  RADEON_ISYNC_CPSCRATCH_IDLEGUI);
 569         RADEON_PURGE_CACHE();
 570         RADEON_PURGE_ZCACHE();
 571         RADEON_WAIT_UNTIL_IDLE();
 572         ADVANCE_RING();
 573         COMMIT_RING();
 574
 575         dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 576 }
 577
 578 /* Reset the Command Processor.  This will not flush any pending
 579  * commands, so you must wait for the CP command stream to complete
 580  * before calling this routine.
 581  */
 582 static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
 583 {
 584         u32 cur_read_ptr;
 585         DRM_DEBUG("\n");
 586
 587         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 588         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 589         SET_RING_HEAD(dev_priv, cur_read_ptr);
 590         dev_priv->ring.tail = cur_read_ptr;
 591 }
 592
 593 /* Stop the Command Processor.  This will not flush any pending
 594  * commands, so you must flush the command stream and wait for the CP
 595  * to go idle before calling this routine.
 596  */
 597 static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
 598 {
 599         DRM_DEBUG("\n");
 600
 601         RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
 602
 603         dev_priv->cp_running = 0;
 604 }
 605
 606 /* Reset the engine.  This will stop the CP if it is running.
 607  */
 608 static int radeon_do_engine_reset(struct drm_device * dev)
 609 {
 610         drm_radeon_private_t *dev_priv = dev->dev_private;
 611         u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
 612         DRM_DEBUG("\n");
 613
 614         radeon_do_pixcache_flush(dev_priv);
 615
 616         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 617                 /* may need something similar for newer chips */
 618                 clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 619                 mclk_cntl = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
 620
 621                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, (mclk_cntl |
 622                                                               RADEON_FORCEON_MCLKA |
 623                                                               RADEON_FORCEON_MCLKB |
 624                                                               RADEON_FORCEON_YCLKA |
 625                                                               RADEON_FORCEON_YCLKB |
 626                                                               RADEON_FORCEON_MC |
 627                                                               RADEON_FORCEON_AIC));
 628         }
 629
 630         rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
 631
 632         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
 633                                               RADEON_SOFT_RESET_CP |
 634                                               RADEON_SOFT_RESET_HI |
 635                                               RADEON_SOFT_RESET_SE |
 636                                               RADEON_SOFT_RESET_RE |
 637                                               RADEON_SOFT_RESET_PP |
 638                                               RADEON_SOFT_RESET_E2 |
 639                                               RADEON_SOFT_RESET_RB));
 640         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 641         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
 642                                               ~(RADEON_SOFT_RESET_CP |
 643                                                 RADEON_SOFT_RESET_HI |
 644                                                 RADEON_SOFT_RESET_SE |
 645                                                 RADEON_SOFT_RESET_RE |
 646                                                 RADEON_SOFT_RESET_PP |
 647                                                 RADEON_SOFT_RESET_E2 |
 648                                                 RADEON_SOFT_RESET_RB)));
 649         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 650
 651         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 652                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, mclk_cntl);
 653                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
 654                 RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
 655         }
 656
 657         /* setup the raster pipes */
 658         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
 659             radeon_init_pipes(dev_priv);
 660
 661         /* Reset the CP ring */
 662         radeon_do_cp_reset(dev_priv);
 663
 664         /* The CP is no longer running after an engine reset */
 665         dev_priv->cp_running = 0;
 666
 667         /* Reset any pending vertex, indirect buffers */
 668         if (dev->dma)
 669                 radeon_freelist_reset(dev);
 670
 671         return 0;
 672 }
 673
 674 static void radeon_cp_init_ring_buffer(struct drm_device * dev,
 675                                        drm_radeon_private_t * dev_priv)
 676 {
 677         u32 ring_start, cur_read_ptr;
 678         u32 tmp;
 679
 680         /* Initialize the memory controller. With new memory map, the fb location
 681          * is not changed, it should have been properly initialized already. Part
 682          * of the problem is that the code below is bogus, assuming the GART is
 683          * always appended to the fb which is not necessarily the case
 684          */
 685         if (!dev_priv->new_memmap)
 686                 radeon_write_fb_location(dev_priv,
 687                                          ((dev_priv->gart_vm_start - 1) & 0xffff0000)
 688                                          | (dev_priv->fb_location >> 16));
 689
 690         if (dev_priv->mm.ring.bo) {
 691                 ring_start = dev_priv->mm.ring.bo->offset +
 692                         dev_priv->gart_vm_start;
 693         } else
 694 #if __OS_HAS_AGP
 695         if (dev_priv->flags & RADEON_IS_AGP) {
 696                 radeon_write_agp_base(dev_priv, dev->agp->base);
 697
 698                 radeon_write_agp_location(dev_priv,
 699                              (((dev_priv->gart_vm_start - 1 +
 700                                 dev_priv->gart_size) & 0xffff0000) |
 701                               (dev_priv->gart_vm_start >> 16)), 0);
 702
 703                 ring_start = (dev_priv->cp_ring->offset
 704                               - dev->agp->base
 705                               + dev_priv->gart_vm_start);
 706         } else
 707 #endif
 708                 ring_start = (dev_priv->cp_ring->offset
 709                               - (unsigned long)dev->sg->virtual
 710                               + dev_priv->gart_vm_start);
 711
 712         RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
 713
 714         /* Set the write pointer delay */
 715         RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
 716
 717         /* Initialize the ring buffer's read and write pointers */
 718         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 719         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 720         SET_RING_HEAD(dev_priv, cur_read_ptr);
 721         dev_priv->ring.tail = cur_read_ptr;
 722
 723
 724         if (dev_priv->mm.ring_read.bo) {
 725                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 726                              dev_priv->mm.ring_read.bo->offset +
 727                              dev_priv->gart_vm_start);
 728         } else
 729 #if __OS_HAS_AGP
 730         if (dev_priv->flags & RADEON_IS_AGP) {
 731                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 732                              dev_priv->ring_rptr->offset
 733                              - dev->agp->base + dev_priv->gart_vm_start);
 734         } else
 735 #endif
 736         {
 737                 struct drm_sg_mem *entry = dev->sg;
 738                 unsigned long tmp_ofs, page_ofs;
 739
 740                 tmp_ofs = dev_priv->ring_rptr->offset -
 741                                 (unsigned long)dev->sg->virtual;
 742                 page_ofs = tmp_ofs >> PAGE_SHIFT;
 743
 744                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR, entry->busaddr[page_ofs]);
 745                 DRM_DEBUG("ring rptr: offset=0x%08lx handle=0x%08lx\n",
 746                           (unsigned long)entry->busaddr[page_ofs],
 747                           entry->handle + tmp_ofs);
 748         }
 749
 750         /* Set ring buffer size */
 751 #ifdef __BIG_ENDIAN
 752         RADEON_WRITE(RADEON_CP_RB_CNTL,
 753                      RADEON_BUF_SWAP_32BIT |
 754                      (dev_priv->ring.fetch_size_l2ow << 18) |
 755                      (dev_priv->ring.rptr_update_l2qw << 8) |
 756                      dev_priv->ring.size_l2qw);
 757 #else
 758         RADEON_WRITE(RADEON_CP_RB_CNTL,
 759                      (dev_priv->ring.fetch_size_l2ow << 18) |
 760                      (dev_priv->ring.rptr_update_l2qw << 8) |
 761                      dev_priv->ring.size_l2qw);
 762 #endif
 763
 764         /* Start with assuming that writeback doesn't work */
 765         dev_priv->writeback_works = 0;
 766
 767         /* Initialize the scratch register pointer.  This will cause
 768          * the scratch register values to be written out to memory
 769          * whenever they are updated.
 770          *
 771          * We simply put this behind the ring read pointer, this works
 772          * with PCI GART as well as (whatever kind of) AGP GART
 773          */
 774         RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
 775                      + RADEON_SCRATCH_REG_OFFSET);
 776
 777         if (dev_priv->mm.ring_read.bo)
 778                 dev_priv->scratch = ((__volatile__ u32 *)
 779                                      dev_priv->mm.ring_read.kmap.virtual +
 780                                      (RADEON_SCRATCH_REG_OFFSET / sizeof(u32)));
 781         else
 782                 dev_priv->scratch = ((__volatile__ u32 *)
 783                                      dev_priv->ring_rptr->handle +
 784                                      (RADEON_SCRATCH_REG_OFFSET / sizeof(u32)));
 785
 786         RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
 787
 788         /* Turn on bus mastering */
 789         tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
 790         RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 791
 792         dev_priv->scratch[0] = 0;
 793         RADEON_WRITE(RADEON_LAST_FRAME_REG, 0);
 794
 795         dev_priv->scratch[1] = 0;
 796         RADEON_WRITE(RADEON_LAST_DISPATCH_REG, 0);
 797
 798         dev_priv->scratch[2] = 0;
 799         RADEON_WRITE(RADEON_LAST_CLEAR_REG, 0);
 800
 801         radeon_do_wait_for_idle(dev_priv);
 802
 803         /* Sync everything up */
 804         if (dev_priv->chip_family > CHIP_RV280) {
 805         RADEON_WRITE(RADEON_ISYNC_CNTL,
 806                      (RADEON_ISYNC_ANY2D_IDLE3D |
 807                       RADEON_ISYNC_ANY3D_IDLE2D |
 808                       RADEON_ISYNC_WAIT_IDLEGUI |
 809                       RADEON_ISYNC_CPSCRATCH_IDLEGUI));
 810         } else {
 811         RADEON_WRITE(RADEON_ISYNC_CNTL,
 812                      (RADEON_ISYNC_ANY2D_IDLE3D |
 813                       RADEON_ISYNC_ANY3D_IDLE2D |
 814                       RADEON_ISYNC_WAIT_IDLEGUI));
 815         }
 816 }
 817
 818 static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
 819 {
 820         u32 tmp;
 821         void *ring_read_ptr;
 822
 823         if (dev_priv->mm.ring_read.bo)
 824                 ring_read_ptr = dev_priv->mm.ring_read.kmap.virtual;
 825         else
 826                 ring_read_ptr = dev_priv->ring_rptr->handle;
 827
 828         /* Writeback doesn't seem to work everywhere, test it here and possibly
 829          * enable it if it appears to work
 830          */
 831         writel(0, ring_read_ptr + RADEON_SCRATCHOFF(1));
 832         RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
 833
 834         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 835                 if (readl(ring_read_ptr + RADEON_SCRATCHOFF(1)) ==
 836                     0xdeadbeef)
 837                         break;
 838                 DRM_UDELAY(1);
 839         }
 840
 841         if (tmp < dev_priv->usec_timeout) {
 842                 dev_priv->writeback_works = 1;
 843                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 844         } else {
 845                 dev_priv->writeback_works = 0;
 846                 DRM_INFO("writeback test failed\n");
 847         }
 848         if (radeon_no_wb == 1) {
 849                 dev_priv->writeback_works = 0;
 850                 DRM_INFO("writeback forced off\n");
 851         }
 852
 853         if (!dev_priv->writeback_works) {
 854                 /* Disable writeback to avoid unnecessary bus master transfers */
 855                 RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) | RADEON_RB_NO_UPDATE);
 856                 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
 857         }
 858 }
 859
 860 /* Enable or disable IGP GART on the chip */
 861 static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
 862 {
 863         u32 temp;
 864
 865         if (on) {
 866                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 867                          dev_priv->gart_vm_start,
 868                          (long)dev_priv->gart_info.bus_addr,
 869                          dev_priv->gart_size);
 870
 871                 temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
 872
 873                 if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
 874                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
 875                                                              RS690_BLOCK_GFX_D3_EN));
 876                 else
 877                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
 878
 879                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 880                                                                RS480_VA_SIZE_32MB));
 881
 882                 temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
 883                 IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
 884                                                         RS480_TLB_ENABLE |
 885                                                         RS480_GTW_LAC_EN |
 886                                                         RS480_1LEVEL_GART));
 887
 888                 temp = dev_priv->gart_info.bus_addr & 0xfffff000;
 889                 temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
 890                 IGP_WRITE_MCIND(RS480_GART_BASE, temp);
 891
 892                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
 893                 IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
 894                                                       RS480_REQ_TYPE_SNOOP_DIS));
 895
 896                 radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
 897
 898                 dev_priv->gart_size = 32*1024*1024;
 899                 temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) &
 900                         0xffff0000) | (dev_priv->gart_vm_start >> 16));
 901
 902                 radeon_write_agp_location(dev_priv, temp, 0);
 903
 904                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
 905                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 906                                                                RS480_VA_SIZE_32MB));
 907
 908                 do {
 909                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 910                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 911                                 break;
 912                         DRM_UDELAY(1);
 913                 } while(1);
 914
 915                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
 916                                 RS480_GART_CACHE_INVALIDATE);
 917
 918                 do {
 919                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 920                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 921                                 break;
 922                         DRM_UDELAY(1);
 923                 } while(1);
 924
 925                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
 926         } else {
 927                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
 928         }
 929 }
 930
 931 static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 932 {
 933         u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
 934         if (on) {
 935
 936                 DRM_DEBUG("programming pcie %08X %08lX %08X\n",
 937                           dev_priv->gart_vm_start,
 938                           (long)dev_priv->gart_info.bus_addr,
 939                           dev_priv->gart_size);
 940                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
 941                                   dev_priv->gart_vm_start);
 942                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
 943                                   dev_priv->gart_info.bus_addr);
 944                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
 945                                   dev_priv->gart_vm_start);
 946                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
 947                                   dev_priv->gart_vm_start +
 948                                   dev_priv->gart_size - 1);
 949
 950                 radeon_write_agp_location(dev_priv, 0xffffffc0, 0); /* ?? */
 951
 952                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 953                                   RADEON_PCIE_TX_GART_EN);
 954         } else {
 955                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 956                                   tmp & ~RADEON_PCIE_TX_GART_EN);
 957         }
 958 }
 959
 960 /* Enable or disable PCI GART on the chip */
 961 void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 962 {
 963         u32 tmp;
 964
 965         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 966             (dev_priv->flags & RADEON_IS_IGPGART)) {
 967                 radeon_set_igpgart(dev_priv, on);
 968                 return;
 969         }
 970
 971         if (dev_priv->flags & RADEON_IS_PCIE) {
 972                 radeon_set_pciegart(dev_priv, on);
 973                 return;
 974         }
 975
 976         tmp = RADEON_READ(RADEON_AIC_CNTL);
 977
 978         if (on) {
 979                 RADEON_WRITE(RADEON_AIC_CNTL,
 980                              tmp | RADEON_PCIGART_TRANSLATE_EN);
 981
 982                 /* set PCI GART page-table base address
 983                  */
 984                 RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
 985
 986                 /* set address range for PCI address translate
 987                  */
 988                 RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
 989                 RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
 990                              + dev_priv->gart_size - 1);
 991
 992                 /* Turn off AGP aperture -- is this required for PCI GART?
 993                  */
 994                 radeon_write_agp_location(dev_priv, 0xffffffc0, 0);
 995                 RADEON_WRITE(RADEON_AGP_COMMAND, 0);    /* clear AGP_COMMAND */
 996         } else {
 997                 RADEON_WRITE(RADEON_AIC_CNTL,
 998                              tmp & ~RADEON_PCIGART_TRANSLATE_EN);
 999         }
1000 }
1001
1002 static int radeon_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
1003                              struct drm_file *file_priv)
1004 {
1005         drm_radeon_private_t *dev_priv = dev->dev_private;
1006         struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
1007
1008         DRM_DEBUG("\n");
1009
1010         /* if we require new memory map but we don't have it fail */
1011         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
1012                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
1013                 radeon_do_cleanup_cp(dev);
1014                 return -EINVAL;
1015         }
1016
1017         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP))
1018         {
1019                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
1020                 dev_priv->flags &= ~RADEON_IS_AGP;
1021         }
1022         else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
1023                  && !init->is_pci)
1024         {
1025                 DRM_DEBUG("Restoring AGP flag\n");
1026                 dev_priv->flags |= RADEON_IS_AGP;
1027         }
1028
1029         if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
1030                 DRM_ERROR("PCI GART memory not allocated!\n");
1031                 radeon_do_cleanup_cp(dev);
1032                 return -EINVAL;
1033         }
1034
1035         dev_priv->usec_timeout = init->usec_timeout;
1036         if (dev_priv->usec_timeout < 1 ||
1037             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1038                 DRM_DEBUG("TIMEOUT problem!\n");
1039                 radeon_do_cleanup_cp(dev);
1040                 return -EINVAL;
1041         }
1042
1043         /* Enable vblank on CRTC1 for older X servers
1044          */
1045         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
1046
1047         dev_priv->do_boxes = 0;
1048         dev_priv->cp_mode = init->cp_mode;
1049
1050         /* We don't support anything other than bus-mastering ring mode,
1051          * but the ring can be in either AGP or PCI space for the ring
1052          * read pointer.
1053          */
1054         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
1055             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
1056                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
1057                 radeon_do_cleanup_cp(dev);
1058                 return -EINVAL;
1059         }
1060
1061         switch (init->fb_bpp) {
1062         case 16:
1063                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
1064                 break;
1065         case 32:
1066         default:
1067                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1068                 break;
1069         }
1070         dev_priv->front_offset = init->front_offset;
1071         dev_priv->front_pitch = init->front_pitch;
1072         dev_priv->back_offset = init->back_offset;
1073         dev_priv->back_pitch = init->back_pitch;
1074
1075         switch (init->depth_bpp) {
1076         case 16:
1077                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
1078                 break;
1079         case 32:
1080         default:
1081                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
1082                 break;
1083         }
1084         dev_priv->depth_offset = init->depth_offset;
1085         dev_priv->depth_pitch = init->depth_pitch;
1086
1087         /* Hardware state for depth clears.  Remove this if/when we no
1088          * longer clear the depth buffer with a 3D rectangle.  Hard-code
1089          * all values to prevent unwanted 3D state from slipping through
1090          * and screwing with the clear operation.
1091          */
1092         dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
1093                                            (dev_priv->color_fmt << 10) |
1094                                            (dev_priv->chip_family < CHIP_R200 ? RADEON_ZBLOCK16 : 0));
1095
1096         dev_priv->depth_clear.rb3d_zstencilcntl =
1097             (dev_priv->depth_fmt |
1098              RADEON_Z_TEST_ALWAYS |
1099              RADEON_STENCIL_TEST_ALWAYS |
1100              RADEON_STENCIL_S_FAIL_REPLACE |
1101              RADEON_STENCIL_ZPASS_REPLACE |
1102              RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
1103
1104         dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
1105                                          RADEON_BFACE_SOLID |
1106                                          RADEON_FFACE_SOLID |
1107                                          RADEON_FLAT_SHADE_VTX_LAST |
1108                                          RADEON_DIFFUSE_SHADE_FLAT |
1109                                          RADEON_ALPHA_SHADE_FLAT |
1110                                          RADEON_SPECULAR_SHADE_FLAT |
1111                                          RADEON_FOG_SHADE_FLAT |
1112                                          RADEON_VTX_PIX_CENTER_OGL |
1113                                          RADEON_ROUND_MODE_TRUNC |
1114                                          RADEON_ROUND_PREC_8TH_PIX);
1115
1116
1117         dev_priv->ring_offset = init->ring_offset;
1118         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
1119         dev_priv->buffers_offset = init->buffers_offset;
1120         dev_priv->gart_textures_offset = init->gart_textures_offset;
1121
1122         master_priv->sarea = drm_getsarea(dev);
1123         if (!master_priv->sarea) {
1124                 DRM_ERROR("could not find sarea!\n");
1125                 radeon_do_cleanup_cp(dev);
1126                 return -EINVAL;
1127         }
1128
1129         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
1130         if (!dev_priv->cp_ring) {
1131                 DRM_ERROR("could not find cp ring region!\n");
1132                 radeon_do_cleanup_cp(dev);
1133                 return -EINVAL;
1134         }
1135         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
1136         if (!dev_priv->ring_rptr) {
1137                 DRM_ERROR("could not find ring read pointer!\n");
1138                 radeon_do_cleanup_cp(dev);
1139                 return -EINVAL;
1140         }
1141         dev->agp_buffer_token = init->buffers_offset;
1142         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1143         if (!dev->agp_buffer_map) {
1144                 DRM_ERROR("could not find dma buffer region!\n");
1145                 radeon_do_cleanup_cp(dev);
1146                 return -EINVAL;
1147         }
1148
1149         if (init->gart_textures_offset) {
1150                 dev_priv->gart_textures =
1151                     drm_core_findmap(dev, init->gart_textures_offset);
1152                 if (!dev_priv->gart_textures) {
1153                         DRM_ERROR("could not find GART texture region!\n");
1154                         radeon_do_cleanup_cp(dev);
1155                         return -EINVAL;
1156                 }
1157         }
1158
1159 #if __OS_HAS_AGP
1160         if (dev_priv->flags & RADEON_IS_AGP) {
1161                 drm_core_ioremap(dev_priv->cp_ring, dev);
1162                 drm_core_ioremap(dev_priv->ring_rptr, dev);
1163                 drm_core_ioremap(dev->agp_buffer_map, dev);
1164                 if (!dev_priv->cp_ring->handle ||
1165                     !dev_priv->ring_rptr->handle ||
1166                     !dev->agp_buffer_map->handle) {
1167                         DRM_ERROR("could not find ioremap agp regions!\n");
1168                         radeon_do_cleanup_cp(dev);
1169                         return -EINVAL;
1170                 }
1171         } else
1172 #endif
1173         {
1174                 dev_priv->cp_ring->handle = (void *)dev_priv->cp_ring->offset;
1175                 dev_priv->ring_rptr->handle =
1176                     (void *)dev_priv->ring_rptr->offset;
1177                 dev->agp_buffer_map->handle =
1178                     (void *)dev->agp_buffer_map->offset;
1179
1180                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
1181                           dev_priv->cp_ring->handle);
1182                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
1183                           dev_priv->ring_rptr->handle);
1184                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
1185                           dev->agp_buffer_map->handle);
1186         }
1187
1188         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
1189         dev_priv->fb_size =
1190                 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
1191                 - dev_priv->fb_location;
1192
1193         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1194                                         ((dev_priv->front_offset
1195                                           + dev_priv->fb_location) >> 10));
1196
1197         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1198                                        ((dev_priv->back_offset
1199                                          + dev_priv->fb_location) >> 10));
1200
1201         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1202                                         ((dev_priv->depth_offset
1203                                           + dev_priv->fb_location) >> 10));
1204
1205         dev_priv->gart_size = init->gart_size;
1206
1207         /* New let's set the memory map ... */
1208         if (dev_priv->new_memmap) {
1209                 u32 base = 0;
1210
1211                 DRM_INFO("Setting GART location based on new memory map\n");
1212
1213                 /* If using AGP, try to locate the AGP aperture at the same
1214                  * location in the card and on the bus, though we have to
1215                  * align it down.
1216                  */
1217 #if __OS_HAS_AGP
1218                 if (dev_priv->flags & RADEON_IS_AGP) {
1219                         base = dev->agp->base;
1220                         /* Check if valid */
1221                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
1222                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
1223                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
1224                                          dev->agp->base);
1225                                 base = 0;
1226                         }
1227                 }
1228 #endif
1229                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
1230                 if (base == 0) {
1231                         base = dev_priv->fb_location + dev_priv->fb_size;
1232                         if (base < dev_priv->fb_location ||
1233                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
1234                                 base = dev_priv->fb_location
1235                                         - dev_priv->gart_size;
1236                 }
1237                 dev_priv->gart_vm_start = base & 0xffc00000u;
1238                 if (dev_priv->gart_vm_start != base)
1239                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
1240                                  base, dev_priv->gart_vm_start);
1241         } else {
1242                 DRM_INFO("Setting GART location based on old memory map\n");
1243                 dev_priv->gart_vm_start = dev_priv->fb_location +
1244                         RADEON_READ(RADEON_CONFIG_APER_SIZE);
1245         }
1246
1247 #if __OS_HAS_AGP
1248         if (dev_priv->flags & RADEON_IS_AGP)
1249                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1250                                                  - dev->agp->base
1251                                                  + dev_priv->gart_vm_start);
1252         else
1253 #endif
1254                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1255                                         - (unsigned long)dev->sg->virtual
1256                                         + dev_priv->gart_vm_start);
1257
1258         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
1259         DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
1260         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
1261                   dev_priv->gart_buffers_offset);
1262
1263         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
1264         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
1265                               + init->ring_size / sizeof(u32));
1266         dev_priv->ring.size = init->ring_size;
1267         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
1268
1269         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
1270         dev_priv->ring.rptr_update_l2qw = drm_order( /* init->rptr_update */ 4096 / 8);
1271
1272         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
1273         dev_priv->ring.fetch_size_l2ow = drm_order( /* init->fetch_size */ 32 / 16);
1274
1275         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1276
1277         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
1278
1279 #if __OS_HAS_AGP
1280         if (dev_priv->flags & RADEON_IS_AGP) {
1281                 /* Turn off PCI GART */
1282                 radeon_set_pcigart(dev_priv, 0);
1283         } else
1284 #endif
1285         {
1286                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
1287                 /* if we have an offset set from userspace */
1288                 if (dev_priv->pcigart_offset_set) {
1289                         /* if it came from userspace - remap it */
1290                         if (dev_priv->pcigart_offset_set == 1) {
1291                                 dev_priv->gart_info.bus_addr =
1292                                         dev_priv->pcigart_offset + dev_priv->fb_location;
1293                                 dev_priv->gart_info.mapping.offset =
1294                                         dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
1295                                 dev_priv->gart_info.mapping.size =
1296                                         dev_priv->gart_info.table_size;
1297
1298                                 /* this is done by the mm now */
1299                                 drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
1300                                 dev_priv->gart_info.addr =
1301                                         dev_priv->gart_info.mapping.handle;
1302
1303                                 memset(dev_priv->gart_info.addr, 0, dev_priv->gart_info.table_size);
1304                                 if (dev_priv->flags & RADEON_IS_PCIE)
1305                                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
1306                                 else
1307                                         dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1308                                 dev_priv->gart_info.gart_table_location =
1309                                         DRM_ATI_GART_FB;
1310
1311                                 DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
1312                                           dev_priv->gart_info.addr,
1313                                           dev_priv->pcigart_offset);
1314                         }
1315                 } else {
1316
1317                         if (dev_priv->flags & RADEON_IS_PCIE) {
1318                                 DRM_ERROR
1319                                     ("Cannot use PCI Express without GART in FB memory\n");
1320                                 radeon_do_cleanup_cp(dev);
1321                                 return -EINVAL;
1322                         }
1323                         if (dev_priv->flags & RADEON_IS_IGPGART)
1324                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
1325                         else
1326                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1327                         dev_priv->gart_info.gart_table_location =
1328                             DRM_ATI_GART_MAIN;
1329                         dev_priv->gart_info.addr = NULL;
1330                         dev_priv->gart_info.bus_addr = 0;
1331
1332                 }
1333
1334                 if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
1335                         DRM_ERROR("failed to init PCI GART!\n");
1336                         radeon_do_cleanup_cp(dev);
1337                         return -ENOMEM;
1338                 }
1339
1340                 /* Turn on PCI GART */
1341                 radeon_set_pcigart(dev_priv, 1);
1342         }
1343
1344         radeon_cp_load_microcode(dev_priv);
1345         radeon_cp_init_ring_buffer(dev, dev_priv);
1346
1347         dev_priv->last_buf = 0;
1348
1349         radeon_do_engine_reset(dev);
1350         radeon_test_writeback(dev_priv);
1351
1352         return 0;
1353 }
1354
1355 static int radeon_do_cleanup_cp(struct drm_device * dev)
1356 {
1357         drm_radeon_private_t *dev_priv = dev->dev_private;
1358         DRM_DEBUG("\n");
1359
1360         /* Make sure interrupts are disabled here because the uninstall ioctl
1361          * may not have been called from userspace and after dev_private
1362          * is freed, it's too late.
1363          */
1364         if (dev->irq_enabled)
1365                 drm_irq_uninstall(dev);
1366
1367 #if __OS_HAS_AGP
1368         if (dev_priv->flags & RADEON_IS_AGP) {
1369                 if (dev_priv->cp_ring != NULL) {
1370                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1371                         dev_priv->cp_ring = NULL;
1372                 }
1373                 if (dev_priv->ring_rptr != NULL) {
1374                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1375                         dev_priv->ring_rptr = NULL;
1376                 }
1377                 if (dev->agp_buffer_map != NULL) {
1378                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1379                         dev->agp_buffer_map = NULL;
1380                 }
1381         } else
1382 #endif
1383         {
1384
1385                 if (dev_priv->gart_info.bus_addr) {
1386                         /* Turn off PCI GART */
1387                         radeon_set_pcigart(dev_priv, 0);
1388                         drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info);
1389                 }
1390
1391                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
1392                 {
1393                         if (dev_priv->pcigart_offset_set == 1) {
1394                                 drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1395                                 dev_priv->gart_info.addr = NULL;
1396                                 dev_priv->pcigart_offset_set = 0;
1397                         }
1398                 }
1399         }
1400         /* only clear to the start of flags */
1401         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1402
1403         return 0;
1404 }
1405
1406 /* This code will reinit the Radeon CP hardware after a resume from disc.
1407  * AFAIK, it would be very difficult to pickle the state at suspend time, so
1408  * here we make sure that all Radeon hardware initialisation is re-done without
1409  * affecting running applications.
1410  *
1411  * Charl P. Botha <http://cpbotha.net>
1412  */
1413 static int radeon_do_resume_cp(struct drm_device * dev)
1414 {
1415         drm_radeon_private_t *dev_priv = dev->dev_private;
1416
1417         if (!dev_priv) {
1418                 DRM_ERROR("Called with no initialization\n");
1419                 return -EINVAL;
1420         }
1421
1422         DRM_DEBUG("Starting radeon_do_resume_cp()\n");
1423
1424 #if __OS_HAS_AGP
1425         if (dev_priv->flags & RADEON_IS_AGP) {
1426                 /* Turn off PCI GART */
1427                 radeon_set_pcigart(dev_priv, 0);
1428         } else
1429 #endif
1430         {
1431                 /* Turn on PCI GART */
1432                 radeon_set_pcigart(dev_priv, 1);
1433         }
1434
1435         radeon_cp_load_microcode(dev_priv);
1436         radeon_cp_init_ring_buffer(dev, dev_priv);
1437
1438         radeon_do_engine_reset(dev);
1439         radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
1440
1441         DRM_DEBUG("radeon_do_resume_cp() complete\n");
1442
1443         return 0;
1444 }
1445
1446 int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
1447 {
1448         drm_radeon_init_t *init = data;
1449
1450         /* on a modesetting driver ignore this stuff */
1451         if (drm_core_check_feature(dev, DRIVER_MODESET))
1452                 return 0;
1453
1454         LOCK_TEST_WITH_RETURN(dev, file_priv);
1455
1456         if (init->func == RADEON_INIT_R300_CP)
1457                 r300_init_reg_flags(dev);
1458
1459         switch (init->func) {
1460         case RADEON_INIT_CP:
1461         case RADEON_INIT_R200_CP:
1462         case RADEON_INIT_R300_CP:
1463                 return radeon_do_init_cp(dev, init, file_priv);
1464         case RADEON_CLEANUP_CP:
1465                 return radeon_do_cleanup_cp(dev);
1466         }
1467
1468         return -EINVAL;
1469 }
1470
1471 int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
1472 {
1473         drm_radeon_private_t *dev_priv = dev->dev_private;
1474         DRM_DEBUG("\n");
1475
1476         if (drm_core_check_feature(dev, DRIVER_MODESET))
1477                 return 0;
1478
1479         LOCK_TEST_WITH_RETURN(dev, file_priv);
1480
1481         if (dev_priv->cp_running) {
1482                 DRM_DEBUG("while CP running\n");
1483                 return 0;
1484         }
1485         if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
1486                 DRM_DEBUG("called with bogus CP mode (%d)\n",
1487                           dev_priv->cp_mode);
1488                 return 0;
1489         }
1490
1491         radeon_do_cp_start(dev_priv);
1492
1493         return 0;
1494 }
1495
1496 /* Stop the CP.  The engine must have been idled before calling this
1497  * routine.
1498  */
1499 int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
1500 {
1501         drm_radeon_private_t *dev_priv = dev->dev_private;
1502         drm_radeon_cp_stop_t *stop = data;
1503         int ret;
1504         DRM_DEBUG("\n");
1505
1506         if (drm_core_check_feature(dev, DRIVER_MODESET))
1507                 return 0;
1508
1509         LOCK_TEST_WITH_RETURN(dev, file_priv);
1510
1511         if (!dev_priv->cp_running)
1512                 return 0;
1513
1514         /* Flush any pending CP commands.  This ensures any outstanding
1515          * commands are exectuted by the engine before we turn it off.
1516          */
1517         if (stop->flush) {
1518                 radeon_do_cp_flush(dev_priv);
1519         }
1520
1521         /* If we fail to make the engine go idle, we return an error
1522          * code so that the DRM ioctl wrapper can try again.
1523          */
1524         if (stop->idle) {
1525                 ret = radeon_do_cp_idle(dev_priv);
1526                 if (ret)
1527                         return ret;
1528         }
1529
1530         /* Finally, we can turn off the CP.  If the engine isn't idle,
1531          * we will get some dropped triangles as they won't be fully
1532          * rendered before the CP is shut down.
1533          */
1534         radeon_do_cp_stop(dev_priv);
1535
1536         /* Reset the engine */
1537         radeon_do_engine_reset(dev);
1538
1539         return 0;
1540 }
1541
1542 void radeon_do_release(struct drm_device * dev)
1543 {
1544         drm_radeon_private_t *dev_priv = dev->dev_private;
1545         int i, ret;
1546
1547         if (drm_core_check_feature(dev, DRIVER_MODESET))
1548                 return;
1549
1550         if (dev_priv) {
1551                 if (dev_priv->cp_running) {
1552                         /* Stop the cp */
1553                         while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
1554                                 DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1555 #ifdef __linux__
1556                                 schedule();
1557 #else
1558 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
1559                                 mtx_sleep(&ret, &dev->dev_lock, PZERO, "rdnrel",
1560                                        1);
1561 #else
1562                                 tsleep(&ret, PZERO, "rdnrel", 1);
1563 #endif
1564 #endif
1565                         }
1566                         radeon_do_cp_stop(dev_priv);
1567                         radeon_do_engine_reset(dev);
1568                 }
1569
1570                 /* Disable *all* interrupts */
1571                 if (dev_priv->mmio)     /* remove this after permanent addmaps */
1572                         RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
1573
1574                 if (dev_priv->mmio) {   /* remove all surfaces */
1575                         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1576                                 RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
1577                                 RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
1578                                              16 * i, 0);
1579                                 RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
1580                                              16 * i, 0);
1581                         }
1582                 }
1583
1584                 /* Free memory heap structures */
1585                 radeon_mem_takedown(&(dev_priv->gart_heap));
1586                 radeon_mem_takedown(&(dev_priv->fb_heap));
1587
1588                 if (dev_priv->user_mm_enable) {
1589                         radeon_gem_mm_fini(dev);
1590                         dev_priv->user_mm_enable = false;
1591                 }
1592
1593                 /* deallocate kernel resources */
1594                 radeon_do_cleanup_cp(dev);
1595         }
1596 }
1597
1598 /* Just reset the CP ring.  Called as part of an X Server engine reset.
1599  */
1600 int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1601 {
1602         drm_radeon_private_t *dev_priv = dev->dev_private;
1603         DRM_DEBUG("\n");
1604
1605         if (drm_core_check_feature(dev, DRIVER_MODESET))
1606                 return 0;
1607
1608         LOCK_TEST_WITH_RETURN(dev, file_priv);
1609
1610         if (!dev_priv) {
1611                 DRM_DEBUG("called before init done\n");
1612                 return -EINVAL;
1613         }
1614
1615         radeon_do_cp_reset(dev_priv);
1616
1617         /* The CP is no longer running after an engine reset */
1618         dev_priv->cp_running = 0;
1619
1620         return 0;
1621 }
1622
1623 int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
1624 {
1625         drm_radeon_private_t *dev_priv = dev->dev_private;
1626         DRM_DEBUG("\n");
1627
1628
1629         if (!drm_core_check_feature(dev, DRIVER_MODESET))
1630                 LOCK_TEST_WITH_RETURN(dev, file_priv);
1631
1632         return radeon_do_cp_idle(dev_priv);
1633 }
1634
1635 /* Added by Charl P. Botha to call radeon_do_resume_cp().
1636  */
1637 int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
1638 {
1639
1640         if (drm_core_check_feature(dev, DRIVER_MODESET))
1641                 return 0;
1642
1643         return radeon_do_resume_cp(dev);
1644 }
1645
1646 int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1647 {
1648         DRM_DEBUG("\n");
1649
1650         if (drm_core_check_feature(dev, DRIVER_MODESET))
1651                 return 0;
1652
1653         LOCK_TEST_WITH_RETURN(dev, file_priv);
1654
1655         return radeon_do_engine_reset(dev);
1656 }
1657
1658 /* ================================================================
1659  * Fullscreen mode
1660  */
1661
1662 /* KW: Deprecated to say the least:
1663  */
1664 int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
1665 {
1666         return 0;
1667 }
1668
1669 /* ================================================================
1670  * Freelist management
1671  */
1672
1673 /* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
1674  *   bufs until freelist code is used.  Note this hides a problem with
1675  *   the scratch register * (used to keep track of last buffer
1676  *   completed) being written to before * the last buffer has actually
1677  *   completed rendering.
1678  *
1679  * KW:  It's also a good way to find free buffers quickly.
1680  *
1681  * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
1682  * sleep.  However, bugs in older versions of radeon_accel.c mean that
1683  * we essentially have to do this, else old clients will break.
1684  *
1685  * However, it does leave open a potential deadlock where all the
1686  * buffers are held by other clients, which can't release them because
1687  * they can't get the lock.
1688  */
1689
1690 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1691 {
1692         struct drm_device_dma *dma = dev->dma;
1693         drm_radeon_private_t *dev_priv = dev->dev_private;
1694         drm_radeon_buf_priv_t *buf_priv;
1695         struct drm_buf *buf;
1696         int i, t;
1697         int start;
1698
1699         if (++dev_priv->last_buf >= dma->buf_count)
1700                 dev_priv->last_buf = 0;
1701
1702         start = dev_priv->last_buf;
1703
1704         for (t = 0; t < dev_priv->usec_timeout; t++) {
1705                 u32 done_age = GET_SCRATCH(1);
1706                 DRM_DEBUG("done_age = %d\n", done_age);
1707                 for (i = start; i < dma->buf_count; i++) {
1708                         buf = dma->buflist[i];
1709                         buf_priv = buf->dev_private;
1710                         if (buf->file_priv == NULL || (buf->pending &&
1711                                                        buf_priv->age <=
1712                                                        done_age)) {
1713                                 dev_priv->stats.requested_bufs++;
1714                                 buf->pending = 0;
1715                                 return buf;
1716                         }
1717                         start = 0;
1718                 }
1719
1720                 if (t) {
1721                         DRM_UDELAY(1);
1722                         dev_priv->stats.freelist_loops++;
1723                 }
1724         }
1725
1726         DRM_DEBUG("returning NULL!\n");
1727         return NULL;
1728 }
1729
1730 #if 0
1731 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1732 {
1733         struct drm_device_dma *dma = dev->dma;
1734         drm_radeon_private_t *dev_priv = dev->dev_private;
1735         drm_radeon_buf_priv_t *buf_priv;
1736         struct drm_buf *buf;
1737         int i, t;
1738         int start;
1739         u32 done_age = DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1));
1740
1741         if (++dev_priv->last_buf >= dma->buf_count)
1742                 dev_priv->last_buf = 0;
1743
1744         start = dev_priv->last_buf;
1745         dev_priv->stats.freelist_loops++;
1746
1747         for (t = 0; t < 2; t++) {
1748                 for (i = start; i < dma->buf_count; i++) {
1749                         buf = dma->buflist[i];
1750                         buf_priv = buf->dev_private;
1751                         if (buf->file_priv == 0 || (buf->pending &&
1752                                                     buf_priv->age <=
1753                                                     done_age)) {
1754                                 dev_priv->stats.requested_bufs++;
1755                                 buf->pending = 0;
1756                                 return buf;
1757                         }
1758                 }
1759                 start = 0;
1760         }
1761
1762         return NULL;
1763 }
1764 #endif
1765
1766 void radeon_freelist_reset(struct drm_device * dev)
1767 {
1768         struct drm_device_dma *dma = dev->dma;
1769         drm_radeon_private_t *dev_priv = dev->dev_private;
1770         int i;
1771
1772         dev_priv->last_buf = 0;
1773         for (i = 0; i < dma->buf_count; i++) {
1774                 struct drm_buf *buf = dma->buflist[i];
1775                 drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1776                 buf_priv->age = 0;
1777         }
1778 }
1779
1780 /* ================================================================
1781  * CP command submission
1782  */
1783
1784 int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
1785 {
1786         drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
1787         int i;
1788         u32 last_head = GET_RING_HEAD(dev_priv);
1789
1790         for (i = 0; i < dev_priv->usec_timeout; i++) {
1791                 u32 head = GET_RING_HEAD(dev_priv);
1792
1793                 ring->space = (head - ring->tail) * sizeof(u32);
1794                 if (ring->space <= 0)
1795                         ring->space += ring->size;
1796                 if (ring->space > n)
1797                         return 0;
1798
1799                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
1800
1801                 if (head != last_head)
1802                         i = 0;
1803                 last_head = head;
1804
1805                 DRM_UDELAY(1);
1806         }
1807
1808         /* FIXME: This return value is ignored in the BEGIN_RING macro! */
1809 #if RADEON_FIFO_DEBUG
1810         radeon_status(dev_priv);
1811         DRM_ERROR("failed!\n");
1812 #endif
1813         return -EBUSY;
1814 }
1815
1816 static int radeon_cp_get_buffers(struct drm_device *dev,
1817                                  struct drm_file *file_priv,
1818                                  struct drm_dma * d)
1819 {
1820         int i;
1821         struct drm_buf *buf;
1822
1823         for (i = d->granted_count; i < d->request_count; i++) {
1824                 buf = radeon_freelist_get(dev);
1825                 if (!buf)
1826                         return -EBUSY;  /* NOTE: broken client */
1827
1828                 buf->file_priv = file_priv;
1829
1830                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
1831                                      sizeof(buf->idx)))
1832                         return -EFAULT;
1833                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
1834                                      sizeof(buf->total)))
1835                         return -EFAULT;
1836
1837                 d->granted_count++;
1838         }
1839         return 0;
1840 }
1841
1842 int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
1843 {
1844         struct drm_device_dma *dma = dev->dma;
1845         int ret = 0;
1846         struct drm_dma *d = data;
1847
1848         LOCK_TEST_WITH_RETURN(dev, file_priv);
1849
1850         /* Please don't send us buffers.
1851          */
1852         if (d->send_count != 0) {
1853                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
1854                           DRM_CURRENTPID, d->send_count);
1855                 return -EINVAL;
1856         }
1857
1858         /* We'll send you buffers.
1859          */
1860         if (d->request_count < 0 || d->request_count > dma->buf_count) {
1861                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
1862                           DRM_CURRENTPID, d->request_count, dma->buf_count);
1863                 return -EINVAL;
1864         }
1865
1866         d->granted_count = 0;
1867
1868         if (d->request_count) {
1869                 ret = radeon_cp_get_buffers(dev, file_priv, d);
1870         }
1871
1872         return ret;
1873 }
1874
1875 static void radeon_get_vram_type(struct drm_device *dev)
1876 {
1877         struct drm_radeon_private *dev_priv = dev->dev_private;
1878         uint32_t tmp;
1879
1880         if (dev_priv->flags & RADEON_IS_IGP || (dev_priv->chip_family >= CHIP_R300))
1881                 dev_priv->is_ddr = true;
1882         else if (RADEON_READ(RADEON_MEM_SDRAM_MODE_REG) & RADEON_MEM_CFG_TYPE_DDR)
1883                 dev_priv->is_ddr = true;
1884         else
1885                 dev_priv->is_ddr = false;
1886
1887         if ((dev_priv->chip_family >= CHIP_R600) &&
1888             (dev_priv->chip_family <= CHIP_RV635)) {
1889                 int chansize;
1890
1891                 tmp = RADEON_READ(R600_RAMCFG);
1892                 if (tmp & R600_CHANSIZE_OVERRIDE)
1893                         chansize = 16;
1894                 else if (tmp & R600_CHANSIZE)
1895                         chansize = 64;
1896                 else
1897                         chansize = 32;
1898
1899                 if (dev_priv->chip_family == CHIP_R600)
1900                         dev_priv->ram_width = 8 * chansize;
1901                 else if (dev_priv->chip_family == CHIP_RV670)
1902                         dev_priv->ram_width = 4 * chansize;
1903                 else if ((dev_priv->chip_family == CHIP_RV610) ||
1904                          (dev_priv->chip_family == CHIP_RV620))
1905                         dev_priv->ram_width = chansize;
1906                 else if ((dev_priv->chip_family == CHIP_RV630) ||
1907                          (dev_priv->chip_family == CHIP_RV635))
1908                         dev_priv->ram_width = 2 * chansize;
1909         } else if (dev_priv->chip_family == CHIP_RV515) {
1910                 tmp = radeon_read_mc_reg(dev_priv, RV515_MC_CNTL);
1911                 tmp &= RV515_MEM_NUM_CHANNELS_MASK;
1912                 switch (tmp) {
1913                 case 0: dev_priv->ram_width = 64; break;
1914                 case 1: dev_priv->ram_width = 128; break;
1915                 default: dev_priv->ram_width = 128; break;
1916                 }
1917         } else if ((dev_priv->chip_family >= CHIP_R520) &&
1918                    (dev_priv->chip_family <= CHIP_RV570)) {
1919                 tmp = radeon_read_mc_reg(dev_priv, R520_MC_CNTL0);
1920                 switch ((tmp & R520_MEM_NUM_CHANNELS_MASK) >> R520_MEM_NUM_CHANNELS_SHIFT) {
1921                 case 0: dev_priv->ram_width = 32; break;
1922                 case 1: dev_priv->ram_width = 64; break;
1923                 case 2: dev_priv->ram_width = 128; break;
1924                 case 3: dev_priv->ram_width = 256; break;
1925                 default: dev_priv->ram_width = 128; break;
1926                 }
1927         } else if ((dev_priv->chip_family == CHIP_RV100) ||
1928                    (dev_priv->chip_family == CHIP_RS100) ||
1929                    (dev_priv->chip_family == CHIP_RS200)) {
1930                 tmp = RADEON_READ(RADEON_MEM_CNTL);
1931                 if (tmp & RV100_HALF_MODE)
1932                         dev_priv->ram_width = 32;
1933                 else
1934                         dev_priv->ram_width = 64;
1935
1936                 if (dev_priv->flags & RADEON_SINGLE_CRTC) {
1937                         dev_priv->ram_width /= 4;
1938                         dev_priv->is_ddr = true;
1939                 }
1940         } else if (dev_priv->chip_family <= CHIP_RV280) {
1941                 tmp = RADEON_READ(RADEON_MEM_CNTL);
1942                 if (tmp & RADEON_MEM_NUM_CHANNELS_MASK)
1943                         dev_priv->ram_width = 128;
1944                 else
1945                         dev_priv->ram_width = 64;
1946         } else {
1947                 /* newer IGPs */
1948                 dev_priv->ram_width = 128;
1949         }
1950         DRM_DEBUG("RAM width %d bits %cDR\n", dev_priv->ram_width, dev_priv->is_ddr ? 'D' : 'S');
1951 }
1952
1953 static void radeon_force_some_clocks(struct drm_device *dev)
1954 {
1955         struct drm_radeon_private *dev_priv = dev->dev_private;
1956         uint32_t tmp;
1957
1958         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
1959         tmp |= RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_VIP;
1960         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
1961 }
1962
1963 static void radeon_set_dynamic_clock(struct drm_device *dev, int mode)
1964 {
1965         struct drm_radeon_private *dev_priv = dev->dev_private;
1966         uint32_t tmp;
1967
1968         switch(mode) {
1969         case 0:
1970                 if (dev_priv->flags & RADEON_SINGLE_CRTC) {
1971                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
1972                         tmp |= (RADEON_SCLK_FORCE_CP   | RADEON_SCLK_FORCE_HDP |
1973                                 RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP |
1974                                 RADEON_SCLK_FORCE_E2   | RADEON_SCLK_FORCE_SE  |
1975                                 RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
1976                                 RADEON_SCLK_FORCE_RE   | RADEON_SCLK_FORCE_PB  |
1977                                 RADEON_SCLK_FORCE_TAM  | RADEON_SCLK_FORCE_TDM |
1978                                 RADEON_SCLK_FORCE_RB);
1979                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
1980                 } else if (dev_priv->chip_family == CHIP_RV350) {
1981                         /* for RV350/M10, no delays are required. */
1982                         tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
1983                         tmp |= (R300_SCLK_FORCE_TCL |
1984                                 R300_SCLK_FORCE_GA |
1985                                 R300_SCLK_FORCE_CBA);
1986                         RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
1987
1988                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
1989                         tmp &= ~(RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP      |
1990                                  RADEON_SCLK_FORCE_HDP   | RADEON_SCLK_FORCE_DISP1   |
1991                                  RADEON_SCLK_FORCE_TOP   | RADEON_SCLK_FORCE_E2      |
1992                                  R300_SCLK_FORCE_VAP     | RADEON_SCLK_FORCE_IDCT    |
1993                                  RADEON_SCLK_FORCE_VIP   | R300_SCLK_FORCE_SR        |
1994                                  R300_SCLK_FORCE_PX      | R300_SCLK_FORCE_TX        |
1995                                  R300_SCLK_FORCE_US      | RADEON_SCLK_FORCE_TV_SCLK |
1996                                  R300_SCLK_FORCE_SU      | RADEON_SCLK_FORCE_OV0);
1997                         tmp |=  RADEON_DYN_STOP_LAT_MASK;
1998                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
1999
2000                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
2001                         tmp &= ~RADEON_SCLK_MORE_FORCEON;
2002                         tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
2003                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
2004
2005                         tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
2006                         tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
2007                                 RADEON_PIXCLK_DAC_ALWAYS_ONb);
2008                         RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
2009
2010                         tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
2011                         tmp |= (RADEON_PIX2CLK_ALWAYS_ONb         |
2012                                 RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
2013                                 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
2014                                 R300_DVOCLK_ALWAYS_ONb            |
2015                                 RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
2016                                 RADEON_PIXCLK_GV_ALWAYS_ONb       |
2017                                 R300_PIXCLK_DVO_ALWAYS_ONb        |
2018                                 RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
2019                                 RADEON_PIXCLK_TMDS_ALWAYS_ONb     |
2020                                 R300_PIXCLK_TRANS_ALWAYS_ONb      |
2021                                 R300_PIXCLK_TVO_ALWAYS_ONb        |
2022                                 R300_P2G2CLK_ALWAYS_ONb           |
2023                                 R300_P2G2CLK_ALWAYS_ONb);
2024                         RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
2025                 } else {
2026                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2027                         tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
2028                         tmp |= RADEON_SCLK_FORCE_SE;
2029
2030                         if ( dev_priv->flags & RADEON_SINGLE_CRTC ) {
2031                                 tmp |= ( RADEON_SCLK_FORCE_RB    |
2032                                          RADEON_SCLK_FORCE_TDM   |
2033                                          RADEON_SCLK_FORCE_TAM   |
2034                                          RADEON_SCLK_FORCE_PB    |
2035                                          RADEON_SCLK_FORCE_RE    |
2036                                          RADEON_SCLK_FORCE_VIP   |
2037                                          RADEON_SCLK_FORCE_IDCT  |
2038                                          RADEON_SCLK_FORCE_TOP   |
2039                                          RADEON_SCLK_FORCE_DISP1 |
2040                                          RADEON_SCLK_FORCE_DISP2 |
2041                                          RADEON_SCLK_FORCE_HDP    );
2042                         } else if ((dev_priv->chip_family == CHIP_R300) ||
2043                                    (dev_priv->chip_family == CHIP_R350)) {
2044                                 tmp |= ( RADEON_SCLK_FORCE_HDP   |
2045                                          RADEON_SCLK_FORCE_DISP1 |
2046                                          RADEON_SCLK_FORCE_DISP2 |
2047                                          RADEON_SCLK_FORCE_TOP   |
2048                                          RADEON_SCLK_FORCE_IDCT  |
2049                                          RADEON_SCLK_FORCE_VIP);
2050                         }
2051
2052                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2053
2054                         udelay(16000);
2055
2056                         if ((dev_priv->chip_family == CHIP_R300) ||
2057                             (dev_priv->chip_family == CHIP_R350)) {
2058                                 tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
2059                                 tmp |= ( R300_SCLK_FORCE_TCL |
2060                                          R300_SCLK_FORCE_GA  |
2061                                          R300_SCLK_FORCE_CBA);
2062                                 RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
2063                                 udelay(16000);
2064                         }
2065
2066                         if (dev_priv->flags & RADEON_IS_IGP) {
2067                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
2068                                 tmp &= ~(RADEON_FORCEON_MCLKA |
2069                                          RADEON_FORCEON_YCLKA);
2070                                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, tmp);
2071                                 udelay(16000);
2072                         }
2073
2074                         if ((dev_priv->chip_family == CHIP_RV200) ||
2075                             (dev_priv->chip_family == CHIP_RV250) ||
2076                             (dev_priv->chip_family == CHIP_RV280)) {
2077                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
2078                                 tmp |= RADEON_SCLK_MORE_FORCEON;
2079                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
2080                                 udelay(16000);
2081                         }
2082
2083                         tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
2084                         tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb         |
2085                                  RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
2086                                  RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
2087                                  RADEON_PIXCLK_GV_ALWAYS_ONb       |
2088                                  RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
2089                                  RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
2090                                  RADEON_PIXCLK_TMDS_ALWAYS_ONb);
2091
2092                         RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
2093                         udelay(16000);
2094
2095                         tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
2096                         tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb  |
2097                                  RADEON_PIXCLK_DAC_ALWAYS_ONb);
2098                         RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
2099                 }
2100                 DRM_DEBUG("Dynamic Clock Scaling Disabled\n");
2101                 break;
2102         case 1:
2103                 if (dev_priv->flags & RADEON_SINGLE_CRTC) {
2104                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2105                         if ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) >
2106                             RADEON_CFG_ATI_REV_A13) {
2107                                 tmp &= ~(RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_RB);
2108                         }
2109                         tmp &= ~(RADEON_SCLK_FORCE_HDP  | RADEON_SCLK_FORCE_DISP1 |
2110                                  RADEON_SCLK_FORCE_TOP  | RADEON_SCLK_FORCE_SE   |
2111                                  RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE   |
2112                                  RADEON_SCLK_FORCE_PB   | RADEON_SCLK_FORCE_TAM  |
2113                                  RADEON_SCLK_FORCE_TDM);
2114                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2115                 } else if ((dev_priv->chip_family == CHIP_R300) ||
2116                            (dev_priv->chip_family == CHIP_R350) ||
2117                            (dev_priv->chip_family == CHIP_RV350)) {
2118                         if (dev_priv->chip_family == CHIP_RV350) {
2119                                 tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
2120                                 tmp &= ~(R300_SCLK_FORCE_TCL |
2121                                          R300_SCLK_FORCE_GA  |
2122                                          R300_SCLK_FORCE_CBA);
2123                                 tmp |=  (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
2124                                          R300_SCLK_GA_MAX_DYN_STOP_LAT  |
2125                                          R300_SCLK_CBA_MAX_DYN_STOP_LAT);
2126                                 RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
2127
2128                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2129                                 tmp &= ~(RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP      |
2130                                          RADEON_SCLK_FORCE_HDP   | RADEON_SCLK_FORCE_DISP1   |
2131                                          RADEON_SCLK_FORCE_TOP   | RADEON_SCLK_FORCE_E2      |
2132                                          R300_SCLK_FORCE_VAP     | RADEON_SCLK_FORCE_IDCT    |
2133                                          RADEON_SCLK_FORCE_VIP   | R300_SCLK_FORCE_SR        |
2134                                          R300_SCLK_FORCE_PX      | R300_SCLK_FORCE_TX        |
2135                                          R300_SCLK_FORCE_US      | RADEON_SCLK_FORCE_TV_SCLK |
2136                                          R300_SCLK_FORCE_SU      | RADEON_SCLK_FORCE_OV0);
2137                                 tmp |=  RADEON_DYN_STOP_LAT_MASK;
2138                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2139
2140                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
2141                                 tmp &= ~RADEON_SCLK_MORE_FORCEON;
2142                                 tmp |=  RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
2143                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
2144
2145                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
2146                                 tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
2147                                         RADEON_PIXCLK_DAC_ALWAYS_ONb);
2148                                 RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
2149
2150                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
2151                                 tmp |= (RADEON_PIX2CLK_ALWAYS_ONb         |
2152                                         RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
2153                                         RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
2154                                         R300_DVOCLK_ALWAYS_ONb            |
2155                                         RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
2156                                         RADEON_PIXCLK_GV_ALWAYS_ONb       |
2157                                         R300_PIXCLK_DVO_ALWAYS_ONb        |
2158                                         RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
2159                                         RADEON_PIXCLK_TMDS_ALWAYS_ONb     |
2160                                         R300_PIXCLK_TRANS_ALWAYS_ONb      |
2161                                         R300_PIXCLK_TVO_ALWAYS_ONb        |
2162                                         R300_P2G2CLK_ALWAYS_ONb           |
2163                                         R300_P2G2CLK_ALWAYS_ONb);
2164                                 RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
2165
2166                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_MISC);
2167                                 tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
2168                                         RADEON_IO_MCLK_DYN_ENABLE);
2169                                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_MISC, tmp);
2170
2171                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
2172                                 tmp |= (RADEON_FORCEON_MCLKA |
2173                                         RADEON_FORCEON_MCLKB);
2174
2175                                 tmp &= ~(RADEON_FORCEON_YCLKA  |
2176                                          RADEON_FORCEON_YCLKB  |
2177                                          RADEON_FORCEON_MC);
2178
2179                                 /* Some releases of vbios have set DISABLE_MC_MCLKA
2180                                    and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
2181                                    bits will cause H/W hang when reading video memory with dynamic clocking
2182                                    enabled. */
2183                                 if ((tmp & R300_DISABLE_MC_MCLKA) &&
2184                                     (tmp & R300_DISABLE_MC_MCLKB)) {
2185                                         /* If both bits are set, then check the active channels */
2186                                         tmp = RADEON_READ_PLL(dev_priv, RADEON_MCLK_CNTL);
2187                                         if (dev_priv->ram_width == 64) {
2188                                                 if (RADEON_READ(RADEON_MEM_CNTL) & R300_MEM_USE_CD_CH_ONLY)
2189                                                         tmp &= ~R300_DISABLE_MC_MCLKB;
2190                                                 else
2191                                                         tmp &= ~R300_DISABLE_MC_MCLKA;
2192                                         } else {
2193                                                 tmp &= ~(R300_DISABLE_MC_MCLKA |
2194                                                          R300_DISABLE_MC_MCLKB);
2195                                         }
2196                                 }
2197
2198                                 RADEON_WRITE_PLL(dev_priv, RADEON_MCLK_CNTL, tmp);
2199                         } else {
2200                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2201                                 tmp &= ~(R300_SCLK_FORCE_VAP);
2202                                 tmp |= RADEON_SCLK_FORCE_CP;
2203                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2204                                 udelay(15000);
2205
2206                                 tmp = RADEON_READ_PLL(dev_priv, R300_SCLK_CNTL2);
2207                                 tmp &= ~(R300_SCLK_FORCE_TCL |
2208                                          R300_SCLK_FORCE_GA  |
2209                                          R300_SCLK_FORCE_CBA);
2210                                 RADEON_WRITE_PLL(dev_priv, R300_SCLK_CNTL2, tmp);
2211                         }
2212                 } else {
2213                         tmp = RADEON_READ_PLL(dev_priv, RADEON_CLK_PWRMGT_CNTL);
2214                         tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK     |
2215                                  RADEON_DISP_DYN_STOP_LAT_MASK   |
2216                                  RADEON_DYN_STOP_MODE_MASK);
2217
2218                         tmp |= (RADEON_ENGIN_DYNCLK_MODE |
2219                                 (0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
2220                         RADEON_WRITE_PLL(dev_priv, RADEON_CLK_PWRMGT_CNTL, tmp);
2221                         udelay(15000);
2222
2223                         tmp = RADEON_READ_PLL(dev_priv, RADEON_CLK_PIN_CNTL);
2224                         tmp |= RADEON_SCLK_DYN_START_CNTL;
2225                         RADEON_WRITE_PLL(dev_priv, RADEON_CLK_PIN_CNTL, tmp);
2226                         udelay(15000);
2227
2228                         /* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
2229                            to lockup randomly, leave them as set by BIOS.
2230                         */
2231                         tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_CNTL);
2232                         /*tmp &= RADEON_SCLK_SRC_SEL_MASK;*/
2233                         tmp &= ~RADEON_SCLK_FORCEON_MASK;
2234
2235                         /*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300*/
2236                         if (((dev_priv->chip_family == CHIP_RV250) &&
2237                              ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
2238                               RADEON_CFG_ATI_REV_A13)) ||
2239                             ((dev_priv->chip_family == CHIP_RV100) &&
2240                              ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <=
2241                               RADEON_CFG_ATI_REV_A13))){
2242                                 tmp |= RADEON_SCLK_FORCE_CP;
2243                                 tmp |= RADEON_SCLK_FORCE_VIP;
2244                         }
2245
2246                         RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_CNTL, tmp);
2247
2248                         if ((dev_priv->chip_family == CHIP_RV200) ||
2249                             (dev_priv->chip_family == CHIP_RV250) ||
2250                             (dev_priv->chip_family == CHIP_RV280)) {
2251                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_SCLK_MORE_CNTL);
2252                                 tmp &= ~RADEON_SCLK_MORE_FORCEON;
2253
2254                                 /* RV200::A11 A12 RV250::A11 A12 */
2255                                 if (((dev_priv->chip_family == CHIP_RV200) ||
2256                                      (dev_priv->chip_family == CHIP_RV250)) &&
2257                                     ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
2258                                      RADEON_CFG_ATI_REV_A13)) {
2259                                         tmp |= RADEON_SCLK_MORE_FORCEON;
2260                                 }
2261                                 RADEON_WRITE_PLL(dev_priv, RADEON_SCLK_MORE_CNTL, tmp);
2262                                 udelay(15000);
2263                         }
2264
2265                         /* RV200::A11 A12, RV250::A11 A12 */
2266                         if (((dev_priv->chip_family == CHIP_RV200) ||
2267                              (dev_priv->chip_family == CHIP_RV250)) &&
2268                             ((RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) <
2269                              RADEON_CFG_ATI_REV_A13)) {
2270                                 tmp = RADEON_READ_PLL(dev_priv, RADEON_PLL_PWRMGT_CNTL);
2271                                 tmp |= RADEON_TCL_BYPASS_DISABLE;
2272                                 RADEON_WRITE_PLL(dev_priv, RADEON_PLL_PWRMGT_CNTL, tmp);
2273                         }
2274                         udelay(15000);
2275
2276                         /*enable dynamic mode for display clocks (PIXCLK and PIX2CLK)*/
2277                         tmp = RADEON_READ_PLL(dev_priv, RADEON_PIXCLKS_CNTL);
2278                         tmp |=  (RADEON_PIX2CLK_ALWAYS_ONb         |
2279                                  RADEON_PIX2CLK_DAC_ALWAYS_ONb     |
2280                                  RADEON_PIXCLK_BLEND_ALWAYS_ONb    |
2281                                  RADEON_PIXCLK_GV_ALWAYS_ONb       |
2282                                  RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
2283                                  RADEON_PIXCLK_LVDS_ALWAYS_ONb     |
2284                                  RADEON_PIXCLK_TMDS_ALWAYS_ONb);
2285
2286                         RADEON_WRITE_PLL(dev_priv, RADEON_PIXCLKS_CNTL, tmp);
2287                         udelay(15000);
2288
2289                         tmp = RADEON_READ_PLL(dev_priv, RADEON_VCLK_ECP_CNTL);
2290                         tmp |= (RADEON_PIXCLK_ALWAYS_ONb  |
2291                                 RADEON_PIXCLK_DAC_ALWAYS_ONb);
2292
2293                         RADEON_WRITE_PLL(dev_priv, RADEON_VCLK_ECP_CNTL, tmp);
2294                         udelay(15000);
2295                 }
2296                 DRM_DEBUG("Dynamic Clock Scaling Enabled\n");
2297                 break;
2298         default:
2299                 break;
2300         }
2301
2302 }
2303
2304 int radeon_modeset_cp_init(struct drm_device *dev)
2305 {
2306         drm_radeon_private_t *dev_priv = dev->dev_private;
2307         uint32_t tmp;
2308
2309         /* allocate a ring and ring rptr bits from GART space */
2310         /* these are allocated in GEM files */
2311
2312         dev_priv->usec_timeout = RADEON_DEFAULT_CP_TIMEOUT;
2313         dev_priv->ring.size = RADEON_DEFAULT_RING_SIZE;
2314         dev_priv->cp_mode = RADEON_CSQ_PRIBM_INDBM;
2315
2316         dev_priv->ring.start = (u32 *)(void *)(unsigned long)dev_priv->mm.ring.kmap.virtual;
2317         dev_priv->ring.end = (u32 *)(void *)(unsigned long)dev_priv->mm.ring.kmap.virtual +
2318                 dev_priv->ring.size / sizeof(u32);
2319         dev_priv->ring.size_l2qw = drm_order(dev_priv->ring.size / 8);
2320         dev_priv->ring.rptr_update = 4096;
2321         dev_priv->ring.rptr_update_l2qw = drm_order(4096 / 8);
2322         dev_priv->ring.fetch_size = 32;
2323         dev_priv->ring.fetch_size_l2ow = drm_order(32 / 16);
2324         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
2325         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
2326
2327         dev_priv->new_memmap = true;
2328
2329         r300_init_reg_flags(dev);
2330
2331         radeon_cp_load_microcode(dev_priv);
2332
2333         DRM_DEBUG("ring offset is %x %x\n", dev_priv->mm.ring.bo->offset, dev_priv->mm.ring_read.bo->offset);
2334
2335         radeon_cp_init_ring_buffer(dev, dev_priv);
2336
2337         /* need to enable BUS mastering in Buscntl */
2338         tmp = RADEON_READ(RADEON_BUS_CNTL);
2339         tmp &= ~RADEON_BUS_MASTER_DIS;
2340         RADEON_WRITE(RADEON_BUS_CNTL, tmp);
2341
2342         radeon_do_engine_reset(dev);
2343         radeon_test_writeback(dev_priv);
2344
2345         radeon_do_cp_start(dev_priv);
2346         return 0;
2347 }
2348
2349 static bool radeon_get_bios(struct drm_device *dev)
2350 {
2351         drm_radeon_private_t *dev_priv = dev->dev_private;
2352         u8 __iomem *bios;
2353         size_t size;
2354         uint16_t tmp;
2355
2356         bios = pci_map_rom(dev->pdev, &size);
2357         if (!bios)
2358                 return -1;
2359
2360         dev_priv->bios = kmalloc(size, GFP_KERNEL);
2361         if (!dev_priv->bios) {
2362                 pci_unmap_rom(dev->pdev, bios);
2363                 return -1;
2364         }
2365
2366         memcpy(dev_priv->bios, bios, size);
2367
2368         pci_unmap_rom(dev->pdev, bios);
2369
2370         if (dev_priv->bios[0] != 0x55 || dev_priv->bios[1] != 0xaa)
2371                 goto free_bios;
2372
2373         dev_priv->bios_header_start = radeon_bios16(dev_priv, 0x48);
2374
2375         if (!dev_priv->bios_header_start)
2376                 goto free_bios;
2377
2378         tmp = dev_priv->bios_header_start + 4;
2379
2380         if (!memcmp(dev_priv->bios + tmp, "ATOM", 4) ||
2381             !memcmp(dev_priv->bios + tmp, "MOTA", 4))
2382                 dev_priv->is_atom_bios = true;
2383         else
2384                 dev_priv->is_atom_bios = false;
2385
2386         DRM_DEBUG("%sBIOS detected\n", dev_priv->is_atom_bios ? "ATOM" : "COM");
2387         return true;
2388 free_bios:
2389         kfree(dev_priv->bios);
2390         dev_priv->bios = NULL;
2391         return false;
2392 }
2393
2394 int radeon_modeset_preinit(struct drm_device *dev)
2395 {
2396         drm_radeon_private_t *dev_priv = dev->dev_private;
2397         static struct card_info card;
2398         int ret;
2399
2400         card.dev = dev;
2401         card.reg_read = cail_reg_read;
2402         card.reg_write = cail_reg_write;
2403         card.mc_read = cail_mc_read;
2404         card.mc_write = cail_mc_write;
2405
2406         ret = radeon_get_bios(dev);
2407         if (!ret)
2408                 return -1;
2409
2410         if (dev_priv->is_atom_bios) {
2411                 dev_priv->mode_info.atom_context = atom_parse(&card, dev_priv->bios);
2412                 radeon_atom_initialize_bios_scratch_regs(dev);
2413         } else
2414                 radeon_combios_initialize_bios_scratch_regs(dev);
2415
2416         radeon_get_clock_info(dev);
2417
2418         return 0;
2419 }
2420
2421
2422 int radeon_driver_load(struct drm_device *dev, unsigned long flags)
2423 {
2424         drm_radeon_private_t *dev_priv;
2425         int ret = 0;
2426
2427         dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
2428         if (dev_priv == NULL)
2429                 return -ENOMEM;
2430
2431         memset(dev_priv, 0, sizeof(drm_radeon_private_t));
2432         dev->dev_private = (void *)dev_priv;
2433         dev_priv->flags = flags;
2434
2435         switch (flags & RADEON_FAMILY_MASK) {
2436         case CHIP_R100:
2437         case CHIP_RV200:
2438         case CHIP_R200:
2439         case CHIP_R300:
2440         case CHIP_R350:
2441         case CHIP_R420:
2442         case CHIP_RV410:
2443         case CHIP_RV515:
2444         case CHIP_R520:
2445         case CHIP_RV570:
2446         case CHIP_R580:
2447                 dev_priv->flags |= RADEON_HAS_HIERZ;
2448                 break;
2449         default:
2450                 /* all other chips have no hierarchical z buffer */
2451                 break;
2452         }
2453
2454         dev_priv->chip_family = flags & RADEON_FAMILY_MASK;
2455         if (drm_device_is_agp(dev))
2456                 dev_priv->flags |= RADEON_IS_AGP;
2457         else if (drm_device_is_pcie(dev))
2458                 dev_priv->flags |= RADEON_IS_PCIE;
2459         else
2460                 dev_priv->flags |= RADEON_IS_PCI;
2461
2462
2463
2464         DRM_DEBUG("%s card detected\n",
2465                   ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
2466
2467         ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
2468                          drm_get_resource_len(dev, 2), _DRM_REGISTERS,
2469                          _DRM_DRIVER | _DRM_READ_ONLY, &dev_priv->mmio);
2470         if (ret != 0)
2471                 return ret;
2472
2473         if (drm_core_check_feature(dev, DRIVER_MODESET))
2474                 radeon_modeset_preinit(dev);
2475
2476
2477         radeon_get_vram_type(dev);
2478
2479         dev_priv->pll_errata = 0;
2480
2481         if (dev_priv->chip_family == CHIP_R300 &&
2482             (RADEON_READ(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) == RADEON_CFG_ATI_REV_A11)
2483                 dev_priv->pll_errata |= CHIP_ERRATA_R300_CG;
2484
2485         if (dev_priv->chip_family == CHIP_RV200 ||
2486             dev_priv->chip_family == CHIP_RS200)
2487                 dev_priv->pll_errata |= CHIP_ERRATA_PLL_DUMMYREADS;
2488
2489
2490         if (dev_priv->chip_family == CHIP_RV100 ||
2491             dev_priv->chip_family == CHIP_RS100 ||
2492             dev_priv->chip_family == CHIP_RS200)
2493                 dev_priv->pll_errata |= CHIP_ERRATA_PLL_DELAY;
2494
2495
2496         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2497                 if ((dev_priv->flags & RADEON_IS_MOBILITY) && !radeon_is_avivo(dev_priv)) {
2498                         radeon_set_dynamic_clock(dev, radeon_dynclks);
2499                 } else if (radeon_is_avivo(dev_priv)) {
2500                         if (radeon_dynclks) {
2501                                 radeon_atom_static_pwrmgt_setup(dev, 1);
2502                                 radeon_atom_dyn_clk_setup(dev, 1);
2503                         }
2504                 }
2505                 radeon_force_some_clocks(dev);
2506         }
2507
2508         /* init memory manager - start with all of VRAM and a 32MB GART aperture for now */
2509         dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
2510
2511         drm_bo_driver_init(dev);
2512
2513         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2514
2515                 radeon_gem_mm_init(dev);
2516                 radeon_modeset_init(dev);
2517
2518                 radeon_modeset_cp_init(dev);
2519                 dev->devname = kstrdup(DRIVER_NAME, GFP_KERNEL);
2520
2521                 drm_irq_install(dev);
2522         }
2523
2524
2525         return ret;
2526 }
2527
2528
2529 int radeon_master_create(struct drm_device *dev, struct drm_master *master)
2530 {
2531         struct drm_radeon_master_private *master_priv;
2532         unsigned long sareapage;
2533         int ret;
2534
2535         master_priv = drm_calloc(1, sizeof(*master_priv), DRM_MEM_DRIVER);
2536         if (!master_priv)
2537                 return -ENOMEM;
2538
2539         /* prebuild the SAREA */
2540         sareapage = max(SAREA_MAX, PAGE_SIZE);
2541         ret = drm_addmap(dev, 0, sareapage, _DRM_SHM, _DRM_CONTAINS_LOCK|_DRM_DRIVER,
2542                          &master_priv->sarea);
2543         if (ret) {
2544                 DRM_ERROR("SAREA setup failed\n");
2545                 return ret;
2546         }
2547         master_priv->sarea_priv = master_priv->sarea->handle + sizeof(struct drm_sarea);
2548         master_priv->sarea_priv->pfCurrentPage = 0;
2549
2550         master->driver_priv = master_priv;
2551         return 0;
2552 }
2553
2554 void radeon_master_destroy(struct drm_device *dev, struct drm_master *master)
2555 {
2556         struct drm_radeon_master_private *master_priv = master->driver_priv;
2557         struct drm_radeon_private *dev_priv = dev->dev_private;
2558
2559         if (!master_priv)
2560                 return;
2561
2562         if (master_priv->sarea_priv &&
2563             master_priv->sarea_priv->pfCurrentPage != 0)
2564                 radeon_cp_dispatch_flip(dev, master);
2565
2566         master_priv->sarea_priv = NULL;
2567         if (master_priv->sarea)
2568                 drm_rmmap_locked(dev, master_priv->sarea);
2569
2570         drm_free(master_priv, sizeof(*master_priv), DRM_MEM_DRIVER);
2571
2572         master->driver_priv = NULL;
2573 }
2574 /* Create mappings for registers and framebuffer so userland doesn't necessarily
2575  * have to find them.
2576  */
2577 int radeon_driver_firstopen(struct drm_device *dev)
2578 {
2579         int ret;
2580         drm_local_map_t *map;
2581         drm_radeon_private_t *dev_priv = dev->dev_private;
2582
2583         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
2584
2585         if (!drm_core_check_feature(dev, DRIVER_MODESET))
2586                 radeon_gem_mm_init(dev);
2587
2588         ret = drm_addmap(dev, dev_priv->fb_aper_offset,
2589                          drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
2590                          _DRM_WRITE_COMBINING, &map);
2591         if (ret != 0)
2592                 return ret;
2593
2594         return 0;
2595 }
2596
2597 int radeon_driver_unload(struct drm_device *dev)
2598 {
2599         drm_radeon_private_t *dev_priv = dev->dev_private;
2600
2601         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2602                 drm_irq_uninstall(dev);
2603                 radeon_modeset_cleanup(dev);
2604                 radeon_gem_mm_fini(dev);
2605         }
2606
2607         drm_bo_driver_finish(dev);
2608         drm_rmmap(dev, dev_priv->mmio);
2609
2610         DRM_DEBUG("\n");
2611         drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
2612
2613         dev->dev_private = NULL;
2614         return 0;
2615 }
2616
2617 void radeon_gart_flush(struct drm_device *dev)
2618 {
2619         drm_radeon_private_t *dev_priv = dev->dev_private;
2620
2621         if (dev_priv->flags & RADEON_IS_IGPGART) {
2622                 IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
2623                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, RS480_GART_CACHE_INVALIDATE);
2624                 IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
2625                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
2626         } else if (dev_priv->flags & RADEON_IS_PCIE) {
2627                 u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
2628                 tmp |= RADEON_PCIE_TX_GART_INVALIDATE_TLB;
2629                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp);
2630                 tmp &= ~RADEON_PCIE_TX_GART_INVALIDATE_TLB;
2631                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp);
2632         } else {
2633
2634
2635         }
2636
2637 }