SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/gpu/drm/radeon/radeon_cp.c

   1 /* radeon_cp.c -- CP support for Radeon -*- linux-c -*- */
   2 /*
   3  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   4  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   5  * Copyright 2007 Advanced Micro Devices, Inc.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  23  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  24  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  *
  27  * Authors:
  28  *    Kevin E. Martin <martin@valinux.com>
  29  *    Gareth Hughes <gareth@valinux.com>
  30  */
  31
  32 #include "drmP.h"
  33 #include "drm.h"
  34 #include "drm_sarea.h"
  35 #include "radeon_drm.h"
  36 #include "radeon_drv.h"
  37 #include "r300_reg.h"
  38
  39 #include "radeon_microcode.h"
  40
  41 #define RADEON_FIFO_DEBUG       0
  42
  43 static int radeon_do_cleanup_cp(struct drm_device * dev);
  44 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv);
  45
  46 static u32 radeon_read_ring_rptr(drm_radeon_private_t *dev_priv, u32 off)
  47 {
  48         u32 val;
  49
  50         if (dev_priv->flags & RADEON_IS_AGP) {
  51                 val = DRM_READ32(dev_priv->ring_rptr, off);
  52         } else {
  53                 val = *(((volatile u32 *)
  54                          dev_priv->ring_rptr->handle) +
  55                         (off / sizeof(u32)));
  56                 val = le32_to_cpu(val);
  57         }
  58         return val;
  59 }
  60
  61 u32 radeon_get_ring_head(drm_radeon_private_t *dev_priv)
  62 {
  63         if (dev_priv->writeback_works)
  64                 return radeon_read_ring_rptr(dev_priv, 0);
  65         else
  66                 return RADEON_READ(RADEON_CP_RB_RPTR);
  67 }
  68
  69 static void radeon_write_ring_rptr(drm_radeon_private_t *dev_priv, u32 off, u32 val)
  70 {
  71         if (dev_priv->flags & RADEON_IS_AGP)
  72                 DRM_WRITE32(dev_priv->ring_rptr, off, val);
  73         else
  74                 *(((volatile u32 *) dev_priv->ring_rptr->handle) +
  75                   (off / sizeof(u32))) = cpu_to_le32(val);
  76 }
  77
  78 void radeon_set_ring_head(drm_radeon_private_t *dev_priv, u32 val)
  79 {
  80         radeon_write_ring_rptr(dev_priv, 0, val);
  81 }
  82
  83 u32 radeon_get_scratch(drm_radeon_private_t *dev_priv, int index)
  84 {
  85         if (dev_priv->writeback_works)
  86                 return radeon_read_ring_rptr(dev_priv,
  87                                              RADEON_SCRATCHOFF(index));
  88         else
  89                 return RADEON_READ(RADEON_SCRATCH_REG0 + 4*index);
  90 }
  91
  92 static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
  93 {
  94         u32 ret;
  95         RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
  96         ret = RADEON_READ(R520_MC_IND_DATA);
  97         RADEON_WRITE(R520_MC_IND_INDEX, 0);
  98         return ret;
  99 }
 100
 101 static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 102 {
 103         u32 ret;
 104         RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
 105         ret = RADEON_READ(RS480_NB_MC_DATA);
 106         RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
 107         return ret;
 108 }
 109
 110 static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 111 {
 112         u32 ret;
 113         RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
 114         ret = RADEON_READ(RS690_MC_DATA);
 115         RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
 116         return ret;
 117 }
 118
 119 static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 120 {
 121         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 122             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 123                 return RS690_READ_MCIND(dev_priv, addr);
 124         else
 125                 return RS480_READ_MCIND(dev_priv, addr);
 126 }
 127
 128 u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
 129 {
 130
 131         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 132                 return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
 133         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 134                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 135                 return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
 136         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 137                 return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
 138         else
 139                 return RADEON_READ(RADEON_MC_FB_LOCATION);
 140 }
 141
 142 static void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
 143 {
 144         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 145                 R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
 146         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 147                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 148                 RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
 149         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 150                 R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
 151         else
 152                 RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
 153 }
 154
 155 static void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc)
 156 {
 157         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 158                 R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
 159         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 160                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 161                 RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
 162         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 163                 R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
 164         else
 165                 RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
 166 }
 167
 168 static void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
 169 {
 170         u32 agp_base_hi = upper_32_bits(agp_base);
 171         u32 agp_base_lo = agp_base & 0xffffffff;
 172
 173         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
 174                 R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
 175                 R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
 176         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 177                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 178                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
 179                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
 180         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
 181                 R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
 182                 R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
 183         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 184                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 185                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 186                 RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
 187         } else {
 188                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 189                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
 190                         RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
 191         }
 192 }
 193
 194 static void radeon_enable_bm(struct drm_radeon_private *dev_priv)
 195 {
 196         u32 tmp;
 197         /* Turn on bus mastering */
 198         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 199             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 200                 /* rs600/rs690/rs740 */
 201                 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RS600_BUS_MASTER_DIS;
 202                 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 203         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV350) ||
 204                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 205                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 206                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 207                 /* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */
 208                 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
 209                 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 210         } /* PCIE cards appears to not need this */
 211 }
 212
 213 static int RADEON_READ_PLL(struct drm_device * dev, int addr)
 214 {
 215         drm_radeon_private_t *dev_priv = dev->dev_private;
 216
 217         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x1f);
 218         return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 219 }
 220
 221 static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
 222 {
 223         RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
 224         return RADEON_READ(RADEON_PCIE_DATA);
 225 }
 226
 227 #if RADEON_FIFO_DEBUG
 228 static void radeon_status(drm_radeon_private_t * dev_priv)
 229 {
 230         printk("%s:\n", __func__);
 231         printk("RBBM_STATUS = 0x%08x\n",
 232                (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
 233         printk("CP_RB_RTPR = 0x%08x\n",
 234                (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
 235         printk("CP_RB_WTPR = 0x%08x\n",
 236                (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
 237         printk("AIC_CNTL = 0x%08x\n",
 238                (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
 239         printk("AIC_STAT = 0x%08x\n",
 240                (unsigned int)RADEON_READ(RADEON_AIC_STAT));
 241         printk("AIC_PT_BASE = 0x%08x\n",
 242                (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
 243         printk("TLB_ADDR = 0x%08x\n",
 244                (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
 245         printk("TLB_DATA = 0x%08x\n",
 246                (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
 247 }
 248 #endif
 249
 250 /* ================================================================
 251  * Engine, FIFO control
 252  */
 253
 254 static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
 255 {
 256         u32 tmp;
 257         int i;
 258
 259         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 260
 261         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
 262                 tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
 263                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 264                 RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
 265
 266                 for (i = 0; i < dev_priv->usec_timeout; i++) {
 267                         if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
 268                               & RADEON_RB3D_DC_BUSY)) {
 269                                 return 0;
 270                         }
 271                         DRM_UDELAY(1);
 272                 }
 273         } else {
 274                 /* don't flush or purge cache here or lockup */
 275                 return 0;
 276         }
 277
 278 #if RADEON_FIFO_DEBUG
 279         DRM_ERROR("failed!\n");
 280         radeon_status(dev_priv);
 281 #endif
 282         return -EBUSY;
 283 }
 284
 285 static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
 286 {
 287         int i;
 288
 289         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 290
 291         for (i = 0; i < dev_priv->usec_timeout; i++) {
 292                 int slots = (RADEON_READ(RADEON_RBBM_STATUS)
 293                              & RADEON_RBBM_FIFOCNT_MASK);
 294                 if (slots >= entries)
 295                         return 0;
 296                 DRM_UDELAY(1);
 297         }
 298         DRM_DEBUG("wait for fifo failed status : 0x%08X 0x%08X\n",
 299                  RADEON_READ(RADEON_RBBM_STATUS),
 300                  RADEON_READ(R300_VAP_CNTL_STATUS));
 301
 302 #if RADEON_FIFO_DEBUG
 303         DRM_ERROR("failed!\n");
 304         radeon_status(dev_priv);
 305 #endif
 306         return -EBUSY;
 307 }
 308
 309 static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
 310 {
 311         int i, ret;
 312
 313         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 314
 315         ret = radeon_do_wait_for_fifo(dev_priv, 64);
 316         if (ret)
 317                 return ret;
 318
 319         for (i = 0; i < dev_priv->usec_timeout; i++) {
 320                 if (!(RADEON_READ(RADEON_RBBM_STATUS)
 321                       & RADEON_RBBM_ACTIVE)) {
 322                         radeon_do_pixcache_flush(dev_priv);
 323                         return 0;
 324                 }
 325                 DRM_UDELAY(1);
 326         }
 327         DRM_DEBUG("wait idle failed status : 0x%08X 0x%08X\n",
 328                  RADEON_READ(RADEON_RBBM_STATUS),
 329                  RADEON_READ(R300_VAP_CNTL_STATUS));
 330
 331 #if RADEON_FIFO_DEBUG
 332         DRM_ERROR("failed!\n");
 333         radeon_status(dev_priv);
 334 #endif
 335         return -EBUSY;
 336 }
 337
 338 static void radeon_init_pipes(drm_radeon_private_t *dev_priv)
 339 {
 340         uint32_t gb_tile_config, gb_pipe_sel = 0;
 341
 342         /* RS4xx/RS6xx/R4xx/R5xx */
 343         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
 344                 gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
 345                 dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
 346         } else {
 347                 /* R3xx */
 348                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 349                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350)) {
 350                         dev_priv->num_gb_pipes = 2;
 351                 } else {
 352                         /* R3Vxx */
 353                         dev_priv->num_gb_pipes = 1;
 354                 }
 355         }
 356         DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
 357
 358         gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
 359
 360         switch (dev_priv->num_gb_pipes) {
 361         case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
 362         case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
 363         case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
 364         default:
 365         case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
 366         }
 367
 368         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 369                 RADEON_WRITE_PLL(R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
 370                 RADEON_WRITE(R500_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
 371         }
 372         RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
 373         radeon_do_wait_for_idle(dev_priv);
 374         RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
 375         RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
 376                                                R300_DC_AUTOFLUSH_ENABLE |
 377                                                R300_DC_DC_DISABLE_IGNORE_PE));
 378
 379
 380 }
 381
 382 /* ================================================================
 383  * CP control, initialization
 384  */
 385
 386 /* Load the microcode for the CP */
 387 static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 388 {
 389         int i;
 390         DRM_DEBUG("\n");
 391
 392         radeon_do_wait_for_idle(dev_priv);
 393
 394         RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 395         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
 396             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
 397             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
 398             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
 399             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
 400                 DRM_INFO("Loading R100 Microcode\n");
 401                 for (i = 0; i < 256; i++) {
 402                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 403                                      R100_cp_microcode[i][1]);
 404                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 405                                      R100_cp_microcode[i][0]);
 406                 }
 407         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
 408                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
 409                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
 410                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
 411                 DRM_INFO("Loading R200 Microcode\n");
 412                 for (i = 0; i < 256; i++) {
 413                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 414                                      R200_cp_microcode[i][1]);
 415                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 416                                      R200_cp_microcode[i][0]);
 417                 }
 418         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 419                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
 420                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
 421                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
 422                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 423                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 424                 DRM_INFO("Loading R300 Microcode\n");
 425                 for (i = 0; i < 256; i++) {
 426                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 427                                      R300_cp_microcode[i][1]);
 428                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 429                                      R300_cp_microcode[i][0]);
 430                 }
 431         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 432                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R423) ||
 433                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
 434                 DRM_INFO("Loading R400 Microcode\n");
 435                 for (i = 0; i < 256; i++) {
 436                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 437                                      R420_cp_microcode[i][1]);
 438                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 439                                      R420_cp_microcode[i][0]);
 440                 }
 441         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 442                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 443                 DRM_INFO("Loading RS690/RS740 Microcode\n");
 444                 for (i = 0; i < 256; i++) {
 445                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 446                                      RS690_cp_microcode[i][1]);
 447                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 448                                      RS690_cp_microcode[i][0]);
 449                 }
 450         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
 451                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
 452                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
 453                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
 454                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
 455                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
 456                 DRM_INFO("Loading R500 Microcode\n");
 457                 for (i = 0; i < 256; i++) {
 458                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 459                                      R520_cp_microcode[i][1]);
 460                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 461                                      R520_cp_microcode[i][0]);
 462                 }
 463         }
 464 }
 465
 466 /* Flush any pending commands to the CP.  This should only be used just
 467  * prior to a wait for idle, as it informs the engine that the command
 468  * stream is ending.
 469  */
 470 static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
 471 {
 472         DRM_DEBUG("\n");
 473 #if 0
 474         u32 tmp;
 475
 476         tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
 477         RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
 478 #endif
 479 }
 480
 481 /* Wait for the CP to go idle.
 482  */
 483 int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
 484 {
 485         RING_LOCALS;
 486         DRM_DEBUG("\n");
 487
 488         BEGIN_RING(6);
 489
 490         RADEON_PURGE_CACHE();
 491         RADEON_PURGE_ZCACHE();
 492         RADEON_WAIT_UNTIL_IDLE();
 493
 494         ADVANCE_RING();
 495         COMMIT_RING();
 496
 497         return radeon_do_wait_for_idle(dev_priv);
 498 }
 499
 500 /* Start the Command Processor.
 501  */
 502 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
 503 {
 504         RING_LOCALS;
 505         DRM_DEBUG("\n");
 506
 507         radeon_do_wait_for_idle(dev_priv);
 508
 509         RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
 510
 511         dev_priv->cp_running = 1;
 512
 513         BEGIN_RING(8);
 514         /* isync can only be written through cp on r5xx write it here */
 515         OUT_RING(CP_PACKET0(RADEON_ISYNC_CNTL, 0));
 516         OUT_RING(RADEON_ISYNC_ANY2D_IDLE3D |
 517                  RADEON_ISYNC_ANY3D_IDLE2D |
 518                  RADEON_ISYNC_WAIT_IDLEGUI |
 519                  RADEON_ISYNC_CPSCRATCH_IDLEGUI);
 520         RADEON_PURGE_CACHE();
 521         RADEON_PURGE_ZCACHE();
 522         RADEON_WAIT_UNTIL_IDLE();
 523         ADVANCE_RING();
 524         COMMIT_RING();
 525
 526         dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 527 }
 528
 529 /* Reset the Command Processor.  This will not flush any pending
 530  * commands, so you must wait for the CP command stream to complete
 531  * before calling this routine.
 532  */
 533 static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
 534 {
 535         u32 cur_read_ptr;
 536         DRM_DEBUG("\n");
 537
 538         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 539         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 540         SET_RING_HEAD(dev_priv, cur_read_ptr);
 541         dev_priv->ring.tail = cur_read_ptr;
 542 }
 543
 544 /* Stop the Command Processor.  This will not flush any pending
 545  * commands, so you must flush the command stream and wait for the CP
 546  * to go idle before calling this routine.
 547  */
 548 static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
 549 {
 550         DRM_DEBUG("\n");
 551
 552         RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
 553
 554         dev_priv->cp_running = 0;
 555 }
 556
 557 /* Reset the engine.  This will stop the CP if it is running.
 558  */
 559 static int radeon_do_engine_reset(struct drm_device * dev)
 560 {
 561         drm_radeon_private_t *dev_priv = dev->dev_private;
 562         u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
 563         DRM_DEBUG("\n");
 564
 565         radeon_do_pixcache_flush(dev_priv);
 566
 567         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 568                 /* may need something similar for newer chips */
 569                 clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 570                 mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
 571
 572                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
 573                                                     RADEON_FORCEON_MCLKA |
 574                                                     RADEON_FORCEON_MCLKB |
 575                                                     RADEON_FORCEON_YCLKA |
 576                                                     RADEON_FORCEON_YCLKB |
 577                                                     RADEON_FORCEON_MC |
 578                                                     RADEON_FORCEON_AIC));
 579         }
 580
 581         rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
 582
 583         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
 584                                               RADEON_SOFT_RESET_CP |
 585                                               RADEON_SOFT_RESET_HI |
 586                                               RADEON_SOFT_RESET_SE |
 587                                               RADEON_SOFT_RESET_RE |
 588                                               RADEON_SOFT_RESET_PP |
 589                                               RADEON_SOFT_RESET_E2 |
 590                                               RADEON_SOFT_RESET_RB));
 591         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 592         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
 593                                               ~(RADEON_SOFT_RESET_CP |
 594                                                 RADEON_SOFT_RESET_HI |
 595                                                 RADEON_SOFT_RESET_SE |
 596                                                 RADEON_SOFT_RESET_RE |
 597                                                 RADEON_SOFT_RESET_PP |
 598                                                 RADEON_SOFT_RESET_E2 |
 599                                                 RADEON_SOFT_RESET_RB)));
 600         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 601
 602         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 603                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
 604                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
 605                 RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
 606         }
 607
 608         /* setup the raster pipes */
 609         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
 610             radeon_init_pipes(dev_priv);
 611
 612         /* Reset the CP ring */
 613         radeon_do_cp_reset(dev_priv);
 614
 615         /* The CP is no longer running after an engine reset */
 616         dev_priv->cp_running = 0;
 617
 618         /* Reset any pending vertex, indirect buffers */
 619         radeon_freelist_reset(dev);
 620
 621         return 0;
 622 }
 623
 624 static void radeon_cp_init_ring_buffer(struct drm_device * dev,
 625                                        drm_radeon_private_t *dev_priv,
 626                                        struct drm_file *file_priv)
 627 {
 628         struct drm_radeon_master_private *master_priv;
 629         u32 ring_start, cur_read_ptr;
 630
 631         /* Initialize the memory controller. With new memory map, the fb location
 632          * is not changed, it should have been properly initialized already. Part
 633          * of the problem is that the code below is bogus, assuming the GART is
 634          * always appended to the fb which is not necessarily the case
 635          */
 636         if (!dev_priv->new_memmap)
 637                 radeon_write_fb_location(dev_priv,
 638                              ((dev_priv->gart_vm_start - 1) & 0xffff0000)
 639                              | (dev_priv->fb_location >> 16));
 640
 641 #if __OS_HAS_AGP
 642         if (dev_priv->flags & RADEON_IS_AGP) {
 643                 radeon_write_agp_base(dev_priv, dev->agp->base);
 644
 645                 radeon_write_agp_location(dev_priv,
 646                              (((dev_priv->gart_vm_start - 1 +
 647                                 dev_priv->gart_size) & 0xffff0000) |
 648                               (dev_priv->gart_vm_start >> 16)));
 649
 650                 ring_start = (dev_priv->cp_ring->offset
 651                               - dev->agp->base
 652                               + dev_priv->gart_vm_start);
 653         } else
 654 #endif
 655                 ring_start = (dev_priv->cp_ring->offset
 656                               - (unsigned long)dev->sg->virtual
 657                               + dev_priv->gart_vm_start);
 658
 659         RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
 660
 661         /* Set the write pointer delay */
 662         RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
 663
 664         /* Initialize the ring buffer's read and write pointers */
 665         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 666         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 667         SET_RING_HEAD(dev_priv, cur_read_ptr);
 668         dev_priv->ring.tail = cur_read_ptr;
 669
 670 #if __OS_HAS_AGP
 671         if (dev_priv->flags & RADEON_IS_AGP) {
 672                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 673                              dev_priv->ring_rptr->offset
 674                              - dev->agp->base + dev_priv->gart_vm_start);
 675         } else
 676 #endif
 677         {
 678                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 679                              dev_priv->ring_rptr->offset
 680                              - ((unsigned long) dev->sg->virtual)
 681                              + dev_priv->gart_vm_start);
 682         }
 683
 684         /* Set ring buffer size */
 685 #ifdef __BIG_ENDIAN
 686         RADEON_WRITE(RADEON_CP_RB_CNTL,
 687                      RADEON_BUF_SWAP_32BIT |
 688                      (dev_priv->ring.fetch_size_l2ow << 18) |
 689                      (dev_priv->ring.rptr_update_l2qw << 8) |
 690                      dev_priv->ring.size_l2qw);
 691 #else
 692         RADEON_WRITE(RADEON_CP_RB_CNTL,
 693                      (dev_priv->ring.fetch_size_l2ow << 18) |
 694                      (dev_priv->ring.rptr_update_l2qw << 8) |
 695                      dev_priv->ring.size_l2qw);
 696 #endif
 697
 698
 699         /* Initialize the scratch register pointer.  This will cause
 700          * the scratch register values to be written out to memory
 701          * whenever they are updated.
 702          *
 703          * We simply put this behind the ring read pointer, this works
 704          * with PCI GART as well as (whatever kind of) AGP GART
 705          */
 706         RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
 707                      + RADEON_SCRATCH_REG_OFFSET);
 708
 709         RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
 710
 711         radeon_enable_bm(dev_priv);
 712
 713         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(0), 0);
 714         RADEON_WRITE(RADEON_LAST_FRAME_REG, 0);
 715
 716         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
 717         RADEON_WRITE(RADEON_LAST_DISPATCH_REG, 0);
 718
 719         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(2), 0);
 720         RADEON_WRITE(RADEON_LAST_CLEAR_REG, 0);
 721
 722         /* reset sarea copies of these */
 723         master_priv = file_priv->master->driver_priv;
 724         if (master_priv->sarea_priv) {
 725                 master_priv->sarea_priv->last_frame = 0;
 726                 master_priv->sarea_priv->last_dispatch = 0;
 727                 master_priv->sarea_priv->last_clear = 0;
 728         }
 729
 730         radeon_do_wait_for_idle(dev_priv);
 731
 732         /* Sync everything up */
 733         RADEON_WRITE(RADEON_ISYNC_CNTL,
 734                      (RADEON_ISYNC_ANY2D_IDLE3D |
 735                       RADEON_ISYNC_ANY3D_IDLE2D |
 736                       RADEON_ISYNC_WAIT_IDLEGUI |
 737                       RADEON_ISYNC_CPSCRATCH_IDLEGUI));
 738
 739 }
 740
 741 static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
 742 {
 743         u32 tmp;
 744
 745         /* Start with assuming that writeback doesn't work */
 746         dev_priv->writeback_works = 0;
 747
 748         /* Writeback doesn't seem to work everywhere, test it here and possibly
 749          * enable it if it appears to work
 750          */
 751         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
 752
 753         RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
 754
 755         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 756                 u32 val;
 757
 758                 val = radeon_read_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1));
 759                 if (val == 0xdeadbeef)
 760                         break;
 761                 DRM_UDELAY(1);
 762         }
 763
 764         if (tmp < dev_priv->usec_timeout) {
 765                 dev_priv->writeback_works = 1;
 766                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 767         } else {
 768                 dev_priv->writeback_works = 0;
 769                 DRM_INFO("writeback test failed\n");
 770         }
 771         if (radeon_no_wb == 1) {
 772                 dev_priv->writeback_works = 0;
 773                 DRM_INFO("writeback forced off\n");
 774         }
 775
 776         if (!dev_priv->writeback_works) {
 777                 /* Disable writeback to avoid unnecessary bus master transfer */
 778                 RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) |
 779                              RADEON_RB_NO_UPDATE);
 780                 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
 781         }
 782 }
 783
 784 /* Enable or disable IGP GART on the chip */
 785 static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
 786 {
 787         u32 temp;
 788
 789         if (on) {
 790                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 791                           dev_priv->gart_vm_start,
 792                           (long)dev_priv->gart_info.bus_addr,
 793                           dev_priv->gart_size);
 794
 795                 temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
 796                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 797                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 798                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
 799                                                              RS690_BLOCK_GFX_D3_EN));
 800                 else
 801                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
 802
 803                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 804                                                                RS480_VA_SIZE_32MB));
 805
 806                 temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
 807                 IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
 808                                                         RS480_TLB_ENABLE |
 809                                                         RS480_GTW_LAC_EN |
 810                                                         RS480_1LEVEL_GART));
 811
 812                 temp = dev_priv->gart_info.bus_addr & 0xfffff000;
 813                 temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
 814                 IGP_WRITE_MCIND(RS480_GART_BASE, temp);
 815
 816                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
 817                 IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
 818                                                       RS480_REQ_TYPE_SNOOP_DIS));
 819
 820                 radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
 821
 822                 dev_priv->gart_size = 32*1024*1024;
 823                 temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) &
 824                          0xffff0000) | (dev_priv->gart_vm_start >> 16));
 825
 826                 radeon_write_agp_location(dev_priv, temp);
 827
 828                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
 829                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 830                                                                RS480_VA_SIZE_32MB));
 831
 832                 do {
 833                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 834                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 835                                 break;
 836                         DRM_UDELAY(1);
 837                 } while (1);
 838
 839                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
 840                                 RS480_GART_CACHE_INVALIDATE);
 841
 842                 do {
 843                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 844                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 845                                 break;
 846                         DRM_UDELAY(1);
 847                 } while (1);
 848
 849                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
 850         } else {
 851                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
 852         }
 853 }
 854
 855 static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 856 {
 857         u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
 858         if (on) {
 859
 860                 DRM_DEBUG("programming pcie %08X %08lX %08X\n",
 861                           dev_priv->gart_vm_start,
 862                           (long)dev_priv->gart_info.bus_addr,
 863                           dev_priv->gart_size);
 864                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
 865                                   dev_priv->gart_vm_start);
 866                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
 867                                   dev_priv->gart_info.bus_addr);
 868                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
 869                                   dev_priv->gart_vm_start);
 870                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
 871                                   dev_priv->gart_vm_start +
 872                                   dev_priv->gart_size - 1);
 873
 874                 radeon_write_agp_location(dev_priv, 0xffffffc0); /* ?? */
 875
 876                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 877                                   RADEON_PCIE_TX_GART_EN);
 878         } else {
 879                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 880                                   tmp & ~RADEON_PCIE_TX_GART_EN);
 881         }
 882 }
 883
 884 /* Enable or disable PCI GART on the chip */
 885 static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 886 {
 887         u32 tmp;
 888
 889         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 890             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740) ||
 891             (dev_priv->flags & RADEON_IS_IGPGART)) {
 892                 radeon_set_igpgart(dev_priv, on);
 893                 return;
 894         }
 895
 896         if (dev_priv->flags & RADEON_IS_PCIE) {
 897                 radeon_set_pciegart(dev_priv, on);
 898                 return;
 899         }
 900
 901         tmp = RADEON_READ(RADEON_AIC_CNTL);
 902
 903         if (on) {
 904                 RADEON_WRITE(RADEON_AIC_CNTL,
 905                              tmp | RADEON_PCIGART_TRANSLATE_EN);
 906
 907                 /* set PCI GART page-table base address
 908                  */
 909                 RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
 910
 911                 /* set address range for PCI address translate
 912                  */
 913                 RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
 914                 RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
 915                              + dev_priv->gart_size - 1);
 916
 917                 /* Turn off AGP aperture -- is this required for PCI GART?
 918                  */
 919                 radeon_write_agp_location(dev_priv, 0xffffffc0);
 920                 RADEON_WRITE(RADEON_AGP_COMMAND, 0);    /* clear AGP_COMMAND */
 921         } else {
 922                 RADEON_WRITE(RADEON_AIC_CNTL,
 923                              tmp & ~RADEON_PCIGART_TRANSLATE_EN);
 924         }
 925 }
 926
 927 static int radeon_setup_pcigart_surface(drm_radeon_private_t *dev_priv)
 928 {
 929         struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
 930         struct radeon_virt_surface *vp;
 931         int i;
 932
 933         for (i = 0; i < RADEON_MAX_SURFACES * 2; i++) {
 934                 if (!dev_priv->virt_surfaces[i].file_priv ||
 935                     dev_priv->virt_surfaces[i].file_priv == PCIGART_FILE_PRIV)
 936                         break;
 937         }
 938         if (i >= 2 * RADEON_MAX_SURFACES)
 939                 return -ENOMEM;
 940         vp = &dev_priv->virt_surfaces[i];
 941
 942         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
 943                 struct radeon_surface *sp = &dev_priv->surfaces[i];
 944                 if (sp->refcount)
 945                         continue;
 946
 947                 vp->surface_index = i;
 948                 vp->lower = gart_info->bus_addr;
 949                 vp->upper = vp->lower + gart_info->table_size;
 950                 vp->flags = 0;
 951                 vp->file_priv = PCIGART_FILE_PRIV;
 952
 953                 sp->refcount = 1;
 954                 sp->lower = vp->lower;
 955                 sp->upper = vp->upper;
 956                 sp->flags = 0;
 957
 958                 RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, sp->flags);
 959                 RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16 * i, sp->lower);
 960                 RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16 * i, sp->upper);
 961                 return 0;
 962         }
 963
 964         return -ENOMEM;
 965 }
 966
 967 static int radeon_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
 968                              struct drm_file *file_priv)
 969 {
 970         drm_radeon_private_t *dev_priv = dev->dev_private;
 971         struct drm_radeon_master_private *master_priv = file_priv->master->driver_priv;
 972
 973         DRM_DEBUG("\n");
 974
 975         /* if we require new memory map but we don't have it fail */
 976         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
 977                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
 978                 radeon_do_cleanup_cp(dev);
 979                 return -EINVAL;
 980         }
 981
 982         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
 983                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
 984                 dev_priv->flags &= ~RADEON_IS_AGP;
 985         } else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
 986                    && !init->is_pci) {
 987                 DRM_DEBUG("Restoring AGP flag\n");
 988                 dev_priv->flags |= RADEON_IS_AGP;
 989         }
 990
 991         if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
 992                 DRM_ERROR("PCI GART memory not allocated!\n");
 993                 radeon_do_cleanup_cp(dev);
 994                 return -EINVAL;
 995         }
 996
 997         dev_priv->usec_timeout = init->usec_timeout;
 998         if (dev_priv->usec_timeout < 1 ||
 999             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1000                 DRM_DEBUG("TIMEOUT problem!\n");
1001                 radeon_do_cleanup_cp(dev);
1002                 return -EINVAL;
1003         }
1004
1005         /* Enable vblank on CRTC1 for older X servers
1006          */
1007         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
1008
1009         switch(init->func) {
1010         case RADEON_INIT_R200_CP:
1011                 dev_priv->microcode_version = UCODE_R200;
1012                 break;
1013         case RADEON_INIT_R300_CP:
1014                 dev_priv->microcode_version = UCODE_R300;
1015                 break;
1016         default:
1017                 dev_priv->microcode_version = UCODE_R100;
1018         }
1019
1020         dev_priv->do_boxes = 0;
1021         dev_priv->cp_mode = init->cp_mode;
1022
1023         /* We don't support anything other than bus-mastering ring mode,
1024          * but the ring can be in either AGP or PCI space for the ring
1025          * read pointer.
1026          */
1027         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
1028             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
1029                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
1030                 radeon_do_cleanup_cp(dev);
1031                 return -EINVAL;
1032         }
1033
1034         switch (init->fb_bpp) {
1035         case 16:
1036                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
1037                 break;
1038         case 32:
1039         default:
1040                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1041                 break;
1042         }
1043         dev_priv->front_offset = init->front_offset;
1044         dev_priv->front_pitch = init->front_pitch;
1045         dev_priv->back_offset = init->back_offset;
1046         dev_priv->back_pitch = init->back_pitch;
1047
1048         switch (init->depth_bpp) {
1049         case 16:
1050                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
1051                 break;
1052         case 32:
1053         default:
1054                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
1055                 break;
1056         }
1057         dev_priv->depth_offset = init->depth_offset;
1058         dev_priv->depth_pitch = init->depth_pitch;
1059
1060         /* Hardware state for depth clears.  Remove this if/when we no
1061          * longer clear the depth buffer with a 3D rectangle.  Hard-code
1062          * all values to prevent unwanted 3D state from slipping through
1063          * and screwing with the clear operation.
1064          */
1065         dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
1066                                            (dev_priv->color_fmt << 10) |
1067                                            (dev_priv->microcode_version ==
1068                                             UCODE_R100 ? RADEON_ZBLOCK16 : 0));
1069
1070         dev_priv->depth_clear.rb3d_zstencilcntl =
1071             (dev_priv->depth_fmt |
1072              RADEON_Z_TEST_ALWAYS |
1073              RADEON_STENCIL_TEST_ALWAYS |
1074              RADEON_STENCIL_S_FAIL_REPLACE |
1075              RADEON_STENCIL_ZPASS_REPLACE |
1076              RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
1077
1078         dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
1079                                          RADEON_BFACE_SOLID |
1080                                          RADEON_FFACE_SOLID |
1081                                          RADEON_FLAT_SHADE_VTX_LAST |
1082                                          RADEON_DIFFUSE_SHADE_FLAT |
1083                                          RADEON_ALPHA_SHADE_FLAT |
1084                                          RADEON_SPECULAR_SHADE_FLAT |
1085                                          RADEON_FOG_SHADE_FLAT |
1086                                          RADEON_VTX_PIX_CENTER_OGL |
1087                                          RADEON_ROUND_MODE_TRUNC |
1088                                          RADEON_ROUND_PREC_8TH_PIX);
1089
1090
1091         dev_priv->ring_offset = init->ring_offset;
1092         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
1093         dev_priv->buffers_offset = init->buffers_offset;
1094         dev_priv->gart_textures_offset = init->gart_textures_offset;
1095
1096         master_priv->sarea = drm_getsarea(dev);
1097         if (!master_priv->sarea) {
1098                 DRM_ERROR("could not find sarea!\n");
1099                 radeon_do_cleanup_cp(dev);
1100                 return -EINVAL;
1101         }
1102
1103         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
1104         if (!dev_priv->cp_ring) {
1105                 DRM_ERROR("could not find cp ring region!\n");
1106                 radeon_do_cleanup_cp(dev);
1107                 return -EINVAL;
1108         }
1109         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
1110         if (!dev_priv->ring_rptr) {
1111                 DRM_ERROR("could not find ring read pointer!\n");
1112                 radeon_do_cleanup_cp(dev);
1113                 return -EINVAL;
1114         }
1115         dev->agp_buffer_token = init->buffers_offset;
1116         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1117         if (!dev->agp_buffer_map) {
1118                 DRM_ERROR("could not find dma buffer region!\n");
1119                 radeon_do_cleanup_cp(dev);
1120                 return -EINVAL;
1121         }
1122
1123         if (init->gart_textures_offset) {
1124                 dev_priv->gart_textures =
1125                     drm_core_findmap(dev, init->gart_textures_offset);
1126                 if (!dev_priv->gart_textures) {
1127                         DRM_ERROR("could not find GART texture region!\n");
1128                         radeon_do_cleanup_cp(dev);
1129                         return -EINVAL;
1130                 }
1131         }
1132
1133 #if __OS_HAS_AGP
1134         if (dev_priv->flags & RADEON_IS_AGP) {
1135                 drm_core_ioremap_wc(dev_priv->cp_ring, dev);
1136                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
1137                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
1138                 if (!dev_priv->cp_ring->handle ||
1139                     !dev_priv->ring_rptr->handle ||
1140                     !dev->agp_buffer_map->handle) {
1141                         DRM_ERROR("could not find ioremap agp regions!\n");
1142                         radeon_do_cleanup_cp(dev);
1143                         return -EINVAL;
1144                 }
1145         } else
1146 #endif
1147         {
1148                 dev_priv->cp_ring->handle =
1149                         (void *)(unsigned long)dev_priv->cp_ring->offset;
1150                 dev_priv->ring_rptr->handle =
1151                         (void *)(unsigned long)dev_priv->ring_rptr->offset;
1152                 dev->agp_buffer_map->handle =
1153                         (void *)(unsigned long)dev->agp_buffer_map->offset;
1154
1155                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
1156                           dev_priv->cp_ring->handle);
1157                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
1158                           dev_priv->ring_rptr->handle);
1159                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
1160                           dev->agp_buffer_map->handle);
1161         }
1162
1163         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
1164         dev_priv->fb_size =
1165                 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
1166                 - dev_priv->fb_location;
1167
1168         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1169                                         ((dev_priv->front_offset
1170                                           + dev_priv->fb_location) >> 10));
1171
1172         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1173                                        ((dev_priv->back_offset
1174                                          + dev_priv->fb_location) >> 10));
1175
1176         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1177                                         ((dev_priv->depth_offset
1178                                           + dev_priv->fb_location) >> 10));
1179
1180         dev_priv->gart_size = init->gart_size;
1181
1182         /* New let's set the memory map ... */
1183         if (dev_priv->new_memmap) {
1184                 u32 base = 0;
1185
1186                 DRM_INFO("Setting GART location based on new memory map\n");
1187
1188                 /* If using AGP, try to locate the AGP aperture at the same
1189                  * location in the card and on the bus, though we have to
1190                  * align it down.
1191                  */
1192 #if __OS_HAS_AGP
1193                 if (dev_priv->flags & RADEON_IS_AGP) {
1194                         base = dev->agp->base;
1195                         /* Check if valid */
1196                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
1197                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
1198                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
1199                                          dev->agp->base);
1200                                 base = 0;
1201                         }
1202                 }
1203 #endif
1204                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
1205                 if (base == 0) {
1206                         base = dev_priv->fb_location + dev_priv->fb_size;
1207                         if (base < dev_priv->fb_location ||
1208                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
1209                                 base = dev_priv->fb_location
1210                                         - dev_priv->gart_size;
1211                 }
1212                 dev_priv->gart_vm_start = base & 0xffc00000u;
1213                 if (dev_priv->gart_vm_start != base)
1214                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
1215                                  base, dev_priv->gart_vm_start);
1216         } else {
1217                 DRM_INFO("Setting GART location based on old memory map\n");
1218                 dev_priv->gart_vm_start = dev_priv->fb_location +
1219                         RADEON_READ(RADEON_CONFIG_APER_SIZE);
1220         }
1221
1222 #if __OS_HAS_AGP
1223         if (dev_priv->flags & RADEON_IS_AGP)
1224                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1225                                                  - dev->agp->base
1226                                                  + dev_priv->gart_vm_start);
1227         else
1228 #endif
1229                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1230                                         - (unsigned long)dev->sg->virtual
1231                                         + dev_priv->gart_vm_start);
1232
1233         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
1234         DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
1235         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
1236                   dev_priv->gart_buffers_offset);
1237
1238         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
1239         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
1240                               + init->ring_size / sizeof(u32));
1241         dev_priv->ring.size = init->ring_size;
1242         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
1243
1244         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
1245         dev_priv->ring.rptr_update_l2qw = drm_order( /* init->rptr_update */ 4096 / 8);
1246
1247         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
1248         dev_priv->ring.fetch_size_l2ow = drm_order( /* init->fetch_size */ 32 / 16);
1249         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1250
1251         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
1252
1253 #if __OS_HAS_AGP
1254         if (dev_priv->flags & RADEON_IS_AGP) {
1255                 /* Turn off PCI GART */
1256                 radeon_set_pcigart(dev_priv, 0);
1257         } else
1258 #endif
1259         {
1260                 u32 sctrl;
1261                 int ret;
1262
1263                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
1264                 /* if we have an offset set from userspace */
1265                 if (dev_priv->pcigart_offset_set) {
1266                         dev_priv->gart_info.bus_addr =
1267                                 (resource_size_t)dev_priv->pcigart_offset + dev_priv->fb_location;
1268                         dev_priv->gart_info.mapping.offset =
1269                             dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
1270                         dev_priv->gart_info.mapping.size =
1271                             dev_priv->gart_info.table_size;
1272
1273                         drm_core_ioremap_wc(&dev_priv->gart_info.mapping, dev);
1274                         dev_priv->gart_info.addr =
1275                             dev_priv->gart_info.mapping.handle;
1276
1277                         if (dev_priv->flags & RADEON_IS_PCIE)
1278                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
1279                         else
1280                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1281                         dev_priv->gart_info.gart_table_location =
1282                             DRM_ATI_GART_FB;
1283
1284                         DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
1285                                   dev_priv->gart_info.addr,
1286                                   dev_priv->pcigart_offset);
1287                 } else {
1288                         if (dev_priv->flags & RADEON_IS_IGPGART)
1289                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
1290                         else
1291                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1292                         dev_priv->gart_info.gart_table_location =
1293                             DRM_ATI_GART_MAIN;
1294                         dev_priv->gart_info.addr = NULL;
1295                         dev_priv->gart_info.bus_addr = 0;
1296                         if (dev_priv->flags & RADEON_IS_PCIE) {
1297                                 DRM_ERROR
1298                                     ("Cannot use PCI Express without GART in FB memory\n");
1299                                 radeon_do_cleanup_cp(dev);
1300                                 return -EINVAL;
1301                         }
1302                 }
1303
1304                 sctrl = RADEON_READ(RADEON_SURFACE_CNTL);
1305                 RADEON_WRITE(RADEON_SURFACE_CNTL, 0);
1306                 ret = drm_ati_pcigart_init(dev, &dev_priv->gart_info);
1307                 RADEON_WRITE(RADEON_SURFACE_CNTL, sctrl);
1308
1309                 if (!ret) {
1310                         DRM_ERROR("failed to init PCI GART!\n");
1311                         radeon_do_cleanup_cp(dev);
1312                         return -ENOMEM;
1313                 }
1314
1315                 ret = radeon_setup_pcigart_surface(dev_priv);
1316                 if (ret) {
1317                         DRM_ERROR("failed to setup GART surface!\n");
1318                         drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info);
1319                         radeon_do_cleanup_cp(dev);
1320                         return ret;
1321                 }
1322
1323                 /* Turn on PCI GART */
1324                 radeon_set_pcigart(dev_priv, 1);
1325         }
1326
1327         radeon_cp_load_microcode(dev_priv);
1328         radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
1329
1330         dev_priv->last_buf = 0;
1331
1332         radeon_do_engine_reset(dev);
1333         radeon_test_writeback(dev_priv);
1334
1335         return 0;
1336 }
1337
1338 static int radeon_do_cleanup_cp(struct drm_device * dev)
1339 {
1340         drm_radeon_private_t *dev_priv = dev->dev_private;
1341         DRM_DEBUG("\n");
1342
1343         /* Make sure interrupts are disabled here because the uninstall ioctl
1344          * may not have been called from userspace and after dev_private
1345          * is freed, it's too late.
1346          */
1347         if (dev->irq_enabled)
1348                 drm_irq_uninstall(dev);
1349
1350 #if __OS_HAS_AGP
1351         if (dev_priv->flags & RADEON_IS_AGP) {
1352                 if (dev_priv->cp_ring != NULL) {
1353                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1354                         dev_priv->cp_ring = NULL;
1355                 }
1356                 if (dev_priv->ring_rptr != NULL) {
1357                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1358                         dev_priv->ring_rptr = NULL;
1359                 }
1360                 if (dev->agp_buffer_map != NULL) {
1361                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1362                         dev->agp_buffer_map = NULL;
1363                 }
1364         } else
1365 #endif
1366         {
1367
1368                 if (dev_priv->gart_info.bus_addr) {
1369                         /* Turn off PCI GART */
1370                         radeon_set_pcigart(dev_priv, 0);
1371                         if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
1372                                 DRM_ERROR("failed to cleanup PCI GART!\n");
1373                 }
1374
1375                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
1376                 {
1377                         drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1378                         dev_priv->gart_info.addr = 0;
1379                 }
1380         }
1381         /* only clear to the start of flags */
1382         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1383
1384         return 0;
1385 }
1386
1387 /* This code will reinit the Radeon CP hardware after a resume from disc.
1388  * AFAIK, it would be very difficult to pickle the state at suspend time, so
1389  * here we make sure that all Radeon hardware initialisation is re-done without
1390  * affecting running applications.
1391  *
1392  * Charl P. Botha <http://cpbotha.net>
1393  */
1394 static int radeon_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
1395 {
1396         drm_radeon_private_t *dev_priv = dev->dev_private;
1397
1398         if (!dev_priv) {
1399                 DRM_ERROR("Called with no initialization\n");
1400                 return -EINVAL;
1401         }
1402
1403         DRM_DEBUG("Starting radeon_do_resume_cp()\n");
1404
1405 #if __OS_HAS_AGP
1406         if (dev_priv->flags & RADEON_IS_AGP) {
1407                 /* Turn off PCI GART */
1408                 radeon_set_pcigart(dev_priv, 0);
1409         } else
1410 #endif
1411         {
1412                 /* Turn on PCI GART */
1413                 radeon_set_pcigart(dev_priv, 1);
1414         }
1415
1416         radeon_cp_load_microcode(dev_priv);
1417         radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
1418
1419         radeon_do_engine_reset(dev);
1420         radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
1421
1422         DRM_DEBUG("radeon_do_resume_cp() complete\n");
1423
1424         return 0;
1425 }
1426
1427 int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
1428 {
1429         drm_radeon_init_t *init = data;
1430
1431         LOCK_TEST_WITH_RETURN(dev, file_priv);
1432
1433         if (init->func == RADEON_INIT_R300_CP)
1434                 r300_init_reg_flags(dev);
1435
1436         switch (init->func) {
1437         case RADEON_INIT_CP:
1438         case RADEON_INIT_R200_CP:
1439         case RADEON_INIT_R300_CP:
1440                 return radeon_do_init_cp(dev, init, file_priv);
1441         case RADEON_CLEANUP_CP:
1442                 return radeon_do_cleanup_cp(dev);
1443         }
1444
1445         return -EINVAL;
1446 }
1447
1448 int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
1449 {
1450         drm_radeon_private_t *dev_priv = dev->dev_private;
1451         DRM_DEBUG("\n");
1452
1453         LOCK_TEST_WITH_RETURN(dev, file_priv);
1454
1455         if (dev_priv->cp_running) {
1456                 DRM_DEBUG("while CP running\n");
1457                 return 0;
1458         }
1459         if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
1460                 DRM_DEBUG("called with bogus CP mode (%d)\n",
1461                           dev_priv->cp_mode);
1462                 return 0;
1463         }
1464
1465         radeon_do_cp_start(dev_priv);
1466
1467         return 0;
1468 }
1469
1470 /* Stop the CP.  The engine must have been idled before calling this
1471  * routine.
1472  */
1473 int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
1474 {
1475         drm_radeon_private_t *dev_priv = dev->dev_private;
1476         drm_radeon_cp_stop_t *stop = data;
1477         int ret;
1478         DRM_DEBUG("\n");
1479
1480         LOCK_TEST_WITH_RETURN(dev, file_priv);
1481
1482         if (!dev_priv->cp_running)
1483                 return 0;
1484
1485         /* Flush any pending CP commands.  This ensures any outstanding
1486          * commands are exectuted by the engine before we turn it off.
1487          */
1488         if (stop->flush) {
1489                 radeon_do_cp_flush(dev_priv);
1490         }
1491
1492         /* If we fail to make the engine go idle, we return an error
1493          * code so that the DRM ioctl wrapper can try again.
1494          */
1495         if (stop->idle) {
1496                 ret = radeon_do_cp_idle(dev_priv);
1497                 if (ret)
1498                         return ret;
1499         }
1500
1501         /* Finally, we can turn off the CP.  If the engine isn't idle,
1502          * we will get some dropped triangles as they won't be fully
1503          * rendered before the CP is shut down.
1504          */
1505         radeon_do_cp_stop(dev_priv);
1506
1507         /* Reset the engine */
1508         radeon_do_engine_reset(dev);
1509
1510         return 0;
1511 }
1512
1513 void radeon_do_release(struct drm_device * dev)
1514 {
1515         drm_radeon_private_t *dev_priv = dev->dev_private;
1516         int i, ret;
1517
1518         if (dev_priv) {
1519                 if (dev_priv->cp_running) {
1520                         /* Stop the cp */
1521                         while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
1522                                 DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1523 #ifdef __linux__
1524                                 schedule();
1525 #else
1526                                 tsleep(&ret, PZERO, "rdnrel", 1);
1527 #endif
1528                         }
1529                         radeon_do_cp_stop(dev_priv);
1530                         radeon_do_engine_reset(dev);
1531                 }
1532
1533                 /* Disable *all* interrupts */
1534                 if (dev_priv->mmio)     /* remove this after permanent addmaps */
1535                         RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
1536
1537                 if (dev_priv->mmio) {   /* remove all surfaces */
1538                         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1539                                 RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
1540                                 RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
1541                                              16 * i, 0);
1542                                 RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
1543                                              16 * i, 0);
1544                         }
1545                 }
1546
1547                 /* Free memory heap structures */
1548                 radeon_mem_takedown(&(dev_priv->gart_heap));
1549                 radeon_mem_takedown(&(dev_priv->fb_heap));
1550
1551                 /* deallocate kernel resources */
1552                 radeon_do_cleanup_cp(dev);
1553         }
1554 }
1555
1556 /* Just reset the CP ring.  Called as part of an X Server engine reset.
1557  */
1558 int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1559 {
1560         drm_radeon_private_t *dev_priv = dev->dev_private;
1561         DRM_DEBUG("\n");
1562
1563         LOCK_TEST_WITH_RETURN(dev, file_priv);
1564
1565         if (!dev_priv) {
1566                 DRM_DEBUG("called before init done\n");
1567                 return -EINVAL;
1568         }
1569
1570         radeon_do_cp_reset(dev_priv);
1571
1572         /* The CP is no longer running after an engine reset */
1573         dev_priv->cp_running = 0;
1574
1575         return 0;
1576 }
1577
1578 int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
1579 {
1580         drm_radeon_private_t *dev_priv = dev->dev_private;
1581         DRM_DEBUG("\n");
1582
1583         LOCK_TEST_WITH_RETURN(dev, file_priv);
1584
1585         return radeon_do_cp_idle(dev_priv);
1586 }
1587
1588 /* Added by Charl P. Botha to call radeon_do_resume_cp().
1589  */
1590 int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
1591 {
1592         return radeon_do_resume_cp(dev, file_priv);
1593 }
1594
1595 int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1596 {
1597         DRM_DEBUG("\n");
1598
1599         LOCK_TEST_WITH_RETURN(dev, file_priv);
1600
1601         return radeon_do_engine_reset(dev);
1602 }
1603
1604 /* ================================================================
1605  * Fullscreen mode
1606  */
1607
1608 /* KW: Deprecated to say the least:
1609  */
1610 int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
1611 {
1612         return 0;
1613 }
1614
1615 /* ================================================================
1616  * Freelist management
1617  */
1618
1619 /* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
1620  *   bufs until freelist code is used.  Note this hides a problem with
1621  *   the scratch register * (used to keep track of last buffer
1622  *   completed) being written to before * the last buffer has actually
1623  *   completed rendering.
1624  *
1625  * KW:  It's also a good way to find free buffers quickly.
1626  *
1627  * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
1628  * sleep.  However, bugs in older versions of radeon_accel.c mean that
1629  * we essentially have to do this, else old clients will break.
1630  *
1631  * However, it does leave open a potential deadlock where all the
1632  * buffers are held by other clients, which can't release them because
1633  * they can't get the lock.
1634  */
1635
1636 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1637 {
1638         struct drm_device_dma *dma = dev->dma;
1639         drm_radeon_private_t *dev_priv = dev->dev_private;
1640         drm_radeon_buf_priv_t *buf_priv;
1641         struct drm_buf *buf;
1642         int i, t;
1643         int start;
1644
1645         if (++dev_priv->last_buf >= dma->buf_count)
1646                 dev_priv->last_buf = 0;
1647
1648         start = dev_priv->last_buf;
1649
1650         for (t = 0; t < dev_priv->usec_timeout; t++) {
1651                 u32 done_age = GET_SCRATCH(dev_priv, 1);
1652                 DRM_DEBUG("done_age = %d\n", done_age);
1653                 for (i = start; i < dma->buf_count; i++) {
1654                         buf = dma->buflist[i];
1655                         buf_priv = buf->dev_private;
1656                         if (buf->file_priv == NULL || (buf->pending &&
1657                                                        buf_priv->age <=
1658                                                        done_age)) {
1659                                 dev_priv->stats.requested_bufs++;
1660                                 buf->pending = 0;
1661                                 return buf;
1662                         }
1663                         start = 0;
1664                 }
1665
1666                 if (t) {
1667                         DRM_UDELAY(1);
1668                         dev_priv->stats.freelist_loops++;
1669                 }
1670         }
1671
1672         DRM_DEBUG("returning NULL!\n");
1673         return NULL;
1674 }
1675
1676 #if 0
1677 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1678 {
1679         struct drm_device_dma *dma = dev->dma;
1680         drm_radeon_private_t *dev_priv = dev->dev_private;
1681         drm_radeon_buf_priv_t *buf_priv;
1682         struct drm_buf *buf;
1683         int i, t;
1684         int start;
1685         u32 done_age;
1686
1687         done_age = radeon_read_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1));
1688         if (++dev_priv->last_buf >= dma->buf_count)
1689                 dev_priv->last_buf = 0;
1690
1691         start = dev_priv->last_buf;
1692         dev_priv->stats.freelist_loops++;
1693
1694         for (t = 0; t < 2; t++) {
1695                 for (i = start; i < dma->buf_count; i++) {
1696                         buf = dma->buflist[i];
1697                         buf_priv = buf->dev_private;
1698                         if (buf->file_priv == 0 || (buf->pending &&
1699                                                     buf_priv->age <=
1700                                                     done_age)) {
1701                                 dev_priv->stats.requested_bufs++;
1702                                 buf->pending = 0;
1703                                 return buf;
1704                         }
1705                 }
1706                 start = 0;
1707         }
1708
1709         return NULL;
1710 }
1711 #endif
1712
1713 void radeon_freelist_reset(struct drm_device * dev)
1714 {
1715         struct drm_device_dma *dma = dev->dma;
1716         drm_radeon_private_t *dev_priv = dev->dev_private;
1717         int i;
1718
1719         dev_priv->last_buf = 0;
1720         for (i = 0; i < dma->buf_count; i++) {
1721                 struct drm_buf *buf = dma->buflist[i];
1722                 drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1723                 buf_priv->age = 0;
1724         }
1725 }
1726
1727 /* ================================================================
1728  * CP command submission
1729  */
1730
1731 int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
1732 {
1733         drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
1734         int i;
1735         u32 last_head = GET_RING_HEAD(dev_priv);
1736
1737         for (i = 0; i < dev_priv->usec_timeout; i++) {
1738                 u32 head = GET_RING_HEAD(dev_priv);
1739
1740                 ring->space = (head - ring->tail) * sizeof(u32);
1741                 if (ring->space <= 0)
1742                         ring->space += ring->size;
1743                 if (ring->space > n)
1744                         return 0;
1745
1746                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
1747
1748                 if (head != last_head)
1749                         i = 0;
1750                 last_head = head;
1751
1752                 DRM_UDELAY(1);
1753         }
1754
1755         /* FIXME: This return value is ignored in the BEGIN_RING macro! */
1756 #if RADEON_FIFO_DEBUG
1757         radeon_status(dev_priv);
1758         DRM_ERROR("failed!\n");
1759 #endif
1760         return -EBUSY;
1761 }
1762
1763 static int radeon_cp_get_buffers(struct drm_device *dev,
1764                                  struct drm_file *file_priv,
1765                                  struct drm_dma * d)
1766 {
1767         int i;
1768         struct drm_buf *buf;
1769
1770         for (i = d->granted_count; i < d->request_count; i++) {
1771                 buf = radeon_freelist_get(dev);
1772                 if (!buf)
1773                         return -EBUSY;  /* NOTE: broken client */
1774
1775                 buf->file_priv = file_priv;
1776
1777                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
1778                                      sizeof(buf->idx)))
1779                         return -EFAULT;
1780                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
1781                                      sizeof(buf->total)))
1782                         return -EFAULT;
1783
1784                 d->granted_count++;
1785         }
1786         return 0;
1787 }
1788
1789 int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
1790 {
1791         struct drm_device_dma *dma = dev->dma;
1792         int ret = 0;
1793         struct drm_dma *d = data;
1794
1795         LOCK_TEST_WITH_RETURN(dev, file_priv);
1796
1797         /* Please don't send us buffers.
1798          */
1799         if (d->send_count != 0) {
1800                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
1801                           DRM_CURRENTPID, d->send_count);
1802                 return -EINVAL;
1803         }
1804
1805         /* We'll send you buffers.
1806          */
1807         if (d->request_count < 0 || d->request_count > dma->buf_count) {
1808                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
1809                           DRM_CURRENTPID, d->request_count, dma->buf_count);
1810                 return -EINVAL;
1811         }
1812
1813         d->granted_count = 0;
1814
1815         if (d->request_count) {
1816                 ret = radeon_cp_get_buffers(dev, file_priv, d);
1817         }
1818
1819         return ret;
1820 }
1821
1822 int radeon_driver_load(struct drm_device *dev, unsigned long flags)
1823 {
1824         drm_radeon_private_t *dev_priv;
1825         int ret = 0;
1826
1827         dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
1828         if (dev_priv == NULL)
1829                 return -ENOMEM;
1830
1831         memset(dev_priv, 0, sizeof(drm_radeon_private_t));
1832         dev->dev_private = (void *)dev_priv;
1833         dev_priv->flags = flags;
1834
1835         switch (flags & RADEON_FAMILY_MASK) {
1836         case CHIP_R100:
1837         case CHIP_RV200:
1838         case CHIP_R200:
1839         case CHIP_R300:
1840         case CHIP_R350:
1841         case CHIP_R420:
1842         case CHIP_R423:
1843         case CHIP_RV410:
1844         case CHIP_RV515:
1845         case CHIP_R520:
1846         case CHIP_RV570:
1847         case CHIP_R580:
1848                 dev_priv->flags |= RADEON_HAS_HIERZ;
1849                 break;
1850         default:
1851                 /* all other chips have no hierarchical z buffer */
1852                 break;
1853         }
1854
1855         if (drm_device_is_agp(dev))
1856                 dev_priv->flags |= RADEON_IS_AGP;
1857         else if (drm_device_is_pcie(dev))
1858                 dev_priv->flags |= RADEON_IS_PCIE;
1859         else
1860                 dev_priv->flags |= RADEON_IS_PCI;
1861
1862         ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
1863                          drm_get_resource_len(dev, 2), _DRM_REGISTERS,
1864                          _DRM_READ_ONLY | _DRM_DRIVER, &dev_priv->mmio);
1865         if (ret != 0)
1866                 return ret;
1867
1868         ret = drm_vblank_init(dev, 2);
1869         if (ret) {
1870                 radeon_driver_unload(dev);
1871                 return ret;
1872         }
1873
1874         DRM_DEBUG("%s card detected\n",
1875                   ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
1876         return ret;
1877 }
1878
1879 int radeon_master_create(struct drm_device *dev, struct drm_master *master)
1880 {
1881         struct drm_radeon_master_private *master_priv;
1882         unsigned long sareapage;
1883         int ret;
1884
1885         master_priv = drm_calloc(1, sizeof(*master_priv), DRM_MEM_DRIVER);
1886         if (!master_priv)
1887                 return -ENOMEM;
1888
1889         /* prebuild the SAREA */
1890         sareapage = max_t(unsigned long, SAREA_MAX, PAGE_SIZE);
1891         ret = drm_addmap(dev, 0, sareapage, _DRM_SHM, _DRM_CONTAINS_LOCK|_DRM_DRIVER,
1892                          &master_priv->sarea);
1893         if (ret) {
1894                 DRM_ERROR("SAREA setup failed\n");
1895                 return ret;
1896         }
1897         master_priv->sarea_priv = master_priv->sarea->handle + sizeof(struct drm_sarea);
1898         master_priv->sarea_priv->pfCurrentPage = 0;
1899
1900         master->driver_priv = master_priv;
1901         return 0;
1902 }
1903
1904 void radeon_master_destroy(struct drm_device *dev, struct drm_master *master)
1905 {
1906         struct drm_radeon_master_private *master_priv = master->driver_priv;
1907
1908         if (!master_priv)
1909                 return;
1910
1911         if (master_priv->sarea_priv &&
1912             master_priv->sarea_priv->pfCurrentPage != 0)
1913                 radeon_cp_dispatch_flip(dev, master);
1914
1915         master_priv->sarea_priv = NULL;
1916         if (master_priv->sarea)
1917                 drm_rmmap_locked(dev, master_priv->sarea);
1918
1919         drm_free(master_priv, sizeof(*master_priv), DRM_MEM_DRIVER);
1920
1921         master->driver_priv = NULL;
1922 }
1923
1924 /* Create mappings for registers and framebuffer so userland doesn't necessarily
1925  * have to find them.
1926  */
1927 int radeon_driver_firstopen(struct drm_device *dev)
1928 {
1929         int ret;
1930         drm_local_map_t *map;
1931         drm_radeon_private_t *dev_priv = dev->dev_private;
1932
1933         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
1934
1935         dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
1936         ret = drm_addmap(dev, dev_priv->fb_aper_offset,
1937                          drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
1938                          _DRM_WRITE_COMBINING, &map);
1939         if (ret != 0)
1940                 return ret;
1941
1942         return 0;
1943 }
1944
1945 int radeon_driver_unload(struct drm_device *dev)
1946 {
1947         drm_radeon_private_t *dev_priv = dev->dev_private;
1948
1949         DRM_DEBUG("\n");
1950
1951         drm_rmmap(dev, dev_priv->mmio);
1952
1953         drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
1954
1955         dev->dev_private = NULL;
1956         return 0;
1957 }
1958
1959 void radeon_commit_ring(drm_radeon_private_t *dev_priv)
1960 {
1961         int i;
1962         u32 *ring;
1963         int tail_aligned;
1964
1965         /* check if the ring is padded out to 16-dword alignment */
1966
1967         tail_aligned = dev_priv->ring.tail & 0xf;
1968         if (tail_aligned) {
1969                 int num_p2 = 16 - tail_aligned;
1970
1971                 ring = dev_priv->ring.start;
1972                 /* pad with some CP_PACKET2 */
1973                 for (i = 0; i < num_p2; i++)
1974                         ring[dev_priv->ring.tail + i] = CP_PACKET2();
1975
1976                 dev_priv->ring.tail += i;
1977
1978                 dev_priv->ring.space -= num_p2 * sizeof(u32);
1979         }
1980
1981         dev_priv->ring.tail &= dev_priv->ring.tail_mask;
1982
1983         DRM_MEMORYBARRIER();
1984         GET_RING_HEAD( dev_priv );
1985
1986         RADEON_WRITE( RADEON_CP_RB_WPTR, dev_priv->ring.tail );
1987         /* read from PCI bus to ensure correct posting */
1988         RADEON_READ( RADEON_CP_RB_RPTR );
1989 }