SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/gpu/drm/r128/r128_cce.c

   1 /* r128_cce.c -- ATI Rage 128 driver -*- linux-c -*-
   2  * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com
   3  */
   4 /*
   5  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   6  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
   7  * All Rights Reserved.
   8  *
   9  * Permission is hereby granted, free of charge, to any person obtaining a
  10  * copy of this software and associated documentation files (the "Software"),
  11  * to deal in the Software without restriction, including without limitation
  12  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13  * and/or sell copies of the Software, and to permit persons to whom the
  14  * Software is furnished to do so, subject to the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the next
  17  * paragraph) shall be included in all copies or substantial portions of the
  18  * Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  23  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  24  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  25  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  26  * DEALINGS IN THE SOFTWARE.
  27  *
  28  * Authors:
  29  *    Gareth Hughes <gareth@valinux.com>
  30  */
  31
  32 #include <linux/firmware.h>
  33 #include <linux/platform_device.h>
  34
  35 #include "drmP.h"
  36 #include "drm.h"
  37 #include "r128_drm.h"
  38 #include "r128_drv.h"
  39
  40 #define R128_FIFO_DEBUG         0
  41
  42 #define FIRMWARE_NAME           "r128/r128_cce.bin"
  43
  44 MODULE_FIRMWARE(FIRMWARE_NAME);
  45
  46 static int R128_READ_PLL(struct drm_device * dev, int addr)
  47 {
  48         drm_r128_private_t *dev_priv = dev->dev_private;
  49
  50         R128_WRITE8(R128_CLOCK_CNTL_INDEX, addr & 0x1f);
  51         return R128_READ(R128_CLOCK_CNTL_DATA);
  52 }
  53
  54 #if R128_FIFO_DEBUG
  55 static void r128_status(drm_r128_private_t * dev_priv)
  56 {
  57         printk("GUI_STAT           = 0x%08x\n",
  58                (unsigned int)R128_READ(R128_GUI_STAT));
  59         printk("PM4_STAT           = 0x%08x\n",
  60                (unsigned int)R128_READ(R128_PM4_STAT));
  61         printk("PM4_BUFFER_DL_WPTR = 0x%08x\n",
  62                (unsigned int)R128_READ(R128_PM4_BUFFER_DL_WPTR));
  63         printk("PM4_BUFFER_DL_RPTR = 0x%08x\n",
  64                (unsigned int)R128_READ(R128_PM4_BUFFER_DL_RPTR));
  65         printk("PM4_MICRO_CNTL     = 0x%08x\n",
  66                (unsigned int)R128_READ(R128_PM4_MICRO_CNTL));
  67         printk("PM4_BUFFER_CNTL    = 0x%08x\n",
  68                (unsigned int)R128_READ(R128_PM4_BUFFER_CNTL));
  69 }
  70 #endif
  71
  72 /* ================================================================
  73  * Engine, FIFO control
  74  */
  75
  76 static int r128_do_pixcache_flush(drm_r128_private_t * dev_priv)
  77 {
  78         u32 tmp;
  79         int i;
  80
  81         tmp = R128_READ(R128_PC_NGUI_CTLSTAT) | R128_PC_FLUSH_ALL;
  82         R128_WRITE(R128_PC_NGUI_CTLSTAT, tmp);
  83
  84         for (i = 0; i < dev_priv->usec_timeout; i++) {
  85                 if (!(R128_READ(R128_PC_NGUI_CTLSTAT) & R128_PC_BUSY)) {
  86                         return 0;
  87                 }
  88                 DRM_UDELAY(1);
  89         }
  90
  91 #if R128_FIFO_DEBUG
  92         DRM_ERROR("failed!\n");
  93 #endif
  94         return -EBUSY;
  95 }
  96
  97 static int r128_do_wait_for_fifo(drm_r128_private_t * dev_priv, int entries)
  98 {
  99         int i;
 100
 101         for (i = 0; i < dev_priv->usec_timeout; i++) {
 102                 int slots = R128_READ(R128_GUI_STAT) & R128_GUI_FIFOCNT_MASK;
 103                 if (slots >= entries)
 104                         return 0;
 105                 DRM_UDELAY(1);
 106         }
 107
 108 #if R128_FIFO_DEBUG
 109         DRM_ERROR("failed!\n");
 110 #endif
 111         return -EBUSY;
 112 }
 113
 114 static int r128_do_wait_for_idle(drm_r128_private_t * dev_priv)
 115 {
 116         int i, ret;
 117
 118         ret = r128_do_wait_for_fifo(dev_priv, 64);
 119         if (ret)
 120                 return ret;
 121
 122         for (i = 0; i < dev_priv->usec_timeout; i++) {
 123                 if (!(R128_READ(R128_GUI_STAT) & R128_GUI_ACTIVE)) {
 124                         r128_do_pixcache_flush(dev_priv);
 125                         return 0;
 126                 }
 127                 DRM_UDELAY(1);
 128         }
 129
 130 #if R128_FIFO_DEBUG
 131         DRM_ERROR("failed!\n");
 132 #endif
 133         return -EBUSY;
 134 }
 135
 136 /* ================================================================
 137  * CCE control, initialization
 138  */
 139
 140 /* Load the microcode for the CCE */
 141 static int r128_cce_load_microcode(drm_r128_private_t *dev_priv)
 142 {
 143         struct platform_device *pdev;
 144         const struct firmware *fw;
 145         const __be32 *fw_data;
 146         int rc, i;
 147
 148         DRM_DEBUG("\n");
 149
 150         pdev = platform_device_register_simple("r128_cce", 0, NULL, 0);
 151         if (IS_ERR(pdev)) {
 152                 printk(KERN_ERR "r128_cce: Failed to register firmware\n");
 153                 return PTR_ERR(pdev);
 154         }
 155         rc = request_firmware(&fw, FIRMWARE_NAME, &pdev->dev);
 156         platform_device_unregister(pdev);
 157         if (rc) {
 158                 printk(KERN_ERR "r128_cce: Failed to load firmware \"%s\"\n",
 159                        FIRMWARE_NAME);
 160                 return rc;
 161         }
 162
 163         if (fw->size != 256 * 8) {
 164                 printk(KERN_ERR
 165                        "r128_cce: Bogus length %zu in firmware \"%s\"\n",
 166                        fw->size, FIRMWARE_NAME);
 167                 rc = -EINVAL;
 168                 goto out_release;
 169         }
 170
 171         r128_do_wait_for_idle(dev_priv);
 172
 173         fw_data = (const __be32 *)fw->data;
 174         R128_WRITE(R128_PM4_MICROCODE_ADDR, 0);
 175         for (i = 0; i < 256; i++) {
 176                 R128_WRITE(R128_PM4_MICROCODE_DATAH,
 177                            be32_to_cpup(&fw_data[i * 2]));
 178                 R128_WRITE(R128_PM4_MICROCODE_DATAL,
 179                            be32_to_cpup(&fw_data[i * 2 + 1]));
 180         }
 181
 182 out_release:
 183         release_firmware(fw);
 184         return rc;
 185 }
 186
 187 /* Flush any pending commands to the CCE.  This should only be used just
 188  * prior to a wait for idle, as it informs the engine that the command
 189  * stream is ending.
 190  */
 191 static void r128_do_cce_flush(drm_r128_private_t * dev_priv)
 192 {
 193         u32 tmp;
 194
 195         tmp = R128_READ(R128_PM4_BUFFER_DL_WPTR) | R128_PM4_BUFFER_DL_DONE;
 196         R128_WRITE(R128_PM4_BUFFER_DL_WPTR, tmp);
 197 }
 198
 199 /* Wait for the CCE to go idle.
 200  */
 201 int r128_do_cce_idle(drm_r128_private_t * dev_priv)
 202 {
 203         int i;
 204
 205         for (i = 0; i < dev_priv->usec_timeout; i++) {
 206                 if (GET_RING_HEAD(dev_priv) == dev_priv->ring.tail) {
 207                         int pm4stat = R128_READ(R128_PM4_STAT);
 208                         if (((pm4stat & R128_PM4_FIFOCNT_MASK) >=
 209                              dev_priv->cce_fifo_size) &&
 210                             !(pm4stat & (R128_PM4_BUSY |
 211                                          R128_PM4_GUI_ACTIVE))) {
 212                                 return r128_do_pixcache_flush(dev_priv);
 213                         }
 214                 }
 215                 DRM_UDELAY(1);
 216         }
 217
 218 #if R128_FIFO_DEBUG
 219         DRM_ERROR("failed!\n");
 220         r128_status(dev_priv);
 221 #endif
 222         return -EBUSY;
 223 }
 224
 225 /* Start the Concurrent Command Engine.
 226  */
 227 static void r128_do_cce_start(drm_r128_private_t * dev_priv)
 228 {
 229         r128_do_wait_for_idle(dev_priv);
 230
 231         R128_WRITE(R128_PM4_BUFFER_CNTL,
 232                    dev_priv->cce_mode | dev_priv->ring.size_l2qw
 233                    | R128_PM4_BUFFER_CNTL_NOUPDATE);
 234         R128_READ(R128_PM4_BUFFER_ADDR);        /* as per the sample code */
 235         R128_WRITE(R128_PM4_MICRO_CNTL, R128_PM4_MICRO_FREERUN);
 236
 237         dev_priv->cce_running = 1;
 238 }
 239
 240 /* Reset the Concurrent Command Engine.  This will not flush any pending
 241  * commands, so you must wait for the CCE command stream to complete
 242  * before calling this routine.
 243  */
 244 static void r128_do_cce_reset(drm_r128_private_t * dev_priv)
 245 {
 246         R128_WRITE(R128_PM4_BUFFER_DL_WPTR, 0);
 247         R128_WRITE(R128_PM4_BUFFER_DL_RPTR, 0);
 248         dev_priv->ring.tail = 0;
 249 }
 250
 251 /* Stop the Concurrent Command Engine.  This will not flush any pending
 252  * commands, so you must flush the command stream and wait for the CCE
 253  * to go idle before calling this routine.
 254  */
 255 static void r128_do_cce_stop(drm_r128_private_t * dev_priv)
 256 {
 257         R128_WRITE(R128_PM4_MICRO_CNTL, 0);
 258         R128_WRITE(R128_PM4_BUFFER_CNTL,
 259                    R128_PM4_NONPM4 | R128_PM4_BUFFER_CNTL_NOUPDATE);
 260
 261         dev_priv->cce_running = 0;
 262 }
 263
 264 /* Reset the engine.  This will stop the CCE if it is running.
 265  */
 266 static int r128_do_engine_reset(struct drm_device * dev)
 267 {
 268         drm_r128_private_t *dev_priv = dev->dev_private;
 269         u32 clock_cntl_index, mclk_cntl, gen_reset_cntl;
 270
 271         r128_do_pixcache_flush(dev_priv);
 272
 273         clock_cntl_index = R128_READ(R128_CLOCK_CNTL_INDEX);
 274         mclk_cntl = R128_READ_PLL(dev, R128_MCLK_CNTL);
 275
 276         R128_WRITE_PLL(R128_MCLK_CNTL,
 277                        mclk_cntl | R128_FORCE_GCP | R128_FORCE_PIPE3D_CP);
 278
 279         gen_reset_cntl = R128_READ(R128_GEN_RESET_CNTL);
 280
 281         /* Taken from the sample code - do not change */
 282         R128_WRITE(R128_GEN_RESET_CNTL, gen_reset_cntl | R128_SOFT_RESET_GUI);
 283         R128_READ(R128_GEN_RESET_CNTL);
 284         R128_WRITE(R128_GEN_RESET_CNTL, gen_reset_cntl & ~R128_SOFT_RESET_GUI);
 285         R128_READ(R128_GEN_RESET_CNTL);
 286
 287         R128_WRITE_PLL(R128_MCLK_CNTL, mclk_cntl);
 288         R128_WRITE(R128_CLOCK_CNTL_INDEX, clock_cntl_index);
 289         R128_WRITE(R128_GEN_RESET_CNTL, gen_reset_cntl);
 290
 291         /* Reset the CCE ring */
 292         r128_do_cce_reset(dev_priv);
 293
 294         /* The CCE is no longer running after an engine reset */
 295         dev_priv->cce_running = 0;
 296
 297         /* Reset any pending vertex, indirect buffers */
 298         r128_freelist_reset(dev);
 299
 300         return 0;
 301 }
 302
 303 static void r128_cce_init_ring_buffer(struct drm_device * dev,
 304                                       drm_r128_private_t * dev_priv)
 305 {
 306         u32 ring_start;
 307         u32 tmp;
 308
 309         DRM_DEBUG("\n");
 310
 311         /* The manual (p. 2) says this address is in "VM space".  This
 312          * means it's an offset from the start of AGP space.
 313          */
 314 #if __OS_HAS_AGP
 315         if (!dev_priv->is_pci)
 316                 ring_start = dev_priv->cce_ring->offset - dev->agp->base;
 317         else
 318 #endif
 319                 ring_start = dev_priv->cce_ring->offset -
 320                     (unsigned long)dev->sg->virtual;
 321
 322         R128_WRITE(R128_PM4_BUFFER_OFFSET, ring_start | R128_AGP_OFFSET);
 323
 324         R128_WRITE(R128_PM4_BUFFER_DL_WPTR, 0);
 325         R128_WRITE(R128_PM4_BUFFER_DL_RPTR, 0);
 326
 327         /* Set watermark control */
 328         R128_WRITE(R128_PM4_BUFFER_WM_CNTL,
 329                    ((R128_WATERMARK_L / 4) << R128_WMA_SHIFT)
 330                    | ((R128_WATERMARK_M / 4) << R128_WMB_SHIFT)
 331                    | ((R128_WATERMARK_N / 4) << R128_WMC_SHIFT)
 332                    | ((R128_WATERMARK_K / 64) << R128_WB_WM_SHIFT));
 333
 334         /* Force read.  Why?  Because it's in the examples... */
 335         R128_READ(R128_PM4_BUFFER_ADDR);
 336
 337         /* Turn on bus mastering */
 338         tmp = R128_READ(R128_BUS_CNTL) & ~R128_BUS_MASTER_DIS;
 339         R128_WRITE(R128_BUS_CNTL, tmp);
 340 }
 341
 342 static int r128_do_init_cce(struct drm_device * dev, drm_r128_init_t * init)
 343 {
 344         drm_r128_private_t *dev_priv;
 345         int rc;
 346
 347         DRM_DEBUG("\n");
 348
 349         dev_priv = kzalloc(sizeof(drm_r128_private_t), GFP_KERNEL);
 350         if (dev_priv == NULL)
 351                 return -ENOMEM;
 352
 353         dev_priv->is_pci = init->is_pci;
 354
 355         if (dev_priv->is_pci && !dev->sg) {
 356                 DRM_ERROR("PCI GART memory not allocated!\n");
 357                 dev->dev_private = (void *)dev_priv;
 358                 r128_do_cleanup_cce(dev);
 359                 return -EINVAL;
 360         }
 361
 362         dev_priv->usec_timeout = init->usec_timeout;
 363         if (dev_priv->usec_timeout < 1 ||
 364             dev_priv->usec_timeout > R128_MAX_USEC_TIMEOUT) {
 365                 DRM_DEBUG("TIMEOUT problem!\n");
 366                 dev->dev_private = (void *)dev_priv;
 367                 r128_do_cleanup_cce(dev);
 368                 return -EINVAL;
 369         }
 370
 371         dev_priv->cce_mode = init->cce_mode;
 372
 373         /* GH: Simple idle check.
 374          */
 375         atomic_set(&dev_priv->idle_count, 0);
 376
 377         /* We don't support anything other than bus-mastering ring mode,
 378          * but the ring can be in either AGP or PCI space for the ring
 379          * read pointer.
 380          */
 381         if ((init->cce_mode != R128_PM4_192BM) &&
 382             (init->cce_mode != R128_PM4_128BM_64INDBM) &&
 383             (init->cce_mode != R128_PM4_64BM_128INDBM) &&
 384             (init->cce_mode != R128_PM4_64BM_64VCBM_64INDBM)) {
 385                 DRM_DEBUG("Bad cce_mode!\n");
 386                 dev->dev_private = (void *)dev_priv;
 387                 r128_do_cleanup_cce(dev);
 388                 return -EINVAL;
 389         }
 390
 391         switch (init->cce_mode) {
 392         case R128_PM4_NONPM4:
 393                 dev_priv->cce_fifo_size = 0;
 394                 break;
 395         case R128_PM4_192PIO:
 396         case R128_PM4_192BM:
 397                 dev_priv->cce_fifo_size = 192;
 398                 break;
 399         case R128_PM4_128PIO_64INDBM:
 400         case R128_PM4_128BM_64INDBM:
 401                 dev_priv->cce_fifo_size = 128;
 402                 break;
 403         case R128_PM4_64PIO_128INDBM:
 404         case R128_PM4_64BM_128INDBM:
 405         case R128_PM4_64PIO_64VCBM_64INDBM:
 406         case R128_PM4_64BM_64VCBM_64INDBM:
 407         case R128_PM4_64PIO_64VCPIO_64INDPIO:
 408                 dev_priv->cce_fifo_size = 64;
 409                 break;
 410         }
 411
 412         switch (init->fb_bpp) {
 413         case 16:
 414                 dev_priv->color_fmt = R128_DATATYPE_RGB565;
 415                 break;
 416         case 32:
 417         default:
 418                 dev_priv->color_fmt = R128_DATATYPE_ARGB8888;
 419                 break;
 420         }
 421         dev_priv->front_offset = init->front_offset;
 422         dev_priv->front_pitch = init->front_pitch;
 423         dev_priv->back_offset = init->back_offset;
 424         dev_priv->back_pitch = init->back_pitch;
 425
 426         switch (init->depth_bpp) {
 427         case 16:
 428                 dev_priv->depth_fmt = R128_DATATYPE_RGB565;
 429                 break;
 430         case 24:
 431         case 32:
 432         default:
 433                 dev_priv->depth_fmt = R128_DATATYPE_ARGB8888;
 434                 break;
 435         }
 436         dev_priv->depth_offset = init->depth_offset;
 437         dev_priv->depth_pitch = init->depth_pitch;
 438         dev_priv->span_offset = init->span_offset;
 439
 440         dev_priv->front_pitch_offset_c = (((dev_priv->front_pitch / 8) << 21) |
 441                                           (dev_priv->front_offset >> 5));
 442         dev_priv->back_pitch_offset_c = (((dev_priv->back_pitch / 8) << 21) |
 443                                          (dev_priv->back_offset >> 5));
 444         dev_priv->depth_pitch_offset_c = (((dev_priv->depth_pitch / 8) << 21) |
 445                                           (dev_priv->depth_offset >> 5) |
 446                                           R128_DST_TILE);
 447         dev_priv->span_pitch_offset_c = (((dev_priv->depth_pitch / 8) << 21) |
 448                                          (dev_priv->span_offset >> 5));
 449
 450         dev_priv->sarea = drm_getsarea(dev);
 451         if (!dev_priv->sarea) {
 452                 DRM_ERROR("could not find sarea!\n");
 453                 dev->dev_private = (void *)dev_priv;
 454                 r128_do_cleanup_cce(dev);
 455                 return -EINVAL;
 456         }
 457
 458         dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
 459         if (!dev_priv->mmio) {
 460                 DRM_ERROR("could not find mmio region!\n");
 461                 dev->dev_private = (void *)dev_priv;
 462                 r128_do_cleanup_cce(dev);
 463                 return -EINVAL;
 464         }
 465         dev_priv->cce_ring = drm_core_findmap(dev, init->ring_offset);
 466         if (!dev_priv->cce_ring) {
 467                 DRM_ERROR("could not find cce ring region!\n");
 468                 dev->dev_private = (void *)dev_priv;
 469                 r128_do_cleanup_cce(dev);
 470                 return -EINVAL;
 471         }
 472         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
 473         if (!dev_priv->ring_rptr) {
 474                 DRM_ERROR("could not find ring read pointer!\n");
 475                 dev->dev_private = (void *)dev_priv;
 476                 r128_do_cleanup_cce(dev);
 477                 return -EINVAL;
 478         }
 479         dev->agp_buffer_token = init->buffers_offset;
 480         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
 481         if (!dev->agp_buffer_map) {
 482                 DRM_ERROR("could not find dma buffer region!\n");
 483                 dev->dev_private = (void *)dev_priv;
 484                 r128_do_cleanup_cce(dev);
 485                 return -EINVAL;
 486         }
 487
 488         if (!dev_priv->is_pci) {
 489                 dev_priv->agp_textures =
 490                     drm_core_findmap(dev, init->agp_textures_offset);
 491                 if (!dev_priv->agp_textures) {
 492                         DRM_ERROR("could not find agp texture region!\n");
 493                         dev->dev_private = (void *)dev_priv;
 494                         r128_do_cleanup_cce(dev);
 495                         return -EINVAL;
 496                 }
 497         }
 498
 499         dev_priv->sarea_priv =
 500             (drm_r128_sarea_t *) ((u8 *) dev_priv->sarea->handle +
 501                                   init->sarea_priv_offset);
 502
 503 #if __OS_HAS_AGP
 504         if (!dev_priv->is_pci) {
 505                 drm_core_ioremap_wc(dev_priv->cce_ring, dev);
 506                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
 507                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
 508                 if (!dev_priv->cce_ring->handle ||
 509                     !dev_priv->ring_rptr->handle ||
 510                     !dev->agp_buffer_map->handle) {
 511                         DRM_ERROR("Could not ioremap agp regions!\n");
 512                         dev->dev_private = (void *)dev_priv;
 513                         r128_do_cleanup_cce(dev);
 514                         return -ENOMEM;
 515                 }
 516         } else
 517 #endif
 518         {
 519                 dev_priv->cce_ring->handle =
 520                         (void *)(unsigned long)dev_priv->cce_ring->offset;
 521                 dev_priv->ring_rptr->handle =
 522                         (void *)(unsigned long)dev_priv->ring_rptr->offset;
 523                 dev->agp_buffer_map->handle =
 524                         (void *)(unsigned long)dev->agp_buffer_map->offset;
 525         }
 526
 527 #if __OS_HAS_AGP
 528         if (!dev_priv->is_pci)
 529                 dev_priv->cce_buffers_offset = dev->agp->base;
 530         else
 531 #endif
 532                 dev_priv->cce_buffers_offset = (unsigned long)dev->sg->virtual;
 533
 534         dev_priv->ring.start = (u32 *) dev_priv->cce_ring->handle;
 535         dev_priv->ring.end = ((u32 *) dev_priv->cce_ring->handle
 536                               + init->ring_size / sizeof(u32));
 537         dev_priv->ring.size = init->ring_size;
 538         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
 539
 540         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
 541
 542         dev_priv->ring.high_mark = 128;
 543
 544         dev_priv->sarea_priv->last_frame = 0;
 545         R128_WRITE(R128_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame);
 546
 547         dev_priv->sarea_priv->last_dispatch = 0;
 548         R128_WRITE(R128_LAST_DISPATCH_REG, dev_priv->sarea_priv->last_dispatch);
 549
 550 #if __OS_HAS_AGP
 551         if (dev_priv->is_pci) {
 552 #endif
 553                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
 554                 dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
 555                 dev_priv->gart_info.table_size = R128_PCIGART_TABLE_SIZE;
 556                 dev_priv->gart_info.addr = NULL;
 557                 dev_priv->gart_info.bus_addr = 0;
 558                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
 559                 if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
 560                         DRM_ERROR("failed to init PCI GART!\n");
 561                         dev->dev_private = (void *)dev_priv;
 562                         r128_do_cleanup_cce(dev);
 563                         return -ENOMEM;
 564                 }
 565                 R128_WRITE(R128_PCI_GART_PAGE, dev_priv->gart_info.bus_addr);
 566 #if __OS_HAS_AGP
 567         }
 568 #endif
 569
 570         r128_cce_init_ring_buffer(dev, dev_priv);
 571         rc = r128_cce_load_microcode(dev_priv);
 572
 573         dev->dev_private = (void *)dev_priv;
 574
 575         r128_do_engine_reset(dev);
 576
 577         if (rc) {
 578                 DRM_ERROR("Failed to load firmware!\n");
 579                 r128_do_cleanup_cce(dev);
 580         }
 581
 582         return rc;
 583 }
 584
 585 int r128_do_cleanup_cce(struct drm_device * dev)
 586 {
 587
 588         /* Make sure interrupts are disabled here because the uninstall ioctl
 589          * may not have been called from userspace and after dev_private
 590          * is freed, it's too late.
 591          */
 592         if (dev->irq_enabled)
 593                 drm_irq_uninstall(dev);
 594
 595         if (dev->dev_private) {
 596                 drm_r128_private_t *dev_priv = dev->dev_private;
 597
 598 #if __OS_HAS_AGP
 599                 if (!dev_priv->is_pci) {
 600                         if (dev_priv->cce_ring != NULL)
 601                                 drm_core_ioremapfree(dev_priv->cce_ring, dev);
 602                         if (dev_priv->ring_rptr != NULL)
 603                                 drm_core_ioremapfree(dev_priv->ring_rptr, dev);
 604                         if (dev->agp_buffer_map != NULL) {
 605                                 drm_core_ioremapfree(dev->agp_buffer_map, dev);
 606                                 dev->agp_buffer_map = NULL;
 607                         }
 608                 } else
 609 #endif
 610                 {
 611                         if (dev_priv->gart_info.bus_addr)
 612                                 if (!drm_ati_pcigart_cleanup(dev,
 613                                                         &dev_priv->gart_info))
 614                                         DRM_ERROR
 615                                             ("failed to cleanup PCI GART!\n");
 616                 }
 617
 618                 kfree(dev->dev_private);
 619                 dev->dev_private = NULL;
 620         }
 621
 622         return 0;
 623 }
 624
 625 int r128_cce_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 626 {
 627         drm_r128_init_t *init = data;
 628
 629         DRM_DEBUG("\n");
 630
 631         LOCK_TEST_WITH_RETURN(dev, file_priv);
 632
 633         switch (init->func) {
 634         case R128_INIT_CCE:
 635                 return r128_do_init_cce(dev, init);
 636         case R128_CLEANUP_CCE:
 637                 return r128_do_cleanup_cce(dev);
 638         }
 639
 640         return -EINVAL;
 641 }
 642
 643 int r128_cce_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
 644 {
 645         drm_r128_private_t *dev_priv = dev->dev_private;
 646         DRM_DEBUG("\n");
 647
 648         LOCK_TEST_WITH_RETURN(dev, file_priv);
 649
 650         if (dev_priv->cce_running || dev_priv->cce_mode == R128_PM4_NONPM4) {
 651                 DRM_DEBUG("while CCE running\n");
 652                 return 0;
 653         }
 654
 655         r128_do_cce_start(dev_priv);
 656
 657         return 0;
 658 }
 659
 660 /* Stop the CCE.  The engine must have been idled before calling this
 661  * routine.
 662  */
 663 int r128_cce_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
 664 {
 665         drm_r128_private_t *dev_priv = dev->dev_private;
 666         drm_r128_cce_stop_t *stop = data;
 667         int ret;
 668         DRM_DEBUG("\n");
 669
 670         LOCK_TEST_WITH_RETURN(dev, file_priv);
 671
 672         /* Flush any pending CCE commands.  This ensures any outstanding
 673          * commands are exectuted by the engine before we turn it off.
 674          */
 675         if (stop->flush) {
 676                 r128_do_cce_flush(dev_priv);
 677         }
 678
 679         /* If we fail to make the engine go idle, we return an error
 680          * code so that the DRM ioctl wrapper can try again.
 681          */
 682         if (stop->idle) {
 683                 ret = r128_do_cce_idle(dev_priv);
 684                 if (ret)
 685                         return ret;
 686         }
 687
 688         /* Finally, we can turn off the CCE.  If the engine isn't idle,
 689          * we will get some dropped triangles as they won't be fully
 690          * rendered before the CCE is shut down.
 691          */
 692         r128_do_cce_stop(dev_priv);
 693
 694         /* Reset the engine */
 695         r128_do_engine_reset(dev);
 696
 697         return 0;
 698 }
 699
 700 /* Just reset the CCE ring.  Called as part of an X Server engine reset.
 701  */
 702 int r128_cce_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 703 {
 704         drm_r128_private_t *dev_priv = dev->dev_private;
 705         DRM_DEBUG("\n");
 706
 707         LOCK_TEST_WITH_RETURN(dev, file_priv);
 708
 709         if (!dev_priv) {
 710                 DRM_DEBUG("called before init done\n");
 711                 return -EINVAL;
 712         }
 713
 714         r128_do_cce_reset(dev_priv);
 715
 716         /* The CCE is no longer running after an engine reset */
 717         dev_priv->cce_running = 0;
 718
 719         return 0;
 720 }
 721
 722 int r128_cce_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
 723 {
 724         drm_r128_private_t *dev_priv = dev->dev_private;
 725         DRM_DEBUG("\n");
 726
 727         LOCK_TEST_WITH_RETURN(dev, file_priv);
 728
 729         if (dev_priv->cce_running) {
 730                 r128_do_cce_flush(dev_priv);
 731         }
 732
 733         return r128_do_cce_idle(dev_priv);
 734 }
 735
 736 int r128_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 737 {
 738         DRM_DEBUG("\n");
 739
 740         LOCK_TEST_WITH_RETURN(dev, file_priv);
 741
 742         return r128_do_engine_reset(dev);
 743 }
 744
 745 int r128_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
 746 {
 747         return -EINVAL;
 748 }
 749
 750 /* ================================================================
 751  * Freelist management
 752  */
 753 #define R128_BUFFER_USED        0xffffffff
 754 #define R128_BUFFER_FREE        0
 755
 756 #if 0
 757 static int r128_freelist_init(struct drm_device * dev)
 758 {
 759         struct drm_device_dma *dma = dev->dma;
 760         drm_r128_private_t *dev_priv = dev->dev_private;
 761         struct drm_buf *buf;
 762         drm_r128_buf_priv_t *buf_priv;
 763         drm_r128_freelist_t *entry;
 764         int i;
 765
 766         dev_priv->head = kzalloc(sizeof(drm_r128_freelist_t), GFP_KERNEL);
 767         if (dev_priv->head == NULL)
 768                 return -ENOMEM;
 769
 770         dev_priv->head->age = R128_BUFFER_USED;
 771
 772         for (i = 0; i < dma->buf_count; i++) {
 773                 buf = dma->buflist[i];
 774                 buf_priv = buf->dev_private;
 775
 776                 entry = kmalloc(sizeof(drm_r128_freelist_t), GFP_KERNEL);
 777                 if (!entry)
 778                         return -ENOMEM;
 779
 780                 entry->age = R128_BUFFER_FREE;
 781                 entry->buf = buf;
 782                 entry->prev = dev_priv->head;
 783                 entry->next = dev_priv->head->next;
 784                 if (!entry->next)
 785                         dev_priv->tail = entry;
 786
 787                 buf_priv->discard = 0;
 788                 buf_priv->dispatched = 0;
 789                 buf_priv->list_entry = entry;
 790
 791                 dev_priv->head->next = entry;
 792
 793                 if (dev_priv->head->next)
 794                         dev_priv->head->next->prev = entry;
 795         }
 796
 797         return 0;
 798
 799 }
 800 #endif
 801
 802 static struct drm_buf *r128_freelist_get(struct drm_device * dev)
 803 {
 804         struct drm_device_dma *dma = dev->dma;
 805         drm_r128_private_t *dev_priv = dev->dev_private;
 806         drm_r128_buf_priv_t *buf_priv;
 807         struct drm_buf *buf;
 808         int i, t;
 809
 810         /* FIXME: Optimize -- use freelist code */
 811
 812         for (i = 0; i < dma->buf_count; i++) {
 813                 buf = dma->buflist[i];
 814                 buf_priv = buf->dev_private;
 815                 if (!buf->file_priv)
 816                         return buf;
 817         }
 818
 819         for (t = 0; t < dev_priv->usec_timeout; t++) {
 820                 u32 done_age = R128_READ(R128_LAST_DISPATCH_REG);
 821
 822                 for (i = 0; i < dma->buf_count; i++) {
 823                         buf = dma->buflist[i];
 824                         buf_priv = buf->dev_private;
 825                         if (buf->pending && buf_priv->age <= done_age) {
 826                                 /* The buffer has been processed, so it
 827                                  * can now be used.
 828                                  */
 829                                 buf->pending = 0;
 830                                 return buf;
 831                         }
 832                 }
 833                 DRM_UDELAY(1);
 834         }
 835
 836         DRM_DEBUG("returning NULL!\n");
 837         return NULL;
 838 }
 839
 840 void r128_freelist_reset(struct drm_device * dev)
 841 {
 842         struct drm_device_dma *dma = dev->dma;
 843         int i;
 844
 845         for (i = 0; i < dma->buf_count; i++) {
 846                 struct drm_buf *buf = dma->buflist[i];
 847                 drm_r128_buf_priv_t *buf_priv = buf->dev_private;
 848                 buf_priv->age = 0;
 849         }
 850 }
 851
 852 /* ================================================================
 853  * CCE command submission
 854  */
 855
 856 int r128_wait_ring(drm_r128_private_t * dev_priv, int n)
 857 {
 858         drm_r128_ring_buffer_t *ring = &dev_priv->ring;
 859         int i;
 860
 861         for (i = 0; i < dev_priv->usec_timeout; i++) {
 862                 r128_update_ring_snapshot(dev_priv);
 863                 if (ring->space >= n)
 864                         return 0;
 865                 DRM_UDELAY(1);
 866         }
 867
 868         /* FIXME: This is being ignored... */
 869         DRM_ERROR("failed!\n");
 870         return -EBUSY;
 871 }
 872
 873 static int r128_cce_get_buffers(struct drm_device * dev,
 874                                 struct drm_file *file_priv,
 875                                 struct drm_dma * d)
 876 {
 877         int i;
 878         struct drm_buf *buf;
 879
 880         for (i = d->granted_count; i < d->request_count; i++) {
 881                 buf = r128_freelist_get(dev);
 882                 if (!buf)
 883                         return -EAGAIN;
 884
 885                 buf->file_priv = file_priv;
 886
 887                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
 888                                      sizeof(buf->idx)))
 889                         return -EFAULT;
 890                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
 891                                      sizeof(buf->total)))
 892                         return -EFAULT;
 893
 894                 d->granted_count++;
 895         }
 896         return 0;
 897 }
 898
 899 int r128_cce_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
 900 {
 901         struct drm_device_dma *dma = dev->dma;
 902         int ret = 0;
 903         struct drm_dma *d = data;
 904
 905         LOCK_TEST_WITH_RETURN(dev, file_priv);
 906
 907         /* Please don't send us buffers.
 908          */
 909         if (d->send_count != 0) {
 910                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
 911                           DRM_CURRENTPID, d->send_count);
 912                 return -EINVAL;
 913         }
 914
 915         /* We'll send you buffers.
 916          */
 917         if (d->request_count < 0 || d->request_count > dma->buf_count) {
 918                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
 919                           DRM_CURRENTPID, d->request_count, dma->buf_count);
 920                 return -EINVAL;
 921         }
 922
 923         d->granted_count = 0;
 924
 925         if (d->request_count) {
 926                 ret = r128_cce_get_buffers(dev, file_priv, d);
 927         }
 928
 929         return ret;
 930 }