SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/slab.h>
  29 #include <linux/seq_file.h>
  30 #include <linux/firmware.h>
  31 #include <linux/platform_device.h>
  32 #include "drmP.h"
  33 #include "radeon_drm.h"
  34 #include "radeon.h"
  35 #include "radeon_asic.h"
  36 #include "radeon_mode.h"
  37 #include "r600d.h"
  38 #include "atom.h"
  39 #include "avivod.h"
  40
  41 #define PFP_UCODE_SIZE 576
  42 #define PM4_UCODE_SIZE 1792
  43 #define RLC_UCODE_SIZE 768
  44 #define R700_PFP_UCODE_SIZE 848
  45 #define R700_PM4_UCODE_SIZE 1360
  46 #define R700_RLC_UCODE_SIZE 1024
  47 #define EVERGREEN_PFP_UCODE_SIZE 1120
  48 #define EVERGREEN_PM4_UCODE_SIZE 1376
  49 #define EVERGREEN_RLC_UCODE_SIZE 768
  50
  51 /* Firmware Names */
  52 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  53 MODULE_FIRMWARE("radeon/R600_me.bin");
  54 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  55 MODULE_FIRMWARE("radeon/RV610_me.bin");
  56 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  57 MODULE_FIRMWARE("radeon/RV630_me.bin");
  58 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  59 MODULE_FIRMWARE("radeon/RV620_me.bin");
  60 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  61 MODULE_FIRMWARE("radeon/RV635_me.bin");
  62 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  63 MODULE_FIRMWARE("radeon/RV670_me.bin");
  64 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  65 MODULE_FIRMWARE("radeon/RS780_me.bin");
  66 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  67 MODULE_FIRMWARE("radeon/RV770_me.bin");
  68 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  69 MODULE_FIRMWARE("radeon/RV730_me.bin");
  70 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  71 MODULE_FIRMWARE("radeon/RV710_me.bin");
  72 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  73 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  74 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  75 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  76 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
  77 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  78 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  79 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
  80 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  81 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  82 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
  83 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
  84 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  85 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
  86
  87 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
  88
  89 /* r600,rv610,rv630,rv620,rv635,rv670 */
  90 int r600_mc_wait_for_idle(struct radeon_device *rdev);
  91 void r600_gpu_init(struct radeon_device *rdev);
  92 void r600_fini(struct radeon_device *rdev);
  93 void r600_irq_disable(struct radeon_device *rdev);
  94
  95 void r600_get_power_state(struct radeon_device *rdev,
  96                           enum radeon_pm_action action)
  97 {
  98         int i;
  99
 100         rdev->pm.can_upclock = true;
 101         rdev->pm.can_downclock = true;
 102
 103         /* power state array is low to high, default is first */
 104         if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
 105                 int min_power_state_index = 0;
 106
 107                 if (rdev->pm.num_power_states > 2)
 108                         min_power_state_index = 1;
 109
 110                 switch (action) {
 111                 case PM_ACTION_MINIMUM:
 112                         rdev->pm.requested_power_state_index = min_power_state_index;
 113                         rdev->pm.requested_clock_mode_index = 0;
 114                         rdev->pm.can_downclock = false;
 115                         break;
 116                 case PM_ACTION_DOWNCLOCK:
 117                         if (rdev->pm.current_power_state_index == min_power_state_index) {
 118                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 119                                 rdev->pm.can_downclock = false;
 120                         } else {
 121                                 if (rdev->pm.active_crtc_count > 1) {
 122                                         for (i = 0; i < rdev->pm.num_power_states; i++) {
 123                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_SINGLE_DISPLAY_ONLY)
 124                                                         continue;
 125                                                 else if (i >= rdev->pm.current_power_state_index) {
 126                                                         rdev->pm.requested_power_state_index =
 127                                                                 rdev->pm.current_power_state_index;
 128                                                         break;
 129                                                 } else {
 130                                                         rdev->pm.requested_power_state_index = i;
 131                                                         break;
 132                                                 }
 133                                         }
 134                                 } else
 135                                         rdev->pm.requested_power_state_index =
 136                                                 rdev->pm.current_power_state_index - 1;
 137                         }
 138                         rdev->pm.requested_clock_mode_index = 0;
 139                         break;
 140                 case PM_ACTION_UPCLOCK:
 141                         if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
 142                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 143                                 rdev->pm.can_upclock = false;
 144                         } else {
 145                                 if (rdev->pm.active_crtc_count > 1) {
 146                                         for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
 147                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_SINGLE_DISPLAY_ONLY)
 148                                                         continue;
 149                                                 else if (i <= rdev->pm.current_power_state_index) {
 150                                                         rdev->pm.requested_power_state_index =
 151                                                                 rdev->pm.current_power_state_index;
 152                                                         break;
 153                                                 } else {
 154                                                         rdev->pm.requested_power_state_index = i;
 155                                                         break;
 156                                                 }
 157                                         }
 158                                 } else
 159                                         rdev->pm.requested_power_state_index =
 160                                                 rdev->pm.current_power_state_index + 1;
 161                         }
 162                         rdev->pm.requested_clock_mode_index = 0;
 163                         break;
 164                 case PM_ACTION_DEFAULT:
 165                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 166                         rdev->pm.requested_clock_mode_index = 0;
 167                         rdev->pm.can_upclock = false;
 168                         break;
 169                 case PM_ACTION_NONE:
 170                 default:
 171                         DRM_ERROR("Requested mode for not defined action\n");
 172                         return;
 173                 }
 174         } else {
 175                 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
 176                 /* for now just select the first power state and switch between clock modes */
 177                 /* power state array is low to high, default is first (0) */
 178                 if (rdev->pm.active_crtc_count > 1) {
 179                         rdev->pm.requested_power_state_index = -1;
 180                         /* start at 1 as we don't want the default mode */
 181                         for (i = 1; i < rdev->pm.num_power_states; i++) {
 182                                 if (rdev->pm.power_state[i].flags & RADEON_PM_SINGLE_DISPLAY_ONLY)
 183                                         continue;
 184                                 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
 185                                          (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
 186                                         rdev->pm.requested_power_state_index = i;
 187                                         break;
 188                                 }
 189                         }
 190                         /* if nothing selected, grab the default state. */
 191                         if (rdev->pm.requested_power_state_index == -1)
 192                                 rdev->pm.requested_power_state_index = 0;
 193                 } else
 194                         rdev->pm.requested_power_state_index = 1;
 195
 196                 switch (action) {
 197                 case PM_ACTION_MINIMUM:
 198                         rdev->pm.requested_clock_mode_index = 0;
 199                         rdev->pm.can_downclock = false;
 200                         break;
 201                 case PM_ACTION_DOWNCLOCK:
 202                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 203                                 if (rdev->pm.current_clock_mode_index == 0) {
 204                                         rdev->pm.requested_clock_mode_index = 0;
 205                                         rdev->pm.can_downclock = false;
 206                                 } else
 207                                         rdev->pm.requested_clock_mode_index =
 208                                                 rdev->pm.current_clock_mode_index - 1;
 209                         } else {
 210                                 rdev->pm.requested_clock_mode_index = 0;
 211                                 rdev->pm.can_downclock = false;
 212                         }
 213                         break;
 214                 case PM_ACTION_UPCLOCK:
 215                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 216                                 if (rdev->pm.current_clock_mode_index ==
 217                                     (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
 218                                         rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
 219                                         rdev->pm.can_upclock = false;
 220                                 } else
 221                                         rdev->pm.requested_clock_mode_index =
 222                                                 rdev->pm.current_clock_mode_index + 1;
 223                         } else {
 224                                 rdev->pm.requested_clock_mode_index =
 225                                         rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
 226                                 rdev->pm.can_upclock = false;
 227                         }
 228                         break;
 229                 case PM_ACTION_DEFAULT:
 230                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 231                         rdev->pm.requested_clock_mode_index = 0;
 232                         rdev->pm.can_upclock = false;
 233                         break;
 234                 case PM_ACTION_NONE:
 235                 default:
 236                         DRM_ERROR("Requested mode for not defined action\n");
 237                         return;
 238                 }
 239         }
 240
 241         DRM_INFO("Requested: e: %d m: %d p: %d\n",
 242                  rdev->pm.power_state[rdev->pm.requested_power_state_index].
 243                  clock_info[rdev->pm.requested_clock_mode_index].sclk,
 244                  rdev->pm.power_state[rdev->pm.requested_power_state_index].
 245                  clock_info[rdev->pm.requested_clock_mode_index].mclk,
 246                  rdev->pm.power_state[rdev->pm.requested_power_state_index].
 247                  pcie_lanes);
 248 }
 249
 250 void r600_set_power_state(struct radeon_device *rdev)
 251 {
 252         u32 sclk, mclk;
 253
 254         if ((rdev->pm.requested_clock_mode_index == rdev->pm.current_clock_mode_index) &&
 255             (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index))
 256                 return;
 257
 258         if (radeon_gui_idle(rdev)) {
 259
 260                 sclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
 261                         clock_info[rdev->pm.requested_clock_mode_index].sclk;
 262                 if (sclk > rdev->clock.default_sclk)
 263                         sclk = rdev->clock.default_sclk;
 264
 265                 mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
 266                         clock_info[rdev->pm.requested_clock_mode_index].mclk;
 267                 if (mclk > rdev->clock.default_mclk)
 268                         mclk = rdev->clock.default_mclk;
 269                 /* don't change the mclk with multiple crtcs */
 270                 if (rdev->pm.active_crtc_count > 1)
 271                         mclk = rdev->clock.default_mclk;
 272
 273                 /* set pcie lanes */
 274                 /* TODO */
 275
 276                 /* set voltage */
 277                 /* TODO */
 278
 279                 /* set engine clock */
 280                 if (sclk != rdev->pm.current_sclk) {
 281                         radeon_sync_with_vblank(rdev);
 282                         radeon_pm_debug_check_in_vbl(rdev, false);
 283                         radeon_set_engine_clock(rdev, sclk);
 284                         radeon_pm_debug_check_in_vbl(rdev, true);
 285                         rdev->pm.current_sclk = sclk;
 286                         DRM_INFO("Setting: e: %d\n", sclk);
 287                 }
 288
 289 #if 0
 290                 /* set memory clock */
 291                 if (rdev->asic->set_memory_clock && (mclk != rdev->pm.current_mclk)) {
 292                         radeon_sync_with_vblank(rdev);
 293                         radeon_pm_debug_check_in_vbl(rdev, false);
 294                         radeon_set_memory_clock(rdev, mclk);
 295                         radeon_pm_debug_check_in_vbl(rdev, true);
 296                         rdev->pm.current_mclk = mclk;
 297                         DRM_INFO("Setting: m: %d\n", mclk);
 298                 }
 299 #endif
 300
 301                 rdev->pm.current_power_state_index = rdev->pm.requested_power_state_index;
 302                 rdev->pm.current_clock_mode_index = rdev->pm.requested_clock_mode_index;
 303         } else
 304                 DRM_INFO("GUI not idle!!!\n");
 305 }
 306
 307 bool r600_gui_idle(struct radeon_device *rdev)
 308 {
 309         if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
 310                 return false;
 311         else
 312                 return true;
 313 }
 314
 315 /* hpd for digital panel detect/disconnect */
 316 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 317 {
 318         bool connected = false;
 319
 320         if (ASIC_IS_DCE3(rdev)) {
 321                 switch (hpd) {
 322                 case RADEON_HPD_1:
 323                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 324                                 connected = true;
 325                         break;
 326                 case RADEON_HPD_2:
 327                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 328                                 connected = true;
 329                         break;
 330                 case RADEON_HPD_3:
 331                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 332                                 connected = true;
 333                         break;
 334                 case RADEON_HPD_4:
 335                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 336                                 connected = true;
 337                         break;
 338                         /* DCE 3.2 */
 339                 case RADEON_HPD_5:
 340                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 341                                 connected = true;
 342                         break;
 343                 case RADEON_HPD_6:
 344                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 345                                 connected = true;
 346                         break;
 347                 default:
 348                         break;
 349                 }
 350         } else {
 351                 switch (hpd) {
 352                 case RADEON_HPD_1:
 353                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 354                                 connected = true;
 355                         break;
 356                 case RADEON_HPD_2:
 357                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 358                                 connected = true;
 359                         break;
 360                 case RADEON_HPD_3:
 361                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 362                                 connected = true;
 363                         break;
 364                 default:
 365                         break;
 366                 }
 367         }
 368         return connected;
 369 }
 370
 371 void r600_hpd_set_polarity(struct radeon_device *rdev,
 372                            enum radeon_hpd_id hpd)
 373 {
 374         u32 tmp;
 375         bool connected = r600_hpd_sense(rdev, hpd);
 376
 377         if (ASIC_IS_DCE3(rdev)) {
 378                 switch (hpd) {
 379                 case RADEON_HPD_1:
 380                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 381                         if (connected)
 382                                 tmp &= ~DC_HPDx_INT_POLARITY;
 383                         else
 384                                 tmp |= DC_HPDx_INT_POLARITY;
 385                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 386                         break;
 387                 case RADEON_HPD_2:
 388                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 389                         if (connected)
 390                                 tmp &= ~DC_HPDx_INT_POLARITY;
 391                         else
 392                                 tmp |= DC_HPDx_INT_POLARITY;
 393                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 394                         break;
 395                 case RADEON_HPD_3:
 396                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 397                         if (connected)
 398                                 tmp &= ~DC_HPDx_INT_POLARITY;
 399                         else
 400                                 tmp |= DC_HPDx_INT_POLARITY;
 401                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 402                         break;
 403                 case RADEON_HPD_4:
 404                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 405                         if (connected)
 406                                 tmp &= ~DC_HPDx_INT_POLARITY;
 407                         else
 408                                 tmp |= DC_HPDx_INT_POLARITY;
 409                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 410                         break;
 411                 case RADEON_HPD_5:
 412                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 413                         if (connected)
 414                                 tmp &= ~DC_HPDx_INT_POLARITY;
 415                         else
 416                                 tmp |= DC_HPDx_INT_POLARITY;
 417                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 418                         break;
 419                         /* DCE 3.2 */
 420                 case RADEON_HPD_6:
 421                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 422                         if (connected)
 423                                 tmp &= ~DC_HPDx_INT_POLARITY;
 424                         else
 425                                 tmp |= DC_HPDx_INT_POLARITY;
 426                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 427                         break;
 428                 default:
 429                         break;
 430                 }
 431         } else {
 432                 switch (hpd) {
 433                 case RADEON_HPD_1:
 434                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 435                         if (connected)
 436                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 437                         else
 438                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 439                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 440                         break;
 441                 case RADEON_HPD_2:
 442                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 443                         if (connected)
 444                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 445                         else
 446                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 447                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 448                         break;
 449                 case RADEON_HPD_3:
 450                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 451                         if (connected)
 452                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 453                         else
 454                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 455                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 456                         break;
 457                 default:
 458                         break;
 459                 }
 460         }
 461 }
 462
 463 void r600_hpd_init(struct radeon_device *rdev)
 464 {
 465         struct drm_device *dev = rdev->ddev;
 466         struct drm_connector *connector;
 467
 468         if (ASIC_IS_DCE3(rdev)) {
 469                 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 470                 if (ASIC_IS_DCE32(rdev))
 471                         tmp |= DC_HPDx_EN;
 472
 473                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 474                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 475                         switch (radeon_connector->hpd.hpd) {
 476                         case RADEON_HPD_1:
 477                                 WREG32(DC_HPD1_CONTROL, tmp);
 478                                 rdev->irq.hpd[0] = true;
 479                                 break;
 480                         case RADEON_HPD_2:
 481                                 WREG32(DC_HPD2_CONTROL, tmp);
 482                                 rdev->irq.hpd[1] = true;
 483                                 break;
 484                         case RADEON_HPD_3:
 485                                 WREG32(DC_HPD3_CONTROL, tmp);
 486                                 rdev->irq.hpd[2] = true;
 487                                 break;
 488                         case RADEON_HPD_4:
 489                                 WREG32(DC_HPD4_CONTROL, tmp);
 490                                 rdev->irq.hpd[3] = true;
 491                                 break;
 492                                 /* DCE 3.2 */
 493                         case RADEON_HPD_5:
 494                                 WREG32(DC_HPD5_CONTROL, tmp);
 495                                 rdev->irq.hpd[4] = true;
 496                                 break;
 497                         case RADEON_HPD_6:
 498                                 WREG32(DC_HPD6_CONTROL, tmp);
 499                                 rdev->irq.hpd[5] = true;
 500                                 break;
 501                         default:
 502                                 break;
 503                         }
 504                 }
 505         } else {
 506                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 507                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 508                         switch (radeon_connector->hpd.hpd) {
 509                         case RADEON_HPD_1:
 510                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 511                                 rdev->irq.hpd[0] = true;
 512                                 break;
 513                         case RADEON_HPD_2:
 514                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 515                                 rdev->irq.hpd[1] = true;
 516                                 break;
 517                         case RADEON_HPD_3:
 518                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 519                                 rdev->irq.hpd[2] = true;
 520                                 break;
 521                         default:
 522                                 break;
 523                         }
 524                 }
 525         }
 526         if (rdev->irq.installed)
 527                 r600_irq_set(rdev);
 528 }
 529
 530 void r600_hpd_fini(struct radeon_device *rdev)
 531 {
 532         struct drm_device *dev = rdev->ddev;
 533         struct drm_connector *connector;
 534
 535         if (ASIC_IS_DCE3(rdev)) {
 536                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 537                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 538                         switch (radeon_connector->hpd.hpd) {
 539                         case RADEON_HPD_1:
 540                                 WREG32(DC_HPD1_CONTROL, 0);
 541                                 rdev->irq.hpd[0] = false;
 542                                 break;
 543                         case RADEON_HPD_2:
 544                                 WREG32(DC_HPD2_CONTROL, 0);
 545                                 rdev->irq.hpd[1] = false;
 546                                 break;
 547                         case RADEON_HPD_3:
 548                                 WREG32(DC_HPD3_CONTROL, 0);
 549                                 rdev->irq.hpd[2] = false;
 550                                 break;
 551                         case RADEON_HPD_4:
 552                                 WREG32(DC_HPD4_CONTROL, 0);
 553                                 rdev->irq.hpd[3] = false;
 554                                 break;
 555                                 /* DCE 3.2 */
 556                         case RADEON_HPD_5:
 557                                 WREG32(DC_HPD5_CONTROL, 0);
 558                                 rdev->irq.hpd[4] = false;
 559                                 break;
 560                         case RADEON_HPD_6:
 561                                 WREG32(DC_HPD6_CONTROL, 0);
 562                                 rdev->irq.hpd[5] = false;
 563                                 break;
 564                         default:
 565                                 break;
 566                         }
 567                 }
 568         } else {
 569                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 570                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 571                         switch (radeon_connector->hpd.hpd) {
 572                         case RADEON_HPD_1:
 573                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 574                                 rdev->irq.hpd[0] = false;
 575                                 break;
 576                         case RADEON_HPD_2:
 577                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 578                                 rdev->irq.hpd[1] = false;
 579                                 break;
 580                         case RADEON_HPD_3:
 581                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 582                                 rdev->irq.hpd[2] = false;
 583                                 break;
 584                         default:
 585                                 break;
 586                         }
 587                 }
 588         }
 589 }
 590
 591 /*
 592  * R600 PCIE GART
 593  */
 594 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 595 {
 596         unsigned i;
 597         u32 tmp;
 598
 599         /* flush hdp cache so updates hit vram */
 600         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 601
 602         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 603         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 604         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 605         for (i = 0; i < rdev->usec_timeout; i++) {
 606                 /* read MC_STATUS */
 607                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 608                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 609                 if (tmp == 2) {
 610                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 611                         return;
 612                 }
 613                 if (tmp) {
 614                         return;
 615                 }
 616                 udelay(1);
 617         }
 618 }
 619
 620 int r600_pcie_gart_init(struct radeon_device *rdev)
 621 {
 622         int r;
 623
 624         if (rdev->gart.table.vram.robj) {
 625                 WARN(1, "R600 PCIE GART already initialized.\n");
 626                 return 0;
 627         }
 628         /* Initialize common gart structure */
 629         r = radeon_gart_init(rdev);
 630         if (r)
 631                 return r;
 632         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 633         return radeon_gart_table_vram_alloc(rdev);
 634 }
 635
 636 int r600_pcie_gart_enable(struct radeon_device *rdev)
 637 {
 638         u32 tmp;
 639         int r, i;
 640
 641         if (rdev->gart.table.vram.robj == NULL) {
 642                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 643                 return -EINVAL;
 644         }
 645         r = radeon_gart_table_vram_pin(rdev);
 646         if (r)
 647                 return r;
 648         radeon_gart_restore(rdev);
 649
 650         /* Setup L2 cache */
 651         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 652                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 653                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 654         WREG32(VM_L2_CNTL2, 0);
 655         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 656         /* Setup TLB control */
 657         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 658                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 659                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 660                 ENABLE_WAIT_L2_QUERY;
 661         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 662         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 663         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 664         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 665         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 666         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 667         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 668         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 669         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 670         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 671         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 672         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 673         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 674         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 675         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 676         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 677         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 678         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 679                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 680         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 681                         (u32)(rdev->dummy_page.addr >> 12));
 682         for (i = 1; i < 7; i++)
 683                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 684
 685         r600_pcie_gart_tlb_flush(rdev);
 686         rdev->gart.ready = true;
 687         return 0;
 688 }
 689
 690 void r600_pcie_gart_disable(struct radeon_device *rdev)
 691 {
 692         u32 tmp;
 693         int i, r;
 694
 695         /* Disable all tables */
 696         for (i = 0; i < 7; i++)
 697                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 698
 699         /* Disable L2 cache */
 700         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 701                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 702         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 703         /* Setup L1 TLB control */
 704         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 705                 ENABLE_WAIT_L2_QUERY;
 706         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 707         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 708         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 709         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 710         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 711         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 712         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 713         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 714         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 715         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 716         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 717         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 718         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 719         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 720         if (rdev->gart.table.vram.robj) {
 721                 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
 722                 if (likely(r == 0)) {
 723                         radeon_bo_kunmap(rdev->gart.table.vram.robj);
 724                         radeon_bo_unpin(rdev->gart.table.vram.robj);
 725                         radeon_bo_unreserve(rdev->gart.table.vram.robj);
 726                 }
 727         }
 728 }
 729
 730 void r600_pcie_gart_fini(struct radeon_device *rdev)
 731 {
 732         radeon_gart_fini(rdev);
 733         r600_pcie_gart_disable(rdev);
 734         radeon_gart_table_vram_free(rdev);
 735 }
 736
 737 void r600_agp_enable(struct radeon_device *rdev)
 738 {
 739         u32 tmp;
 740         int i;
 741
 742         /* Setup L2 cache */
 743         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 744                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 745                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 746         WREG32(VM_L2_CNTL2, 0);
 747         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 748         /* Setup TLB control */
 749         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 750                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 751                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 752                 ENABLE_WAIT_L2_QUERY;
 753         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 754         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 755         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 756         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 757         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 758         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 759         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 760         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 761         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 762         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 763         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 764         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 765         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 766         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 767         for (i = 0; i < 7; i++)
 768                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 769 }
 770
 771 int r600_mc_wait_for_idle(struct radeon_device *rdev)
 772 {
 773         unsigned i;
 774         u32 tmp;
 775
 776         for (i = 0; i < rdev->usec_timeout; i++) {
 777                 /* read MC_STATUS */
 778                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
 779                 if (!tmp)
 780                         return 0;
 781                 udelay(1);
 782         }
 783         return -1;
 784 }
 785
 786 static void r600_mc_program(struct radeon_device *rdev)
 787 {
 788         struct rv515_mc_save save;
 789         u32 tmp;
 790         int i, j;
 791
 792         /* Initialize HDP */
 793         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 794                 WREG32((0x2c14 + j), 0x00000000);
 795                 WREG32((0x2c18 + j), 0x00000000);
 796                 WREG32((0x2c1c + j), 0x00000000);
 797                 WREG32((0x2c20 + j), 0x00000000);
 798                 WREG32((0x2c24 + j), 0x00000000);
 799         }
 800         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
 801
 802         rv515_mc_stop(rdev, &save);
 803         if (r600_mc_wait_for_idle(rdev)) {
 804                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 805         }
 806         /* Lockout access through VGA aperture (doesn't exist before R600) */
 807         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
 808         /* Update configuration */
 809         if (rdev->flags & RADEON_IS_AGP) {
 810                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
 811                         /* VRAM before AGP */
 812                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 813                                 rdev->mc.vram_start >> 12);
 814                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 815                                 rdev->mc.gtt_end >> 12);
 816                 } else {
 817                         /* VRAM after AGP */
 818                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 819                                 rdev->mc.gtt_start >> 12);
 820                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 821                                 rdev->mc.vram_end >> 12);
 822                 }
 823         } else {
 824                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
 825                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
 826         }
 827         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 828         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
 829         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
 830         WREG32(MC_VM_FB_LOCATION, tmp);
 831         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
 832         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
 833         WREG32(HDP_NONSURFACE_SIZE, rdev->mc.mc_vram_size | 0x3FF);
 834         if (rdev->flags & RADEON_IS_AGP) {
 835                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
 836                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
 837                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
 838         } else {
 839                 WREG32(MC_VM_AGP_BASE, 0);
 840                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
 841                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
 842         }
 843         if (r600_mc_wait_for_idle(rdev)) {
 844                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 845         }
 846         rv515_mc_resume(rdev, &save);
 847         /* we need to own VRAM, so turn off the VGA renderer here
 848          * to stop it overwriting our objects */
 849         rv515_vga_render_disable(rdev);
 850 }
 851
 852 /**
 853  * r600_vram_gtt_location - try to find VRAM & GTT location
 854  * @rdev: radeon device structure holding all necessary informations
 855  * @mc: memory controller structure holding memory informations
 856  *
 857  * Function will place try to place VRAM at same place as in CPU (PCI)
 858  * address space as some GPU seems to have issue when we reprogram at
 859  * different address space.
 860  *
 861  * If there is not enough space to fit the unvisible VRAM after the
 862  * aperture then we limit the VRAM size to the aperture.
 863  *
 864  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
 865  * them to be in one from GPU point of view so that we can program GPU to
 866  * catch access outside them (weird GPU policy see ??).
 867  *
 868  * This function will never fails, worst case are limiting VRAM or GTT.
 869  *
 870  * Note: GTT start, end, size should be initialized before calling this
 871  * function on AGP platform.
 872  */
 873 void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 874 {
 875         u64 size_bf, size_af;
 876
 877         if (mc->mc_vram_size > 0xE0000000) {
 878                 /* leave room for at least 512M GTT */
 879                 dev_warn(rdev->dev, "limiting VRAM\n");
 880                 mc->real_vram_size = 0xE0000000;
 881                 mc->mc_vram_size = 0xE0000000;
 882         }
 883         if (rdev->flags & RADEON_IS_AGP) {
 884                 size_bf = mc->gtt_start;
 885                 size_af = 0xFFFFFFFF - mc->gtt_end + 1;
 886                 if (size_bf > size_af) {
 887                         if (mc->mc_vram_size > size_bf) {
 888                                 dev_warn(rdev->dev, "limiting VRAM\n");
 889                                 mc->real_vram_size = size_bf;
 890                                 mc->mc_vram_size = size_bf;
 891                         }
 892                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
 893                 } else {
 894                         if (mc->mc_vram_size > size_af) {
 895                                 dev_warn(rdev->dev, "limiting VRAM\n");
 896                                 mc->real_vram_size = size_af;
 897                                 mc->mc_vram_size = size_af;
 898                         }
 899                         mc->vram_start = mc->gtt_end;
 900                 }
 901                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 902                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
 903                                 mc->mc_vram_size >> 20, mc->vram_start,
 904                                 mc->vram_end, mc->real_vram_size >> 20);
 905         } else {
 906                 u64 base = 0;
 907                 if (rdev->flags & RADEON_IS_IGP)
 908                         base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
 909                 radeon_vram_location(rdev, &rdev->mc, base);
 910                 radeon_gtt_location(rdev, mc);
 911         }
 912 }
 913
 914 int r600_mc_init(struct radeon_device *rdev)
 915 {
 916         u32 tmp;
 917         int chansize, numchan;
 918
 919         /* Get VRAM informations */
 920         rdev->mc.vram_is_ddr = true;
 921         tmp = RREG32(RAMCFG);
 922         if (tmp & CHANSIZE_OVERRIDE) {
 923                 chansize = 16;
 924         } else if (tmp & CHANSIZE_MASK) {
 925                 chansize = 64;
 926         } else {
 927                 chansize = 32;
 928         }
 929         tmp = RREG32(CHMAP);
 930         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
 931         case 0:
 932         default:
 933                 numchan = 1;
 934                 break;
 935         case 1:
 936                 numchan = 2;
 937                 break;
 938         case 2:
 939                 numchan = 4;
 940                 break;
 941         case 3:
 942                 numchan = 8;
 943                 break;
 944         }
 945         rdev->mc.vram_width = numchan * chansize;
 946         /* Could aper size report 0 ? */
 947         rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
 948         rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
 949         /* Setup GPU memory space */
 950         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
 951         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
 952         rdev->mc.visible_vram_size = rdev->mc.aper_size;
 953         r600_vram_gtt_location(rdev, &rdev->mc);
 954
 955         if (rdev->flags & RADEON_IS_IGP)
 956                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
 957         radeon_update_bandwidth_info(rdev);
 958         return 0;
 959 }
 960
 961 /* We doesn't check that the GPU really needs a reset we simply do the
 962  * reset, it's up to the caller to determine if the GPU needs one. We
 963  * might add an helper function to check that.
 964  */
 965 int r600_gpu_soft_reset(struct radeon_device *rdev)
 966 {
 967         struct rv515_mc_save save;
 968         u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) |
 969                                 S_008010_VGT_BUSY(1) | S_008010_TA03_BUSY(1) |
 970                                 S_008010_TC_BUSY(1) | S_008010_SX_BUSY(1) |
 971                                 S_008010_SH_BUSY(1) | S_008010_SPI03_BUSY(1) |
 972                                 S_008010_SMX_BUSY(1) | S_008010_SC_BUSY(1) |
 973                                 S_008010_PA_BUSY(1) | S_008010_DB03_BUSY(1) |
 974                                 S_008010_CR_BUSY(1) | S_008010_CB03_BUSY(1) |
 975                                 S_008010_GUI_ACTIVE(1);
 976         u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) | S_008014_SPI1_BUSY(1) |
 977                         S_008014_SPI2_BUSY(1) | S_008014_SPI3_BUSY(1) |
 978                         S_008014_TA0_BUSY(1) | S_008014_TA1_BUSY(1) |
 979                         S_008014_TA2_BUSY(1) | S_008014_TA3_BUSY(1) |
 980                         S_008014_DB0_BUSY(1) | S_008014_DB1_BUSY(1) |
 981                         S_008014_DB2_BUSY(1) | S_008014_DB3_BUSY(1) |
 982                         S_008014_CB0_BUSY(1) | S_008014_CB1_BUSY(1) |
 983                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
 984         u32 tmp;
 985
 986         dev_info(rdev->dev, "GPU softreset \n");
 987         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
 988                 RREG32(R_008010_GRBM_STATUS));
 989         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
 990                 RREG32(R_008014_GRBM_STATUS2));
 991         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
 992                 RREG32(R_000E50_SRBM_STATUS));
 993         rv515_mc_stop(rdev, &save);
 994         if (r600_mc_wait_for_idle(rdev)) {
 995                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 996         }
 997         /* Disable CP parsing/prefetching */
 998         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
 999         /* Check if any of the rendering block is busy and reset it */
1000         if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) ||
1001             (RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
1002                 tmp = S_008020_SOFT_RESET_CR(1) |
1003                         S_008020_SOFT_RESET_DB(1) |
1004                         S_008020_SOFT_RESET_CB(1) |
1005                         S_008020_SOFT_RESET_PA(1) |
1006                         S_008020_SOFT_RESET_SC(1) |
1007                         S_008020_SOFT_RESET_SMX(1) |
1008                         S_008020_SOFT_RESET_SPI(1) |
1009                         S_008020_SOFT_RESET_SX(1) |
1010                         S_008020_SOFT_RESET_SH(1) |
1011                         S_008020_SOFT_RESET_TC(1) |
1012                         S_008020_SOFT_RESET_TA(1) |
1013                         S_008020_SOFT_RESET_VC(1) |
1014                         S_008020_SOFT_RESET_VGT(1);
1015                 dev_info(rdev->dev, "  R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1016                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1017                 RREG32(R_008020_GRBM_SOFT_RESET);
1018                 mdelay(15);
1019                 WREG32(R_008020_GRBM_SOFT_RESET, 0);
1020         }
1021         /* Reset CP (we always reset CP) */
1022         tmp = S_008020_SOFT_RESET_CP(1);
1023         dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1024         WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1025         RREG32(R_008020_GRBM_SOFT_RESET);
1026         mdelay(15);
1027         WREG32(R_008020_GRBM_SOFT_RESET, 0);
1028         /* Wait a little for things to settle down */
1029         mdelay(1);
1030         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
1031                 RREG32(R_008010_GRBM_STATUS));
1032         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
1033                 RREG32(R_008014_GRBM_STATUS2));
1034         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
1035                 RREG32(R_000E50_SRBM_STATUS));
1036         rv515_mc_resume(rdev, &save);
1037         return 0;
1038 }
1039
1040 bool r600_gpu_is_lockup(struct radeon_device *rdev)
1041 {
1042         u32 srbm_status;
1043         u32 grbm_status;
1044         u32 grbm_status2;
1045         int r;
1046
1047         srbm_status = RREG32(R_000E50_SRBM_STATUS);
1048         grbm_status = RREG32(R_008010_GRBM_STATUS);
1049         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
1050         if (!G_008010_GUI_ACTIVE(grbm_status)) {
1051                 r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
1052                 return false;
1053         }
1054         /* force CP activities */
1055         r = radeon_ring_lock(rdev, 2);
1056         if (!r) {
1057                 /* PACKET2 NOP */
1058                 radeon_ring_write(rdev, 0x80000000);
1059                 radeon_ring_write(rdev, 0x80000000);
1060                 radeon_ring_unlock_commit(rdev);
1061         }
1062         rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
1063         return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
1064 }
1065
1066 int r600_asic_reset(struct radeon_device *rdev)
1067 {
1068         return r600_gpu_soft_reset(rdev);
1069 }
1070
1071 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
1072                                              u32 num_backends,
1073                                              u32 backend_disable_mask)
1074 {
1075         u32 backend_map = 0;
1076         u32 enabled_backends_mask;
1077         u32 enabled_backends_count;
1078         u32 cur_pipe;
1079         u32 swizzle_pipe[R6XX_MAX_PIPES];
1080         u32 cur_backend;
1081         u32 i;
1082
1083         if (num_tile_pipes > R6XX_MAX_PIPES)
1084                 num_tile_pipes = R6XX_MAX_PIPES;
1085         if (num_tile_pipes < 1)
1086                 num_tile_pipes = 1;
1087         if (num_backends > R6XX_MAX_BACKENDS)
1088                 num_backends = R6XX_MAX_BACKENDS;
1089         if (num_backends < 1)
1090                 num_backends = 1;
1091
1092         enabled_backends_mask = 0;
1093         enabled_backends_count = 0;
1094         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
1095                 if (((backend_disable_mask >> i) & 1) == 0) {
1096                         enabled_backends_mask |= (1 << i);
1097                         ++enabled_backends_count;
1098                 }
1099                 if (enabled_backends_count == num_backends)
1100                         break;
1101         }
1102
1103         if (enabled_backends_count == 0) {
1104                 enabled_backends_mask = 1;
1105                 enabled_backends_count = 1;
1106         }
1107
1108         if (enabled_backends_count != num_backends)
1109                 num_backends = enabled_backends_count;
1110
1111         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
1112         switch (num_tile_pipes) {
1113         case 1:
1114                 swizzle_pipe[0] = 0;
1115                 break;
1116         case 2:
1117                 swizzle_pipe[0] = 0;
1118                 swizzle_pipe[1] = 1;
1119                 break;
1120         case 3:
1121                 swizzle_pipe[0] = 0;
1122                 swizzle_pipe[1] = 1;
1123                 swizzle_pipe[2] = 2;
1124                 break;
1125         case 4:
1126                 swizzle_pipe[0] = 0;
1127                 swizzle_pipe[1] = 1;
1128                 swizzle_pipe[2] = 2;
1129                 swizzle_pipe[3] = 3;
1130                 break;
1131         case 5:
1132                 swizzle_pipe[0] = 0;
1133                 swizzle_pipe[1] = 1;
1134                 swizzle_pipe[2] = 2;
1135                 swizzle_pipe[3] = 3;
1136                 swizzle_pipe[4] = 4;
1137                 break;
1138         case 6:
1139                 swizzle_pipe[0] = 0;
1140                 swizzle_pipe[1] = 2;
1141                 swizzle_pipe[2] = 4;
1142                 swizzle_pipe[3] = 5;
1143                 swizzle_pipe[4] = 1;
1144                 swizzle_pipe[5] = 3;
1145                 break;
1146         case 7:
1147                 swizzle_pipe[0] = 0;
1148                 swizzle_pipe[1] = 2;
1149                 swizzle_pipe[2] = 4;
1150                 swizzle_pipe[3] = 6;
1151                 swizzle_pipe[4] = 1;
1152                 swizzle_pipe[5] = 3;
1153                 swizzle_pipe[6] = 5;
1154                 break;
1155         case 8:
1156                 swizzle_pipe[0] = 0;
1157                 swizzle_pipe[1] = 2;
1158                 swizzle_pipe[2] = 4;
1159                 swizzle_pipe[3] = 6;
1160                 swizzle_pipe[4] = 1;
1161                 swizzle_pipe[5] = 3;
1162                 swizzle_pipe[6] = 5;
1163                 swizzle_pipe[7] = 7;
1164                 break;
1165         }
1166
1167         cur_backend = 0;
1168         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1169                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1170                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
1171
1172                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
1173
1174                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
1175         }
1176
1177         return backend_map;
1178 }
1179
1180 int r600_count_pipe_bits(uint32_t val)
1181 {
1182         int i, ret = 0;
1183
1184         for (i = 0; i < 32; i++) {
1185                 ret += val & 1;
1186                 val >>= 1;
1187         }
1188         return ret;
1189 }
1190
1191 void r600_gpu_init(struct radeon_device *rdev)
1192 {
1193         u32 tiling_config;
1194         u32 ramcfg;
1195         u32 backend_map;
1196         u32 cc_rb_backend_disable;
1197         u32 cc_gc_shader_pipe_config;
1198         u32 tmp;
1199         int i, j;
1200         u32 sq_config;
1201         u32 sq_gpr_resource_mgmt_1 = 0;
1202         u32 sq_gpr_resource_mgmt_2 = 0;
1203         u32 sq_thread_resource_mgmt = 0;
1204         u32 sq_stack_resource_mgmt_1 = 0;
1205         u32 sq_stack_resource_mgmt_2 = 0;
1206
1207         /* FIXME: implement */
1208         switch (rdev->family) {
1209         case CHIP_R600:
1210                 rdev->config.r600.max_pipes = 4;
1211                 rdev->config.r600.max_tile_pipes = 8;
1212                 rdev->config.r600.max_simds = 4;
1213                 rdev->config.r600.max_backends = 4;
1214                 rdev->config.r600.max_gprs = 256;
1215                 rdev->config.r600.max_threads = 192;
1216                 rdev->config.r600.max_stack_entries = 256;
1217                 rdev->config.r600.max_hw_contexts = 8;
1218                 rdev->config.r600.max_gs_threads = 16;
1219                 rdev->config.r600.sx_max_export_size = 128;
1220                 rdev->config.r600.sx_max_export_pos_size = 16;
1221                 rdev->config.r600.sx_max_export_smx_size = 128;
1222                 rdev->config.r600.sq_num_cf_insts = 2;
1223                 break;
1224         case CHIP_RV630:
1225         case CHIP_RV635:
1226                 rdev->config.r600.max_pipes = 2;
1227                 rdev->config.r600.max_tile_pipes = 2;
1228                 rdev->config.r600.max_simds = 3;
1229                 rdev->config.r600.max_backends = 1;
1230                 rdev->config.r600.max_gprs = 128;
1231                 rdev->config.r600.max_threads = 192;
1232                 rdev->config.r600.max_stack_entries = 128;
1233                 rdev->config.r600.max_hw_contexts = 8;
1234                 rdev->config.r600.max_gs_threads = 4;
1235                 rdev->config.r600.sx_max_export_size = 128;
1236                 rdev->config.r600.sx_max_export_pos_size = 16;
1237                 rdev->config.r600.sx_max_export_smx_size = 128;
1238                 rdev->config.r600.sq_num_cf_insts = 2;
1239                 break;
1240         case CHIP_RV610:
1241         case CHIP_RV620:
1242         case CHIP_RS780:
1243         case CHIP_RS880:
1244                 rdev->config.r600.max_pipes = 1;
1245                 rdev->config.r600.max_tile_pipes = 1;
1246                 rdev->config.r600.max_simds = 2;
1247                 rdev->config.r600.max_backends = 1;
1248                 rdev->config.r600.max_gprs = 128;
1249                 rdev->config.r600.max_threads = 192;
1250                 rdev->config.r600.max_stack_entries = 128;
1251                 rdev->config.r600.max_hw_contexts = 4;
1252                 rdev->config.r600.max_gs_threads = 4;
1253                 rdev->config.r600.sx_max_export_size = 128;
1254                 rdev->config.r600.sx_max_export_pos_size = 16;
1255                 rdev->config.r600.sx_max_export_smx_size = 128;
1256                 rdev->config.r600.sq_num_cf_insts = 1;
1257                 break;
1258         case CHIP_RV670:
1259                 rdev->config.r600.max_pipes = 4;
1260                 rdev->config.r600.max_tile_pipes = 4;
1261                 rdev->config.r600.max_simds = 4;
1262                 rdev->config.r600.max_backends = 4;
1263                 rdev->config.r600.max_gprs = 192;
1264                 rdev->config.r600.max_threads = 192;
1265                 rdev->config.r600.max_stack_entries = 256;
1266                 rdev->config.r600.max_hw_contexts = 8;
1267                 rdev->config.r600.max_gs_threads = 16;
1268                 rdev->config.r600.sx_max_export_size = 128;
1269                 rdev->config.r600.sx_max_export_pos_size = 16;
1270                 rdev->config.r600.sx_max_export_smx_size = 128;
1271                 rdev->config.r600.sq_num_cf_insts = 2;
1272                 break;
1273         default:
1274                 break;
1275         }
1276
1277         /* Initialize HDP */
1278         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1279                 WREG32((0x2c14 + j), 0x00000000);
1280                 WREG32((0x2c18 + j), 0x00000000);
1281                 WREG32((0x2c1c + j), 0x00000000);
1282                 WREG32((0x2c20 + j), 0x00000000);
1283                 WREG32((0x2c24 + j), 0x00000000);
1284         }
1285
1286         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1287
1288         /* Setup tiling */
1289         tiling_config = 0;
1290         ramcfg = RREG32(RAMCFG);
1291         switch (rdev->config.r600.max_tile_pipes) {
1292         case 1:
1293                 tiling_config |= PIPE_TILING(0);
1294                 break;
1295         case 2:
1296                 tiling_config |= PIPE_TILING(1);
1297                 break;
1298         case 4:
1299                 tiling_config |= PIPE_TILING(2);
1300                 break;
1301         case 8:
1302                 tiling_config |= PIPE_TILING(3);
1303                 break;
1304         default:
1305                 break;
1306         }
1307         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1308         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1309         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1310         tiling_config |= GROUP_SIZE(0);
1311         rdev->config.r600.tiling_group_size = 256;
1312         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1313         if (tmp > 3) {
1314                 tiling_config |= ROW_TILING(3);
1315                 tiling_config |= SAMPLE_SPLIT(3);
1316         } else {
1317                 tiling_config |= ROW_TILING(tmp);
1318                 tiling_config |= SAMPLE_SPLIT(tmp);
1319         }
1320         tiling_config |= BANK_SWAPS(1);
1321
1322         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1323         cc_rb_backend_disable |=
1324                 BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
1325
1326         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1327         cc_gc_shader_pipe_config |=
1328                 INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
1329         cc_gc_shader_pipe_config |=
1330                 INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
1331
1332         backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
1333                                                         (R6XX_MAX_BACKENDS -
1334                                                          r600_count_pipe_bits((cc_rb_backend_disable &
1335                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
1336                                                         (cc_rb_backend_disable >> 16));
1337
1338         tiling_config |= BACKEND_MAP(backend_map);
1339         WREG32(GB_TILING_CONFIG, tiling_config);
1340         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1341         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1342
1343         /* Setup pipes */
1344         WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1345         WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1346         WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1347
1348         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1349         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1350         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1351
1352         /* Setup some CP states */
1353         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1354         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1355
1356         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1357                              SYNC_WALKER | SYNC_ALIGNER));
1358         /* Setup various GPU states */
1359         if (rdev->family == CHIP_RV670)
1360                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1361
1362         tmp = RREG32(SX_DEBUG_1);
1363         tmp |= SMX_EVENT_RELEASE;
1364         if ((rdev->family > CHIP_R600))
1365                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1366         WREG32(SX_DEBUG_1, tmp);
1367
1368         if (((rdev->family) == CHIP_R600) ||
1369             ((rdev->family) == CHIP_RV630) ||
1370             ((rdev->family) == CHIP_RV610) ||
1371             ((rdev->family) == CHIP_RV620) ||
1372             ((rdev->family) == CHIP_RS780) ||
1373             ((rdev->family) == CHIP_RS880)) {
1374                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1375         } else {
1376                 WREG32(DB_DEBUG, 0);
1377         }
1378         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1379                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1380
1381         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1382         WREG32(VGT_NUM_INSTANCES, 0);
1383
1384         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1385         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1386
1387         tmp = RREG32(SQ_MS_FIFO_SIZES);
1388         if (((rdev->family) == CHIP_RV610) ||
1389             ((rdev->family) == CHIP_RV620) ||
1390             ((rdev->family) == CHIP_RS780) ||
1391             ((rdev->family) == CHIP_RS880)) {
1392                 tmp = (CACHE_FIFO_SIZE(0xa) |
1393                        FETCH_FIFO_HIWATER(0xa) |
1394                        DONE_FIFO_HIWATER(0xe0) |
1395                        ALU_UPDATE_FIFO_HIWATER(0x8));
1396         } else if (((rdev->family) == CHIP_R600) ||
1397                    ((rdev->family) == CHIP_RV630)) {
1398                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1399                 tmp |= DONE_FIFO_HIWATER(0x4);
1400         }
1401         WREG32(SQ_MS_FIFO_SIZES, tmp);
1402
1403         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1404          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1405          */
1406         sq_config = RREG32(SQ_CONFIG);
1407         sq_config &= ~(PS_PRIO(3) |
1408                        VS_PRIO(3) |
1409                        GS_PRIO(3) |
1410                        ES_PRIO(3));
1411         sq_config |= (DX9_CONSTS |
1412                       VC_ENABLE |
1413                       PS_PRIO(0) |
1414                       VS_PRIO(1) |
1415                       GS_PRIO(2) |
1416                       ES_PRIO(3));
1417
1418         if ((rdev->family) == CHIP_R600) {
1419                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1420                                           NUM_VS_GPRS(124) |
1421                                           NUM_CLAUSE_TEMP_GPRS(4));
1422                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1423                                           NUM_ES_GPRS(0));
1424                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1425                                            NUM_VS_THREADS(48) |
1426                                            NUM_GS_THREADS(4) |
1427                                            NUM_ES_THREADS(4));
1428                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1429                                             NUM_VS_STACK_ENTRIES(128));
1430                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1431                                             NUM_ES_STACK_ENTRIES(0));
1432         } else if (((rdev->family) == CHIP_RV610) ||
1433                    ((rdev->family) == CHIP_RV620) ||
1434                    ((rdev->family) == CHIP_RS780) ||
1435                    ((rdev->family) == CHIP_RS880)) {
1436                 /* no vertex cache */
1437                 sq_config &= ~VC_ENABLE;
1438
1439                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1440                                           NUM_VS_GPRS(44) |
1441                                           NUM_CLAUSE_TEMP_GPRS(2));
1442                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1443                                           NUM_ES_GPRS(17));
1444                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1445                                            NUM_VS_THREADS(78) |
1446                                            NUM_GS_THREADS(4) |
1447                                            NUM_ES_THREADS(31));
1448                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1449                                             NUM_VS_STACK_ENTRIES(40));
1450                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1451                                             NUM_ES_STACK_ENTRIES(16));
1452         } else if (((rdev->family) == CHIP_RV630) ||
1453                    ((rdev->family) == CHIP_RV635)) {
1454                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1455                                           NUM_VS_GPRS(44) |
1456                                           NUM_CLAUSE_TEMP_GPRS(2));
1457                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1458                                           NUM_ES_GPRS(18));
1459                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1460                                            NUM_VS_THREADS(78) |
1461                                            NUM_GS_THREADS(4) |
1462                                            NUM_ES_THREADS(31));
1463                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1464                                             NUM_VS_STACK_ENTRIES(40));
1465                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1466                                             NUM_ES_STACK_ENTRIES(16));
1467         } else if ((rdev->family) == CHIP_RV670) {
1468                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1469                                           NUM_VS_GPRS(44) |
1470                                           NUM_CLAUSE_TEMP_GPRS(2));
1471                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1472                                           NUM_ES_GPRS(17));
1473                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1474                                            NUM_VS_THREADS(78) |
1475                                            NUM_GS_THREADS(4) |
1476                                            NUM_ES_THREADS(31));
1477                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1478                                             NUM_VS_STACK_ENTRIES(64));
1479                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1480                                             NUM_ES_STACK_ENTRIES(64));
1481         }
1482
1483         WREG32(SQ_CONFIG, sq_config);
1484         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1485         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1486         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1487         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1488         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1489
1490         if (((rdev->family) == CHIP_RV610) ||
1491             ((rdev->family) == CHIP_RV620) ||
1492             ((rdev->family) == CHIP_RS780) ||
1493             ((rdev->family) == CHIP_RS880)) {
1494                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1495         } else {
1496                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1497         }
1498
1499         /* More default values. 2D/3D driver should adjust as needed */
1500         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1501                                          S1_X(0x4) | S1_Y(0xc)));
1502         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1503                                          S1_X(0x2) | S1_Y(0x2) |
1504                                          S2_X(0xa) | S2_Y(0x6) |
1505                                          S3_X(0x6) | S3_Y(0xa)));
1506         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1507                                              S1_X(0x4) | S1_Y(0xc) |
1508                                              S2_X(0x1) | S2_Y(0x6) |
1509                                              S3_X(0xa) | S3_Y(0xe)));
1510         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1511                                              S5_X(0x0) | S5_Y(0x0) |
1512                                              S6_X(0xb) | S6_Y(0x4) |
1513                                              S7_X(0x7) | S7_Y(0x8)));
1514
1515         WREG32(VGT_STRMOUT_EN, 0);
1516         tmp = rdev->config.r600.max_pipes * 16;
1517         switch (rdev->family) {
1518         case CHIP_RV610:
1519         case CHIP_RV620:
1520         case CHIP_RS780:
1521         case CHIP_RS880:
1522                 tmp += 32;
1523                 break;
1524         case CHIP_RV670:
1525                 tmp += 128;
1526                 break;
1527         default:
1528                 break;
1529         }
1530         if (tmp > 256) {
1531                 tmp = 256;
1532         }
1533         WREG32(VGT_ES_PER_GS, 128);
1534         WREG32(VGT_GS_PER_ES, tmp);
1535         WREG32(VGT_GS_PER_VS, 2);
1536         WREG32(VGT_GS_VERTEX_REUSE, 16);
1537
1538         /* more default values. 2D/3D driver should adjust as needed */
1539         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1540         WREG32(VGT_STRMOUT_EN, 0);
1541         WREG32(SX_MISC, 0);
1542         WREG32(PA_SC_MODE_CNTL, 0);
1543         WREG32(PA_SC_AA_CONFIG, 0);
1544         WREG32(PA_SC_LINE_STIPPLE, 0);
1545         WREG32(SPI_INPUT_Z, 0);
1546         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1547         WREG32(CB_COLOR7_FRAG, 0);
1548
1549         /* Clear render buffer base addresses */
1550         WREG32(CB_COLOR0_BASE, 0);
1551         WREG32(CB_COLOR1_BASE, 0);
1552         WREG32(CB_COLOR2_BASE, 0);
1553         WREG32(CB_COLOR3_BASE, 0);
1554         WREG32(CB_COLOR4_BASE, 0);
1555         WREG32(CB_COLOR5_BASE, 0);
1556         WREG32(CB_COLOR6_BASE, 0);
1557         WREG32(CB_COLOR7_BASE, 0);
1558         WREG32(CB_COLOR7_FRAG, 0);
1559
1560         switch (rdev->family) {
1561         case CHIP_RV610:
1562         case CHIP_RV620:
1563         case CHIP_RS780:
1564         case CHIP_RS880:
1565                 tmp = TC_L2_SIZE(8);
1566                 break;
1567         case CHIP_RV630:
1568         case CHIP_RV635:
1569                 tmp = TC_L2_SIZE(4);
1570                 break;
1571         case CHIP_R600:
1572                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
1573                 break;
1574         default:
1575                 tmp = TC_L2_SIZE(0);
1576                 break;
1577         }
1578         WREG32(TC_CNTL, tmp);
1579
1580         tmp = RREG32(HDP_HOST_PATH_CNTL);
1581         WREG32(HDP_HOST_PATH_CNTL, tmp);
1582
1583         tmp = RREG32(ARB_POP);
1584         tmp |= ENABLE_TC128;
1585         WREG32(ARB_POP, tmp);
1586
1587         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1588         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1589                                NUM_CLIP_SEQ(3)));
1590         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
1591 }
1592
1593
1594 /*
1595  * Indirect registers accessor
1596  */
1597 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1598 {
1599         u32 r;
1600
1601         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1602         (void)RREG32(PCIE_PORT_INDEX);
1603         r = RREG32(PCIE_PORT_DATA);
1604         return r;
1605 }
1606
1607 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1608 {
1609         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1610         (void)RREG32(PCIE_PORT_INDEX);
1611         WREG32(PCIE_PORT_DATA, (v));
1612         (void)RREG32(PCIE_PORT_DATA);
1613 }
1614
1615 /*
1616  * CP & Ring
1617  */
1618 void r600_cp_stop(struct radeon_device *rdev)
1619 {
1620         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1621 }
1622
1623 int r600_init_microcode(struct radeon_device *rdev)
1624 {
1625         struct platform_device *pdev;
1626         const char *chip_name;
1627         const char *rlc_chip_name;
1628         size_t pfp_req_size, me_req_size, rlc_req_size;
1629         char fw_name[30];
1630         int err;
1631
1632         DRM_DEBUG("\n");
1633
1634         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1635         err = IS_ERR(pdev);
1636         if (err) {
1637                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1638                 return -EINVAL;
1639         }
1640
1641         switch (rdev->family) {
1642         case CHIP_R600:
1643                 chip_name = "R600";
1644                 rlc_chip_name = "R600";
1645                 break;
1646         case CHIP_RV610:
1647                 chip_name = "RV610";
1648                 rlc_chip_name = "R600";
1649                 break;
1650         case CHIP_RV630:
1651                 chip_name = "RV630";
1652                 rlc_chip_name = "R600";
1653                 break;
1654         case CHIP_RV620:
1655                 chip_name = "RV620";
1656                 rlc_chip_name = "R600";
1657                 break;
1658         case CHIP_RV635:
1659                 chip_name = "RV635";
1660                 rlc_chip_name = "R600";
1661                 break;
1662         case CHIP_RV670:
1663                 chip_name = "RV670";
1664                 rlc_chip_name = "R600";
1665                 break;
1666         case CHIP_RS780:
1667         case CHIP_RS880:
1668                 chip_name = "RS780";
1669                 rlc_chip_name = "R600";
1670                 break;
1671         case CHIP_RV770:
1672                 chip_name = "RV770";
1673                 rlc_chip_name = "R700";
1674                 break;
1675         case CHIP_RV730:
1676         case CHIP_RV740:
1677                 chip_name = "RV730";
1678                 rlc_chip_name = "R700";
1679                 break;
1680         case CHIP_RV710:
1681                 chip_name = "RV710";
1682                 rlc_chip_name = "R700";
1683                 break;
1684         case CHIP_CEDAR:
1685                 chip_name = "CEDAR";
1686                 rlc_chip_name = "CEDAR";
1687                 break;
1688         case CHIP_REDWOOD:
1689                 chip_name = "REDWOOD";
1690                 rlc_chip_name = "REDWOOD";
1691                 break;
1692         case CHIP_JUNIPER:
1693                 chip_name = "JUNIPER";
1694                 rlc_chip_name = "JUNIPER";
1695                 break;
1696         case CHIP_CYPRESS:
1697         case CHIP_HEMLOCK:
1698                 chip_name = "CYPRESS";
1699                 rlc_chip_name = "CYPRESS";
1700                 break;
1701         default: BUG();
1702         }
1703
1704         if (rdev->family >= CHIP_CEDAR) {
1705                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
1706                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
1707                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
1708         } else if (rdev->family >= CHIP_RV770) {
1709                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
1710                 me_req_size = R700_PM4_UCODE_SIZE * 4;
1711                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
1712         } else {
1713                 pfp_req_size = PFP_UCODE_SIZE * 4;
1714                 me_req_size = PM4_UCODE_SIZE * 12;
1715                 rlc_req_size = RLC_UCODE_SIZE * 4;
1716         }
1717
1718         DRM_INFO("Loading %s Microcode\n", chip_name);
1719
1720         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1721         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1722         if (err)
1723                 goto out;
1724         if (rdev->pfp_fw->size != pfp_req_size) {
1725                 printk(KERN_ERR
1726                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1727                        rdev->pfp_fw->size, fw_name);
1728                 err = -EINVAL;
1729                 goto out;
1730         }
1731
1732         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1733         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1734         if (err)
1735                 goto out;
1736         if (rdev->me_fw->size != me_req_size) {
1737                 printk(KERN_ERR
1738                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1739                        rdev->me_fw->size, fw_name);
1740                 err = -EINVAL;
1741         }
1742
1743         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1744         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1745         if (err)
1746                 goto out;
1747         if (rdev->rlc_fw->size != rlc_req_size) {
1748                 printk(KERN_ERR
1749                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
1750                        rdev->rlc_fw->size, fw_name);
1751                 err = -EINVAL;
1752         }
1753
1754 out:
1755         platform_device_unregister(pdev);
1756
1757         if (err) {
1758                 if (err != -EINVAL)
1759                         printk(KERN_ERR
1760                                "r600_cp: Failed to load firmware \"%s\"\n",
1761                                fw_name);
1762                 release_firmware(rdev->pfp_fw);
1763                 rdev->pfp_fw = NULL;
1764                 release_firmware(rdev->me_fw);
1765                 rdev->me_fw = NULL;
1766                 release_firmware(rdev->rlc_fw);
1767                 rdev->rlc_fw = NULL;
1768         }
1769         return err;
1770 }
1771
1772 static int r600_cp_load_microcode(struct radeon_device *rdev)
1773 {
1774         const __be32 *fw_data;
1775         int i;
1776
1777         if (!rdev->me_fw || !rdev->pfp_fw)
1778                 return -EINVAL;
1779
1780         r600_cp_stop(rdev);
1781
1782         WREG32(CP_RB_CNTL, RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1783
1784         /* Reset cp */
1785         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1786         RREG32(GRBM_SOFT_RESET);
1787         mdelay(15);
1788         WREG32(GRBM_SOFT_RESET, 0);
1789
1790         WREG32(CP_ME_RAM_WADDR, 0);
1791
1792         fw_data = (const __be32 *)rdev->me_fw->data;
1793         WREG32(CP_ME_RAM_WADDR, 0);
1794         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
1795                 WREG32(CP_ME_RAM_DATA,
1796                        be32_to_cpup(fw_data++));
1797
1798         fw_data = (const __be32 *)rdev->pfp_fw->data;
1799         WREG32(CP_PFP_UCODE_ADDR, 0);
1800         for (i = 0; i < PFP_UCODE_SIZE; i++)
1801                 WREG32(CP_PFP_UCODE_DATA,
1802                        be32_to_cpup(fw_data++));
1803
1804         WREG32(CP_PFP_UCODE_ADDR, 0);
1805         WREG32(CP_ME_RAM_WADDR, 0);
1806         WREG32(CP_ME_RAM_RADDR, 0);
1807         return 0;
1808 }
1809
1810 int r600_cp_start(struct radeon_device *rdev)
1811 {
1812         int r;
1813         uint32_t cp_me;
1814
1815         r = radeon_ring_lock(rdev, 7);
1816         if (r) {
1817                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1818                 return r;
1819         }
1820         radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1821         radeon_ring_write(rdev, 0x1);
1822         if (rdev->family >= CHIP_CEDAR) {
1823                 radeon_ring_write(rdev, 0x0);
1824                 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1825         } else if (rdev->family >= CHIP_RV770) {
1826                 radeon_ring_write(rdev, 0x0);
1827                 radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
1828         } else {
1829                 radeon_ring_write(rdev, 0x3);
1830                 radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
1831         }
1832         radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1833         radeon_ring_write(rdev, 0);
1834         radeon_ring_write(rdev, 0);
1835         radeon_ring_unlock_commit(rdev);
1836
1837         cp_me = 0xff;
1838         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
1839         return 0;
1840 }
1841
1842 int r600_cp_resume(struct radeon_device *rdev)
1843 {
1844         u32 tmp;
1845         u32 rb_bufsz;
1846         int r;
1847
1848         /* Reset cp */
1849         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1850         RREG32(GRBM_SOFT_RESET);
1851         mdelay(15);
1852         WREG32(GRBM_SOFT_RESET, 0);
1853
1854         /* Set ring buffer size */
1855         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1856         tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1857 #ifdef __BIG_ENDIAN
1858         tmp |= BUF_SWAP_32BIT;
1859 #endif
1860         WREG32(CP_RB_CNTL, tmp);
1861         WREG32(CP_SEM_WAIT_TIMER, 0x4);
1862
1863         /* Set the write pointer delay */
1864         WREG32(CP_RB_WPTR_DELAY, 0);
1865
1866         /* Initialize the ring buffer's read and write pointers */
1867         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1868         WREG32(CP_RB_RPTR_WR, 0);
1869         WREG32(CP_RB_WPTR, 0);
1870         WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
1871         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
1872         mdelay(1);
1873         WREG32(CP_RB_CNTL, tmp);
1874
1875         WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1876         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1877
1878         rdev->cp.rptr = RREG32(CP_RB_RPTR);
1879         rdev->cp.wptr = RREG32(CP_RB_WPTR);
1880
1881         r600_cp_start(rdev);
1882         rdev->cp.ready = true;
1883         r = radeon_ring_test(rdev);
1884         if (r) {
1885                 rdev->cp.ready = false;
1886                 return r;
1887         }
1888         return 0;
1889 }
1890
1891 void r600_cp_commit(struct radeon_device *rdev)
1892 {
1893         WREG32(CP_RB_WPTR, rdev->cp.wptr);
1894         (void)RREG32(CP_RB_WPTR);
1895 }
1896
1897 void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
1898 {
1899         u32 rb_bufsz;
1900
1901         /* Align ring size */
1902         rb_bufsz = drm_order(ring_size / 8);
1903         ring_size = (1 << (rb_bufsz + 1)) * 4;
1904         rdev->cp.ring_size = ring_size;
1905         rdev->cp.align_mask = 16 - 1;
1906 }
1907
1908 void r600_cp_fini(struct radeon_device *rdev)
1909 {
1910         r600_cp_stop(rdev);
1911         radeon_ring_fini(rdev);
1912 }
1913
1914
1915 /*
1916  * GPU scratch registers helpers function.
1917  */
1918 void r600_scratch_init(struct radeon_device *rdev)
1919 {
1920         int i;
1921
1922         rdev->scratch.num_reg = 7;
1923         for (i = 0; i < rdev->scratch.num_reg; i++) {
1924                 rdev->scratch.free[i] = true;
1925                 rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4);
1926         }
1927 }
1928
1929 int r600_ring_test(struct radeon_device *rdev)
1930 {
1931         uint32_t scratch;
1932         uint32_t tmp = 0;
1933         unsigned i;
1934         int r;
1935
1936         r = radeon_scratch_get(rdev, &scratch);
1937         if (r) {
1938                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
1939                 return r;
1940         }
1941         WREG32(scratch, 0xCAFEDEAD);
1942         r = radeon_ring_lock(rdev, 3);
1943         if (r) {
1944                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1945                 radeon_scratch_free(rdev, scratch);
1946                 return r;
1947         }
1948         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1949         radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1950         radeon_ring_write(rdev, 0xDEADBEEF);
1951         radeon_ring_unlock_commit(rdev);
1952         for (i = 0; i < rdev->usec_timeout; i++) {
1953                 tmp = RREG32(scratch);
1954                 if (tmp == 0xDEADBEEF)
1955                         break;
1956                 DRM_UDELAY(1);
1957         }
1958         if (i < rdev->usec_timeout) {
1959                 DRM_INFO("ring test succeeded in %d usecs\n", i);
1960         } else {
1961                 DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
1962                           scratch, tmp);
1963                 r = -EINVAL;
1964         }
1965         radeon_scratch_free(rdev, scratch);
1966         return r;
1967 }
1968
1969 void r600_wb_disable(struct radeon_device *rdev)
1970 {
1971         int r;
1972
1973         WREG32(SCRATCH_UMSK, 0);
1974         if (rdev->wb.wb_obj) {
1975                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
1976                 if (unlikely(r != 0))
1977                         return;
1978                 radeon_bo_kunmap(rdev->wb.wb_obj);
1979                 radeon_bo_unpin(rdev->wb.wb_obj);
1980                 radeon_bo_unreserve(rdev->wb.wb_obj);
1981         }
1982 }
1983
1984 void r600_wb_fini(struct radeon_device *rdev)
1985 {
1986         r600_wb_disable(rdev);
1987         if (rdev->wb.wb_obj) {
1988                 radeon_bo_unref(&rdev->wb.wb_obj);
1989                 rdev->wb.wb = NULL;
1990                 rdev->wb.wb_obj = NULL;
1991         }
1992 }
1993
1994 int r600_wb_enable(struct radeon_device *rdev)
1995 {
1996         int r;
1997
1998         if (rdev->wb.wb_obj == NULL) {
1999                 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
2000                                 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
2001                 if (r) {
2002                         dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
2003                         return r;
2004                 }
2005                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
2006                 if (unlikely(r != 0)) {
2007                         r600_wb_fini(rdev);
2008                         return r;
2009                 }
2010                 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
2011                                 &rdev->wb.gpu_addr);
2012                 if (r) {
2013                         radeon_bo_unreserve(rdev->wb.wb_obj);
2014                         dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
2015                         r600_wb_fini(rdev);
2016                         return r;
2017                 }
2018                 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
2019                 radeon_bo_unreserve(rdev->wb.wb_obj);
2020                 if (r) {
2021                         dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
2022                         r600_wb_fini(rdev);
2023                         return r;
2024                 }
2025         }
2026         WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
2027         WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
2028         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
2029         WREG32(SCRATCH_UMSK, 0xff);
2030         return 0;
2031 }
2032
2033 void r600_fence_ring_emit(struct radeon_device *rdev,
2034                           struct radeon_fence *fence)
2035 {
2036         /* Also consider EVENT_WRITE_EOP.  it handles the interrupts + timestamps + events */
2037
2038         radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
2039         radeon_ring_write(rdev, CACHE_FLUSH_AND_INV_EVENT);
2040         /* wait for 3D idle clean */
2041         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2042         radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2043         radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2044         /* Emit fence sequence & fire IRQ */
2045         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2046         radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2047         radeon_ring_write(rdev, fence->seq);
2048         /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2049         radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
2050         radeon_ring_write(rdev, RB_INT_STAT);
2051 }
2052
2053 int r600_copy_blit(struct radeon_device *rdev,
2054                    uint64_t src_offset, uint64_t dst_offset,
2055                    unsigned num_pages, struct radeon_fence *fence)
2056 {
2057         int r;
2058
2059         mutex_lock(&rdev->r600_blit.mutex);
2060         rdev->r600_blit.vb_ib = NULL;
2061         r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
2062         if (r) {
2063                 if (rdev->r600_blit.vb_ib)
2064                         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
2065                 mutex_unlock(&rdev->r600_blit.mutex);
2066                 return r;
2067         }
2068         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
2069         r600_blit_done_copy(rdev, fence);
2070         mutex_unlock(&rdev->r600_blit.mutex);
2071         return 0;
2072 }
2073
2074 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
2075                          uint32_t tiling_flags, uint32_t pitch,
2076                          uint32_t offset, uint32_t obj_size)
2077 {
2078         /* FIXME: implement */
2079         return 0;
2080 }
2081
2082 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
2083 {
2084         /* FIXME: implement */
2085 }
2086
2087
2088 bool r600_card_posted(struct radeon_device *rdev)
2089 {
2090         uint32_t reg;
2091
2092         /* first check CRTCs */
2093         reg = RREG32(D1CRTC_CONTROL) |
2094                 RREG32(D2CRTC_CONTROL);
2095         if (reg & CRTC_EN)
2096                 return true;
2097
2098         /* then check MEM_SIZE, in case the crtcs are off */
2099         if (RREG32(CONFIG_MEMSIZE))
2100                 return true;
2101
2102         return false;
2103 }
2104
2105 int r600_startup(struct radeon_device *rdev)
2106 {
2107         int r;
2108
2109         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2110                 r = r600_init_microcode(rdev);
2111                 if (r) {
2112                         DRM_ERROR("Failed to load firmware!\n");
2113                         return r;
2114                 }
2115         }
2116
2117         r600_mc_program(rdev);
2118         if (rdev->flags & RADEON_IS_AGP) {
2119                 r600_agp_enable(rdev);
2120         } else {
2121                 r = r600_pcie_gart_enable(rdev);
2122                 if (r)
2123                         return r;
2124         }
2125         r600_gpu_init(rdev);
2126         r = r600_blit_init(rdev);
2127         if (r) {
2128                 r600_blit_fini(rdev);
2129                 rdev->asic->copy = NULL;
2130                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2131         }
2132         /* pin copy shader into vram */
2133         if (rdev->r600_blit.shader_obj) {
2134                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
2135                 if (unlikely(r != 0))
2136                         return r;
2137                 r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
2138                                 &rdev->r600_blit.shader_gpu_addr);
2139                 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
2140                 if (r) {
2141                         dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
2142                         return r;
2143                 }
2144         }
2145         /* Enable IRQ */
2146         r = r600_irq_init(rdev);
2147         if (r) {
2148                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2149                 radeon_irq_kms_fini(rdev);
2150                 return r;
2151         }
2152         r600_irq_set(rdev);
2153
2154         r = radeon_ring_init(rdev, rdev->cp.ring_size);
2155         if (r)
2156                 return r;
2157         r = r600_cp_load_microcode(rdev);
2158         if (r)
2159                 return r;
2160         r = r600_cp_resume(rdev);
2161         if (r)
2162                 return r;
2163         /* write back buffer are not vital so don't worry about failure */
2164         r600_wb_enable(rdev);
2165         return 0;
2166 }
2167
2168 void r600_vga_set_state(struct radeon_device *rdev, bool state)
2169 {
2170         uint32_t temp;
2171
2172         temp = RREG32(CONFIG_CNTL);
2173         if (state == false) {
2174                 temp &= ~(1<<0);
2175                 temp |= (1<<1);
2176         } else {
2177                 temp &= ~(1<<1);
2178         }
2179         WREG32(CONFIG_CNTL, temp);
2180 }
2181
2182 int r600_resume(struct radeon_device *rdev)
2183 {
2184         int r;
2185
2186         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
2187          * posting will perform necessary task to bring back GPU into good
2188          * shape.
2189          */
2190         /* post card */
2191         atom_asic_init(rdev->mode_info.atom_context);
2192         /* Initialize clocks */
2193         r = radeon_clocks_init(rdev);
2194         if (r) {
2195                 return r;
2196         }
2197
2198         r = r600_startup(rdev);
2199         if (r) {
2200                 DRM_ERROR("r600 startup failed on resume\n");
2201                 return r;
2202         }
2203
2204         r = r600_ib_test(rdev);
2205         if (r) {
2206                 DRM_ERROR("radeon: failled testing IB (%d).\n", r);
2207                 return r;
2208         }
2209
2210         r = r600_audio_init(rdev);
2211         if (r) {
2212                 DRM_ERROR("radeon: audio resume failed\n");
2213                 return r;
2214         }
2215
2216         return r;
2217 }
2218
2219 int r600_suspend(struct radeon_device *rdev)
2220 {
2221         int r;
2222
2223         r600_audio_fini(rdev);
2224         /* FIXME: we should wait for ring to be empty */
2225         r600_cp_stop(rdev);
2226         rdev->cp.ready = false;
2227         r600_irq_suspend(rdev);
2228         r600_wb_disable(rdev);
2229         r600_pcie_gart_disable(rdev);
2230         /* unpin shaders bo */
2231         if (rdev->r600_blit.shader_obj) {
2232                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
2233                 if (!r) {
2234                         radeon_bo_unpin(rdev->r600_blit.shader_obj);
2235                         radeon_bo_unreserve(rdev->r600_blit.shader_obj);
2236                 }
2237         }
2238         return 0;
2239 }
2240
2241 /* Plan is to move initialization in that function and use
2242  * helper function so that radeon_device_init pretty much
2243  * do nothing more than calling asic specific function. This
2244  * should also allow to remove a bunch of callback function
2245  * like vram_info.
2246  */
2247 int r600_init(struct radeon_device *rdev)
2248 {
2249         int r;
2250
2251         r = radeon_dummy_page_init(rdev);
2252         if (r)
2253                 return r;
2254         if (r600_debugfs_mc_info_init(rdev)) {
2255                 DRM_ERROR("Failed to register debugfs file for mc !\n");
2256         }
2257         /* This don't do much */
2258         r = radeon_gem_init(rdev);
2259         if (r)
2260                 return r;
2261         /* Read BIOS */
2262         if (!radeon_get_bios(rdev)) {
2263                 if (ASIC_IS_AVIVO(rdev))
2264                         return -EINVAL;
2265         }
2266         /* Must be an ATOMBIOS */
2267         if (!rdev->is_atom_bios) {
2268                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
2269                 return -EINVAL;
2270         }
2271         r = radeon_atombios_init(rdev);
2272         if (r)
2273                 return r;
2274         /* Post card if necessary */
2275         if (!r600_card_posted(rdev)) {
2276                 if (!rdev->bios) {
2277                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2278                         return -EINVAL;
2279                 }
2280                 DRM_INFO("GPU not posted. posting now...\n");
2281                 atom_asic_init(rdev->mode_info.atom_context);
2282         }
2283         /* Initialize scratch registers */
2284         r600_scratch_init(rdev);
2285         /* Initialize surface registers */
2286         radeon_surface_init(rdev);
2287         /* Initialize clocks */
2288         radeon_get_clock_info(rdev->ddev);
2289         r = radeon_clocks_init(rdev);
2290         if (r)
2291                 return r;
2292         /* Initialize power management */
2293         radeon_pm_init(rdev);
2294         /* Fence driver */
2295         r = radeon_fence_driver_init(rdev);
2296         if (r)
2297                 return r;
2298         if (rdev->flags & RADEON_IS_AGP) {
2299                 r = radeon_agp_init(rdev);
2300                 if (r)
2301                         radeon_agp_disable(rdev);
2302         }
2303         r = r600_mc_init(rdev);
2304         if (r)
2305                 return r;
2306         /* Memory manager */
2307         r = radeon_bo_init(rdev);
2308         if (r)
2309                 return r;
2310
2311         r = radeon_irq_kms_init(rdev);
2312         if (r)
2313                 return r;
2314
2315         rdev->cp.ring_obj = NULL;
2316         r600_ring_init(rdev, 1024 * 1024);
2317
2318         rdev->ih.ring_obj = NULL;
2319         r600_ih_ring_init(rdev, 64 * 1024);
2320
2321         r = r600_pcie_gart_init(rdev);
2322         if (r)
2323                 return r;
2324
2325         rdev->accel_working = true;
2326         r = r600_startup(rdev);
2327         if (r) {
2328                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2329                 r600_cp_fini(rdev);
2330                 r600_wb_fini(rdev);
2331                 r600_irq_fini(rdev);
2332                 radeon_irq_kms_fini(rdev);
2333                 r600_pcie_gart_fini(rdev);
2334                 rdev->accel_working = false;
2335         }
2336         if (rdev->accel_working) {
2337                 r = radeon_ib_pool_init(rdev);
2338                 if (r) {
2339                         dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2340                         rdev->accel_working = false;
2341                 } else {
2342                         r = r600_ib_test(rdev);
2343                         if (r) {
2344                                 dev_err(rdev->dev, "IB test failed (%d).\n", r);
2345                                 rdev->accel_working = false;
2346                         }
2347                 }
2348         }
2349
2350         r = r600_audio_init(rdev);
2351         if (r)
2352                 return r; /* TODO error handling */
2353         return 0;
2354 }
2355
2356 void r600_fini(struct radeon_device *rdev)
2357 {
2358         radeon_pm_fini(rdev);
2359         r600_audio_fini(rdev);
2360         r600_blit_fini(rdev);
2361         r600_cp_fini(rdev);
2362         r600_wb_fini(rdev);
2363         r600_irq_fini(rdev);
2364         radeon_irq_kms_fini(rdev);
2365         r600_pcie_gart_fini(rdev);
2366         radeon_agp_fini(rdev);
2367         radeon_gem_fini(rdev);
2368         radeon_fence_driver_fini(rdev);
2369         radeon_clocks_fini(rdev);
2370         radeon_bo_fini(rdev);
2371         radeon_atombios_fini(rdev);
2372         kfree(rdev->bios);
2373         rdev->bios = NULL;
2374         radeon_dummy_page_fini(rdev);
2375 }
2376
2377
2378 /*
2379  * CS stuff
2380  */
2381 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2382 {
2383         /* FIXME: implement */
2384         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2385         radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
2386         radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
2387         radeon_ring_write(rdev, ib->length_dw);
2388 }
2389
2390 int r600_ib_test(struct radeon_device *rdev)
2391 {
2392         struct radeon_ib *ib;
2393         uint32_t scratch;
2394         uint32_t tmp = 0;
2395         unsigned i;
2396         int r;
2397
2398         r = radeon_scratch_get(rdev, &scratch);
2399         if (r) {
2400                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
2401                 return r;
2402         }
2403         WREG32(scratch, 0xCAFEDEAD);
2404         r = radeon_ib_get(rdev, &ib);
2405         if (r) {
2406                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2407                 return r;
2408         }
2409         ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
2410         ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2411         ib->ptr[2] = 0xDEADBEEF;
2412         ib->ptr[3] = PACKET2(0);
2413         ib->ptr[4] = PACKET2(0);
2414         ib->ptr[5] = PACKET2(0);
2415         ib->ptr[6] = PACKET2(0);
2416         ib->ptr[7] = PACKET2(0);
2417         ib->ptr[8] = PACKET2(0);
2418         ib->ptr[9] = PACKET2(0);
2419         ib->ptr[10] = PACKET2(0);
2420         ib->ptr[11] = PACKET2(0);
2421         ib->ptr[12] = PACKET2(0);
2422         ib->ptr[13] = PACKET2(0);
2423         ib->ptr[14] = PACKET2(0);
2424         ib->ptr[15] = PACKET2(0);
2425         ib->length_dw = 16;
2426         r = radeon_ib_schedule(rdev, ib);
2427         if (r) {
2428                 radeon_scratch_free(rdev, scratch);
2429                 radeon_ib_free(rdev, &ib);
2430                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2431                 return r;
2432         }
2433         r = radeon_fence_wait(ib->fence, false);
2434         if (r) {
2435                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2436                 return r;
2437         }
2438         for (i = 0; i < rdev->usec_timeout; i++) {
2439                 tmp = RREG32(scratch);
2440                 if (tmp == 0xDEADBEEF)
2441                         break;
2442                 DRM_UDELAY(1);
2443         }
2444         if (i < rdev->usec_timeout) {
2445                 DRM_INFO("ib test succeeded in %u usecs\n", i);
2446         } else {
2447                 DRM_ERROR("radeon: ib test failed (sracth(0x%04X)=0x%08X)\n",
2448                           scratch, tmp);
2449                 r = -EINVAL;
2450         }
2451         radeon_scratch_free(rdev, scratch);
2452         radeon_ib_free(rdev, &ib);
2453         return r;
2454 }
2455
2456 /*
2457  * Interrupts
2458  *
2459  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
2460  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
2461  * writing to the ring and the GPU consuming, the GPU writes to the ring
2462  * and host consumes.  As the host irq handler processes interrupts, it
2463  * increments the rptr.  When the rptr catches up with the wptr, all the
2464  * current interrupts have been processed.
2465  */
2466
2467 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
2468 {
2469         u32 rb_bufsz;
2470
2471         /* Align ring size */
2472         rb_bufsz = drm_order(ring_size / 4);
2473         ring_size = (1 << rb_bufsz) * 4;
2474         rdev->ih.ring_size = ring_size;
2475         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
2476         rdev->ih.rptr = 0;
2477 }
2478
2479 static int r600_ih_ring_alloc(struct radeon_device *rdev)
2480 {
2481         int r;
2482
2483         /* Allocate ring buffer */
2484         if (rdev->ih.ring_obj == NULL) {
2485                 r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
2486                                      true,
2487                                      RADEON_GEM_DOMAIN_GTT,
2488                                      &rdev->ih.ring_obj);
2489                 if (r) {
2490                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
2491                         return r;
2492                 }
2493                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2494                 if (unlikely(r != 0))
2495                         return r;
2496                 r = radeon_bo_pin(rdev->ih.ring_obj,
2497                                   RADEON_GEM_DOMAIN_GTT,
2498                                   &rdev->ih.gpu_addr);
2499                 if (r) {
2500                         radeon_bo_unreserve(rdev->ih.ring_obj);
2501                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
2502                         return r;
2503                 }
2504                 r = radeon_bo_kmap(rdev->ih.ring_obj,
2505                                    (void **)&rdev->ih.ring);
2506                 radeon_bo_unreserve(rdev->ih.ring_obj);
2507                 if (r) {
2508                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
2509                         return r;
2510                 }
2511         }
2512         return 0;
2513 }
2514
2515 static void r600_ih_ring_fini(struct radeon_device *rdev)
2516 {
2517         int r;
2518         if (rdev->ih.ring_obj) {
2519                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2520                 if (likely(r == 0)) {
2521                         radeon_bo_kunmap(rdev->ih.ring_obj);
2522                         radeon_bo_unpin(rdev->ih.ring_obj);
2523                         radeon_bo_unreserve(rdev->ih.ring_obj);
2524                 }
2525                 radeon_bo_unref(&rdev->ih.ring_obj);
2526                 rdev->ih.ring = NULL;
2527                 rdev->ih.ring_obj = NULL;
2528         }
2529 }
2530
2531 void r600_rlc_stop(struct radeon_device *rdev)
2532 {
2533
2534         if ((rdev->family >= CHIP_RV770) &&
2535             (rdev->family <= CHIP_RV740)) {
2536                 /* r7xx asics need to soft reset RLC before halting */
2537                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
2538                 RREG32(SRBM_SOFT_RESET);
2539                 udelay(15000);
2540                 WREG32(SRBM_SOFT_RESET, 0);
2541                 RREG32(SRBM_SOFT_RESET);
2542         }
2543
2544         WREG32(RLC_CNTL, 0);
2545 }
2546
2547 static void r600_rlc_start(struct radeon_device *rdev)
2548 {
2549         WREG32(RLC_CNTL, RLC_ENABLE);
2550 }
2551
2552 static int r600_rlc_init(struct radeon_device *rdev)
2553 {
2554         u32 i;
2555         const __be32 *fw_data;
2556
2557         if (!rdev->rlc_fw)
2558                 return -EINVAL;
2559
2560         r600_rlc_stop(rdev);
2561
2562         WREG32(RLC_HB_BASE, 0);
2563         WREG32(RLC_HB_CNTL, 0);
2564         WREG32(RLC_HB_RPTR, 0);
2565         WREG32(RLC_HB_WPTR, 0);
2566         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
2567         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2568         WREG32(RLC_MC_CNTL, 0);
2569         WREG32(RLC_UCODE_CNTL, 0);
2570
2571         fw_data = (const __be32 *)rdev->rlc_fw->data;
2572         if (rdev->family >= CHIP_CEDAR) {
2573                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
2574                         WREG32(RLC_UCODE_ADDR, i);
2575                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2576                 }
2577         } else if (rdev->family >= CHIP_RV770) {
2578                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
2579                         WREG32(RLC_UCODE_ADDR, i);
2580                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2581                 }
2582         } else {
2583                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
2584                         WREG32(RLC_UCODE_ADDR, i);
2585                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2586                 }
2587         }
2588         WREG32(RLC_UCODE_ADDR, 0);
2589
2590         r600_rlc_start(rdev);
2591
2592         return 0;
2593 }
2594
2595 static void r600_enable_interrupts(struct radeon_device *rdev)
2596 {
2597         u32 ih_cntl = RREG32(IH_CNTL);
2598         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2599
2600         ih_cntl |= ENABLE_INTR;
2601         ih_rb_cntl |= IH_RB_ENABLE;
2602         WREG32(IH_CNTL, ih_cntl);
2603         WREG32(IH_RB_CNTL, ih_rb_cntl);
2604         rdev->ih.enabled = true;
2605 }
2606
2607 void r600_disable_interrupts(struct radeon_device *rdev)
2608 {
2609         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2610         u32 ih_cntl = RREG32(IH_CNTL);
2611
2612         ih_rb_cntl &= ~IH_RB_ENABLE;
2613         ih_cntl &= ~ENABLE_INTR;
2614         WREG32(IH_RB_CNTL, ih_rb_cntl);
2615         WREG32(IH_CNTL, ih_cntl);
2616         /* set rptr, wptr to 0 */
2617         WREG32(IH_RB_RPTR, 0);
2618         WREG32(IH_RB_WPTR, 0);
2619         rdev->ih.enabled = false;
2620         rdev->ih.wptr = 0;
2621         rdev->ih.rptr = 0;
2622 }
2623
2624 static void r600_disable_interrupt_state(struct radeon_device *rdev)
2625 {
2626         u32 tmp;
2627
2628         WREG32(CP_INT_CNTL, 0);
2629         WREG32(GRBM_INT_CNTL, 0);
2630         WREG32(DxMODE_INT_MASK, 0);
2631         if (ASIC_IS_DCE3(rdev)) {
2632                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
2633                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
2634                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2635                 WREG32(DC_HPD1_INT_CONTROL, tmp);
2636                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2637                 WREG32(DC_HPD2_INT_CONTROL, tmp);
2638                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2639                 WREG32(DC_HPD3_INT_CONTROL, tmp);
2640                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2641                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2642                 if (ASIC_IS_DCE32(rdev)) {
2643                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2644                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2645                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2646                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2647                 }
2648         } else {
2649                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2650                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2651                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2652                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2653                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2654                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2655                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2656                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2657         }
2658 }
2659
2660 int r600_irq_init(struct radeon_device *rdev)
2661 {
2662         int ret = 0;
2663         int rb_bufsz;
2664         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
2665
2666         /* allocate ring */
2667         ret = r600_ih_ring_alloc(rdev);
2668         if (ret)
2669                 return ret;
2670
2671         /* disable irqs */
2672         r600_disable_interrupts(rdev);
2673
2674         /* init rlc */
2675         ret = r600_rlc_init(rdev);
2676         if (ret) {
2677                 r600_ih_ring_fini(rdev);
2678                 return ret;
2679         }
2680
2681         /* setup interrupt control */
2682         /* set dummy read address to ring address */
2683         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
2684         interrupt_cntl = RREG32(INTERRUPT_CNTL);
2685         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
2686          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
2687          */
2688         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
2689         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
2690         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
2691         WREG32(INTERRUPT_CNTL, interrupt_cntl);
2692
2693         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
2694         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
2695
2696         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
2697                       IH_WPTR_OVERFLOW_CLEAR |
2698                       (rb_bufsz << 1));
2699         /* WPTR writeback, not yet */
2700         /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/
2701         WREG32(IH_RB_WPTR_ADDR_LO, 0);
2702         WREG32(IH_RB_WPTR_ADDR_HI, 0);
2703
2704         WREG32(IH_RB_CNTL, ih_rb_cntl);
2705
2706         /* set rptr, wptr to 0 */
2707         WREG32(IH_RB_RPTR, 0);
2708         WREG32(IH_RB_WPTR, 0);
2709
2710         /* Default settings for IH_CNTL (disabled at first) */
2711         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
2712         /* RPTR_REARM only works if msi's are enabled */
2713         if (rdev->msi_enabled)
2714                 ih_cntl |= RPTR_REARM;
2715
2716 #ifdef __BIG_ENDIAN
2717         ih_cntl |= IH_MC_SWAP(IH_MC_SWAP_32BIT);
2718 #endif
2719         WREG32(IH_CNTL, ih_cntl);
2720
2721         /* force the active interrupt state to all disabled */
2722         if (rdev->family >= CHIP_CEDAR)
2723                 evergreen_disable_interrupt_state(rdev);
2724         else
2725                 r600_disable_interrupt_state(rdev);
2726
2727         /* enable irqs */
2728         r600_enable_interrupts(rdev);
2729
2730         return ret;
2731 }
2732
2733 void r600_irq_suspend(struct radeon_device *rdev)
2734 {
2735         r600_irq_disable(rdev);
2736         r600_rlc_stop(rdev);
2737 }
2738
2739 void r600_irq_fini(struct radeon_device *rdev)
2740 {
2741         r600_irq_suspend(rdev);
2742         r600_ih_ring_fini(rdev);
2743 }
2744
2745 int r600_irq_set(struct radeon_device *rdev)
2746 {
2747         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2748         u32 mode_int = 0;
2749         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2750         u32 grbm_int_cntl = 0;
2751         u32 hdmi1, hdmi2;
2752
2753         if (!rdev->irq.installed) {
2754                 WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
2755                 return -EINVAL;
2756         }
2757         /* don't enable anything if the ih is disabled */
2758         if (!rdev->ih.enabled) {
2759                 r600_disable_interrupts(rdev);
2760                 /* force the active interrupt state to all disabled */
2761                 r600_disable_interrupt_state(rdev);
2762                 return 0;
2763         }
2764
2765         hdmi1 = RREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2766         if (ASIC_IS_DCE3(rdev)) {
2767                 hdmi2 = RREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2768                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2769                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2770                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2771                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2772                 if (ASIC_IS_DCE32(rdev)) {
2773                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2774                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2775                 }
2776         } else {
2777                 hdmi2 = RREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2778                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2779                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2780                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2781         }
2782
2783         if (rdev->irq.sw_int) {
2784                 DRM_DEBUG("r600_irq_set: sw int\n");
2785                 cp_int_cntl |= RB_INT_ENABLE;
2786         }
2787         if (rdev->irq.crtc_vblank_int[0]) {
2788                 DRM_DEBUG("r600_irq_set: vblank 0\n");
2789                 mode_int |= D1MODE_VBLANK_INT_MASK;
2790         }
2791         if (rdev->irq.crtc_vblank_int[1]) {
2792                 DRM_DEBUG("r600_irq_set: vblank 1\n");
2793                 mode_int |= D2MODE_VBLANK_INT_MASK;
2794         }
2795         if (rdev->irq.hpd[0]) {
2796                 DRM_DEBUG("r600_irq_set: hpd 1\n");
2797                 hpd1 |= DC_HPDx_INT_EN;
2798         }
2799         if (rdev->irq.hpd[1]) {
2800                 DRM_DEBUG("r600_irq_set: hpd 2\n");
2801                 hpd2 |= DC_HPDx_INT_EN;
2802         }
2803         if (rdev->irq.hpd[2]) {
2804                 DRM_DEBUG("r600_irq_set: hpd 3\n");
2805                 hpd3 |= DC_HPDx_INT_EN;
2806         }
2807         if (rdev->irq.hpd[3]) {
2808                 DRM_DEBUG("r600_irq_set: hpd 4\n");
2809                 hpd4 |= DC_HPDx_INT_EN;
2810         }
2811         if (rdev->irq.hpd[4]) {
2812                 DRM_DEBUG("r600_irq_set: hpd 5\n");
2813                 hpd5 |= DC_HPDx_INT_EN;
2814         }
2815         if (rdev->irq.hpd[5]) {
2816                 DRM_DEBUG("r600_irq_set: hpd 6\n");
2817                 hpd6 |= DC_HPDx_INT_EN;
2818         }
2819         if (rdev->irq.hdmi[0]) {
2820                 DRM_DEBUG("r600_irq_set: hdmi 1\n");
2821                 hdmi1 |= R600_HDMI_INT_EN;
2822         }
2823         if (rdev->irq.hdmi[1]) {
2824                 DRM_DEBUG("r600_irq_set: hdmi 2\n");
2825                 hdmi2 |= R600_HDMI_INT_EN;
2826         }
2827         if (rdev->irq.gui_idle) {
2828                 DRM_DEBUG("gui idle\n");
2829                 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2830         }
2831
2832         WREG32(CP_INT_CNTL, cp_int_cntl);
2833         WREG32(DxMODE_INT_MASK, mode_int);
2834         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2835         WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
2836         if (ASIC_IS_DCE3(rdev)) {
2837                 WREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, hdmi2);
2838                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2839                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2840                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2841                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2842                 if (ASIC_IS_DCE32(rdev)) {
2843                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
2844                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
2845                 }
2846         } else {
2847                 WREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, hdmi2);
2848                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
2849                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
2850                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2851         }
2852
2853         return 0;
2854 }
2855
2856 static inline void r600_irq_ack(struct radeon_device *rdev,
2857                                 u32 *disp_int,
2858                                 u32 *disp_int_cont,
2859                                 u32 *disp_int_cont2)
2860 {
2861         u32 tmp;
2862
2863         if (ASIC_IS_DCE3(rdev)) {
2864                 *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
2865                 *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
2866                 *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2867         } else {
2868                 *disp_int = RREG32(DISP_INTERRUPT_STATUS);
2869                 *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2870                 *disp_int_cont2 = 0;
2871         }
2872
2873         if (*disp_int & LB_D1_VBLANK_INTERRUPT)
2874                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2875         if (*disp_int & LB_D1_VLINE_INTERRUPT)
2876                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2877         if (*disp_int & LB_D2_VBLANK_INTERRUPT)
2878                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2879         if (*disp_int & LB_D2_VLINE_INTERRUPT)
2880                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2881         if (*disp_int & DC_HPD1_INTERRUPT) {
2882                 if (ASIC_IS_DCE3(rdev)) {
2883                         tmp = RREG32(DC_HPD1_INT_CONTROL);
2884                         tmp |= DC_HPDx_INT_ACK;
2885                         WREG32(DC_HPD1_INT_CONTROL, tmp);
2886                 } else {
2887                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
2888                         tmp |= DC_HPDx_INT_ACK;
2889                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2890                 }
2891         }
2892         if (*disp_int & DC_HPD2_INTERRUPT) {
2893                 if (ASIC_IS_DCE3(rdev)) {
2894                         tmp = RREG32(DC_HPD2_INT_CONTROL);
2895                         tmp |= DC_HPDx_INT_ACK;
2896                         WREG32(DC_HPD2_INT_CONTROL, tmp);
2897                 } else {
2898                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
2899                         tmp |= DC_HPDx_INT_ACK;
2900                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2901                 }
2902         }
2903         if (*disp_int_cont & DC_HPD3_INTERRUPT) {
2904                 if (ASIC_IS_DCE3(rdev)) {
2905                         tmp = RREG32(DC_HPD3_INT_CONTROL);
2906                         tmp |= DC_HPDx_INT_ACK;
2907                         WREG32(DC_HPD3_INT_CONTROL, tmp);
2908                 } else {
2909                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
2910                         tmp |= DC_HPDx_INT_ACK;
2911                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2912                 }
2913         }
2914         if (*disp_int_cont & DC_HPD4_INTERRUPT) {
2915                 tmp = RREG32(DC_HPD4_INT_CONTROL);
2916                 tmp |= DC_HPDx_INT_ACK;
2917                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2918         }
2919         if (ASIC_IS_DCE32(rdev)) {
2920                 if (*disp_int_cont2 & DC_HPD5_INTERRUPT) {
2921                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2922                         tmp |= DC_HPDx_INT_ACK;
2923                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2924                 }
2925                 if (*disp_int_cont2 & DC_HPD6_INTERRUPT) {
2926                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2927                         tmp |= DC_HPDx_INT_ACK;
2928                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2929                 }
2930         }
2931         if (RREG32(R600_HDMI_BLOCK1 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2932                 WREG32_P(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2933         }
2934         if (ASIC_IS_DCE3(rdev)) {
2935                 if (RREG32(R600_HDMI_BLOCK3 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2936                         WREG32_P(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2937                 }
2938         } else {
2939                 if (RREG32(R600_HDMI_BLOCK2 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2940                         WREG32_P(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2941                 }
2942         }
2943 }
2944
2945 void r600_irq_disable(struct radeon_device *rdev)
2946 {
2947         u32 disp_int, disp_int_cont, disp_int_cont2;
2948
2949         r600_disable_interrupts(rdev);
2950         /* Wait and acknowledge irq */
2951         mdelay(1);
2952         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2953         r600_disable_interrupt_state(rdev);
2954 }
2955
2956 static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
2957 {
2958         u32 wptr, tmp;
2959
2960         /* XXX use writeback */
2961         wptr = RREG32(IH_RB_WPTR);
2962
2963         if (wptr & RB_OVERFLOW) {
2964                 /* When a ring buffer overflow happen start parsing interrupt
2965                  * from the last not overwritten vector (wptr + 16). Hopefully
2966                  * this should allow us to catchup.
2967                  */
2968                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2969                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2970                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2971                 tmp = RREG32(IH_RB_CNTL);
2972                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
2973                 WREG32(IH_RB_CNTL, tmp);
2974         }
2975         return (wptr & rdev->ih.ptr_mask);
2976 }
2977
2978 /*        r600 IV Ring
2979  * Each IV ring entry is 128 bits:
2980  * [7:0]    - interrupt source id
2981  * [31:8]   - reserved
2982  * [59:32]  - interrupt source data
2983  * [127:60]  - reserved
2984  *
2985  * The basic interrupt vector entries
2986  * are decoded as follows:
2987  * src_id  src_data  description
2988  *      1         0  D1 Vblank
2989  *      1         1  D1 Vline
2990  *      5         0  D2 Vblank
2991  *      5         1  D2 Vline
2992  *     19         0  FP Hot plug detection A
2993  *     19         1  FP Hot plug detection B
2994  *     19         2  DAC A auto-detection
2995  *     19         3  DAC B auto-detection
2996  *     21         4  HDMI block A
2997  *     21         5  HDMI block B
2998  *    176         -  CP_INT RB
2999  *    177         -  CP_INT IB1
3000  *    178         -  CP_INT IB2
3001  *    181         -  EOP Interrupt
3002  *    233         -  GUI Idle
3003  *
3004  * Note, these are based on r600 and may need to be
3005  * adjusted or added to on newer asics
3006  */
3007
3008 int r600_irq_process(struct radeon_device *rdev)
3009 {
3010         u32 wptr = r600_get_ih_wptr(rdev);
3011         u32 rptr = rdev->ih.rptr;
3012         u32 src_id, src_data;
3013         u32 ring_index, disp_int, disp_int_cont, disp_int_cont2;
3014         unsigned long flags;
3015         bool queue_hotplug = false;
3016
3017         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3018         if (!rdev->ih.enabled)
3019                 return IRQ_NONE;
3020
3021         spin_lock_irqsave(&rdev->ih.lock, flags);
3022
3023         if (rptr == wptr) {
3024                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
3025                 return IRQ_NONE;
3026         }
3027         if (rdev->shutdown) {
3028                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
3029                 return IRQ_NONE;
3030         }
3031
3032 restart_ih:
3033         /* display interrupts */
3034         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
3035
3036         rdev->ih.wptr = wptr;
3037         while (rptr != wptr) {
3038                 /* wptr/rptr are in bytes! */
3039                 ring_index = rptr / 4;
3040                 src_id =  rdev->ih.ring[ring_index] & 0xff;
3041                 src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
3042
3043                 switch (src_id) {
3044                 case 1: /* D1 vblank/vline */
3045                         switch (src_data) {
3046                         case 0: /* D1 vblank */
3047                                 if (disp_int & LB_D1_VBLANK_INTERRUPT) {
3048                                         drm_handle_vblank(rdev->ddev, 0);
3049                                         rdev->pm.vblank_sync = true;
3050                                         wake_up(&rdev->irq.vblank_queue);
3051                                         disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3052                                         DRM_DEBUG("IH: D1 vblank\n");
3053                                 }
3054                                 break;
3055                         case 1: /* D1 vline */
3056                                 if (disp_int & LB_D1_VLINE_INTERRUPT) {
3057                                         disp_int &= ~LB_D1_VLINE_INTERRUPT;
3058                                         DRM_DEBUG("IH: D1 vline\n");
3059                                 }
3060                                 break;
3061                         default:
3062                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3063                                 break;
3064                         }
3065                         break;
3066                 case 5: /* D2 vblank/vline */
3067                         switch (src_data) {
3068                         case 0: /* D2 vblank */
3069                                 if (disp_int & LB_D2_VBLANK_INTERRUPT) {
3070                                         drm_handle_vblank(rdev->ddev, 1);
3071                                         rdev->pm.vblank_sync = true;
3072                                         wake_up(&rdev->irq.vblank_queue);
3073                                         disp_int &= ~LB_D2_VBLANK_INTERRUPT;
3074                                         DRM_DEBUG("IH: D2 vblank\n");
3075                                 }
3076                                 break;
3077                         case 1: /* D1 vline */
3078                                 if (disp_int & LB_D2_VLINE_INTERRUPT) {
3079                                         disp_int &= ~LB_D2_VLINE_INTERRUPT;
3080                                         DRM_DEBUG("IH: D2 vline\n");
3081                                 }
3082                                 break;
3083                         default:
3084                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3085                                 break;
3086                         }
3087                         break;
3088                 case 19: /* HPD/DAC hotplug */
3089                         switch (src_data) {
3090                         case 0:
3091                                 if (disp_int & DC_HPD1_INTERRUPT) {
3092                                         disp_int &= ~DC_HPD1_INTERRUPT;
3093                                         queue_hotplug = true;
3094                                         DRM_DEBUG("IH: HPD1\n");
3095                                 }
3096                                 break;
3097                         case 1:
3098                                 if (disp_int & DC_HPD2_INTERRUPT) {
3099                                         disp_int &= ~DC_HPD2_INTERRUPT;
3100                                         queue_hotplug = true;
3101                                         DRM_DEBUG("IH: HPD2\n");
3102                                 }
3103                                 break;
3104                         case 4:
3105                                 if (disp_int_cont & DC_HPD3_INTERRUPT) {
3106                                         disp_int_cont &= ~DC_HPD3_INTERRUPT;
3107                                         queue_hotplug = true;
3108                                         DRM_DEBUG("IH: HPD3\n");
3109                                 }
3110                                 break;
3111                         case 5:
3112                                 if (disp_int_cont & DC_HPD4_INTERRUPT) {
3113                                         disp_int_cont &= ~DC_HPD4_INTERRUPT;
3114                                         queue_hotplug = true;
3115                                         DRM_DEBUG("IH: HPD4\n");
3116                                 }
3117                                 break;
3118                         case 10:
3119                                 if (disp_int_cont2 & DC_HPD5_INTERRUPT) {
3120                                         disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
3121                                         queue_hotplug = true;
3122                                         DRM_DEBUG("IH: HPD5\n");
3123                                 }
3124                                 break;
3125                         case 12:
3126                                 if (disp_int_cont2 & DC_HPD6_INTERRUPT) {
3127                                         disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
3128                                         queue_hotplug = true;
3129                                         DRM_DEBUG("IH: HPD6\n");
3130                                 }
3131                                 break;
3132                         default:
3133                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3134                                 break;
3135                         }
3136                         break;
3137                 case 21: /* HDMI */
3138                         DRM_DEBUG("IH: HDMI: 0x%x\n", src_data);
3139                         r600_audio_schedule_polling(rdev);
3140                         break;
3141                 case 176: /* CP_INT in ring buffer */
3142                 case 177: /* CP_INT in IB1 */
3143                 case 178: /* CP_INT in IB2 */
3144                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
3145                         radeon_fence_process(rdev);
3146                         break;
3147                 case 181: /* CP EOP event */
3148                         DRM_DEBUG("IH: CP EOP\n");
3149                         break;
3150                 case 233: /* GUI IDLE */
3151                         DRM_DEBUG("IH: CP EOP\n");
3152                         rdev->pm.gui_idle = true;
3153                         wake_up(&rdev->irq.idle_queue);
3154                         break;
3155                 default:
3156                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3157                         break;
3158                 }
3159
3160                 /* wptr/rptr are in bytes! */
3161                 rptr += 16;
3162                 rptr &= rdev->ih.ptr_mask;
3163         }
3164         /* make sure wptr hasn't changed while processing */
3165         wptr = r600_get_ih_wptr(rdev);
3166         if (wptr != rdev->ih.wptr)
3167                 goto restart_ih;
3168         if (queue_hotplug)
3169                 queue_work(rdev->wq, &rdev->hotplug_work);
3170         rdev->ih.rptr = rptr;
3171         WREG32(IH_RB_RPTR, rdev->ih.rptr);
3172         spin_unlock_irqrestore(&rdev->ih.lock, flags);
3173         return IRQ_HANDLED;
3174 }
3175
3176 /*
3177  * Debugfs info
3178  */
3179 #if defined(CONFIG_DEBUG_FS)
3180
3181 static int r600_debugfs_cp_ring_info(struct seq_file *m, void *data)
3182 {
3183         struct drm_info_node *node = (struct drm_info_node *) m->private;
3184         struct drm_device *dev = node->minor->dev;
3185         struct radeon_device *rdev = dev->dev_private;
3186         unsigned count, i, j;
3187
3188         radeon_ring_free_size(rdev);
3189         count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
3190         seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
3191         seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
3192         seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
3193         seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
3194         seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
3195         seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
3196         seq_printf(m, "%u dwords in ring\n", count);
3197         i = rdev->cp.rptr;
3198         for (j = 0; j <= count; j++) {
3199                 seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
3200                 i = (i + 1) & rdev->cp.ptr_mask;
3201         }
3202         return 0;
3203 }
3204
3205 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
3206 {
3207         struct drm_info_node *node = (struct drm_info_node *) m->private;
3208         struct drm_device *dev = node->minor->dev;
3209         struct radeon_device *rdev = dev->dev_private;
3210
3211         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
3212         DREG32_SYS(m, rdev, VM_L2_STATUS);
3213         return 0;
3214 }
3215
3216 static struct drm_info_list r600_mc_info_list[] = {
3217         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
3218         {"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
3219 };
3220 #endif
3221
3222 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
3223 {
3224 #if defined(CONFIG_DEBUG_FS)
3225         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
3226 #else
3227         return 0;
3228 #endif
3229 }
3230
3231 /**
3232  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
3233  * rdev: radeon device structure
3234  * bo: buffer object struct which userspace is waiting for idle
3235  *
3236  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
3237  * through ring buffer, this leads to corruption in rendering, see
3238  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
3239  * directly perform HDP flush by writing register through MMIO.
3240  */
3241 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
3242 {
3243         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3244 }