SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/slab.h>
  29 #include <linux/seq_file.h>
  30 #include <linux/firmware.h>
  31 #include <linux/platform_device.h>
  32 #include "drmP.h"
  33 #include "radeon_drm.h"
  34 #include "radeon.h"
  35 #include "radeon_asic.h"
  36 #include "radeon_mode.h"
  37 #include "r600d.h"
  38 #include "atom.h"
  39 #include "avivod.h"
  40
  41 #define PFP_UCODE_SIZE 576
  42 #define PM4_UCODE_SIZE 1792
  43 #define RLC_UCODE_SIZE 768
  44 #define R700_PFP_UCODE_SIZE 848
  45 #define R700_PM4_UCODE_SIZE 1360
  46 #define R700_RLC_UCODE_SIZE 1024
  47 #define EVERGREEN_PFP_UCODE_SIZE 1120
  48 #define EVERGREEN_PM4_UCODE_SIZE 1376
  49 #define EVERGREEN_RLC_UCODE_SIZE 768
  50
  51 /* Firmware Names */
  52 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  53 MODULE_FIRMWARE("radeon/R600_me.bin");
  54 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  55 MODULE_FIRMWARE("radeon/RV610_me.bin");
  56 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  57 MODULE_FIRMWARE("radeon/RV630_me.bin");
  58 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  59 MODULE_FIRMWARE("radeon/RV620_me.bin");
  60 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  61 MODULE_FIRMWARE("radeon/RV635_me.bin");
  62 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  63 MODULE_FIRMWARE("radeon/RV670_me.bin");
  64 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  65 MODULE_FIRMWARE("radeon/RS780_me.bin");
  66 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  67 MODULE_FIRMWARE("radeon/RV770_me.bin");
  68 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  69 MODULE_FIRMWARE("radeon/RV730_me.bin");
  70 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  71 MODULE_FIRMWARE("radeon/RV710_me.bin");
  72 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  73 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  74 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  75 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  76 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
  77 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  78 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  79 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
  80 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  81 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  82 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
  83 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
  84 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  85 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
  86
  87 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
  88
  89 /* r600,rv610,rv630,rv620,rv635,rv670 */
  90 int r600_mc_wait_for_idle(struct radeon_device *rdev);
  91 void r600_gpu_init(struct radeon_device *rdev);
  92 void r600_fini(struct radeon_device *rdev);
  93 void r600_irq_disable(struct radeon_device *rdev);
  94
  95 void r600_get_power_state(struct radeon_device *rdev,
  96                           enum radeon_pm_action action)
  97 {
  98         int i;
  99
 100         rdev->pm.can_upclock = true;
 101         rdev->pm.can_downclock = true;
 102
 103         /* power state array is low to high, default is first */
 104         if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
 105                 int min_power_state_index = 0;
 106
 107                 if (rdev->pm.num_power_states > 2)
 108                         min_power_state_index = 1;
 109
 110                 switch (action) {
 111                 case PM_ACTION_MINIMUM:
 112                         rdev->pm.requested_power_state_index = min_power_state_index;
 113                         rdev->pm.requested_clock_mode_index = 0;
 114                         rdev->pm.can_downclock = false;
 115                         break;
 116                 case PM_ACTION_DOWNCLOCK:
 117                         if (rdev->pm.current_power_state_index == min_power_state_index) {
 118                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 119                                 rdev->pm.can_downclock = false;
 120                         } else {
 121                                 if (rdev->pm.active_crtc_count > 1) {
 122                                         for (i = 0; i < rdev->pm.num_power_states; i++) {
 123                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_SINGLE_DISPLAY_ONLY)
 124                                                         continue;
 125                                                 else if (i >= rdev->pm.current_power_state_index) {
 126                                                         rdev->pm.requested_power_state_index =
 127                                                                 rdev->pm.current_power_state_index;
 128                                                         break;
 129                                                 } else {
 130                                                         rdev->pm.requested_power_state_index = i;
 131                                                         break;
 132                                                 }
 133                                         }
 134                                 } else
 135                                         rdev->pm.requested_power_state_index =
 136                                                 rdev->pm.current_power_state_index - 1;
 137                         }
 138                         rdev->pm.requested_clock_mode_index = 0;
 139                         break;
 140                 case PM_ACTION_UPCLOCK:
 141                         if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
 142                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 143                                 rdev->pm.can_upclock = false;
 144                         } else {
 145                                 if (rdev->pm.active_crtc_count > 1) {
 146                                         for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
 147                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_SINGLE_DISPLAY_ONLY)
 148                                                         continue;
 149                                                 else if (i <= rdev->pm.current_power_state_index) {
 150                                                         rdev->pm.requested_power_state_index =
 151                                                                 rdev->pm.current_power_state_index;
 152                                                         break;
 153                                                 } else {
 154                                                         rdev->pm.requested_power_state_index = i;
 155                                                         break;
 156                                                 }
 157                                         }
 158                                 } else
 159                                         rdev->pm.requested_power_state_index =
 160                                                 rdev->pm.current_power_state_index + 1;
 161                         }
 162                         rdev->pm.requested_clock_mode_index = 0;
 163                         break;
 164                 case PM_ACTION_DEFAULT:
 165                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 166                         rdev->pm.requested_clock_mode_index = 0;
 167                         rdev->pm.can_upclock = false;
 168                         break;
 169                 case PM_ACTION_NONE:
 170                 default:
 171                         DRM_ERROR("Requested mode for not defined action\n");
 172                         return;
 173                 }
 174         } else {
 175                 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
 176                 /* for now just select the first power state and switch between clock modes */
 177                 /* power state array is low to high, default is first (0) */
 178                 if (rdev->pm.active_crtc_count > 1) {
 179                         rdev->pm.requested_power_state_index = -1;
 180                         /* start at 1 as we don't want the default mode */
 181                         for (i = 1; i < rdev->pm.num_power_states; i++) {
 182                                 if (rdev->pm.power_state[i].flags & RADEON_PM_SINGLE_DISPLAY_ONLY)
 183                                         continue;
 184                                 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
 185                                          (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
 186                                         rdev->pm.requested_power_state_index = i;
 187                                         break;
 188                                 }
 189                         }
 190                         /* if nothing selected, grab the default state. */
 191                         if (rdev->pm.requested_power_state_index == -1)
 192                                 rdev->pm.requested_power_state_index = 0;
 193                 } else
 194                         rdev->pm.requested_power_state_index = 1;
 195
 196                 switch (action) {
 197                 case PM_ACTION_MINIMUM:
 198                         rdev->pm.requested_clock_mode_index = 0;
 199                         rdev->pm.can_downclock = false;
 200                         break;
 201                 case PM_ACTION_DOWNCLOCK:
 202                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 203                                 if (rdev->pm.current_clock_mode_index == 0) {
 204                                         rdev->pm.requested_clock_mode_index = 0;
 205                                         rdev->pm.can_downclock = false;
 206                                 } else
 207                                         rdev->pm.requested_clock_mode_index =
 208                                                 rdev->pm.current_clock_mode_index - 1;
 209                         } else {
 210                                 rdev->pm.requested_clock_mode_index = 0;
 211                                 rdev->pm.can_downclock = false;
 212                         }
 213                         break;
 214                 case PM_ACTION_UPCLOCK:
 215                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 216                                 if (rdev->pm.current_clock_mode_index ==
 217                                     (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
 218                                         rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
 219                                         rdev->pm.can_upclock = false;
 220                                 } else
 221                                         rdev->pm.requested_clock_mode_index =
 222                                                 rdev->pm.current_clock_mode_index + 1;
 223                         } else {
 224                                 rdev->pm.requested_clock_mode_index =
 225                                         rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
 226                                 rdev->pm.can_upclock = false;
 227                         }
 228                         break;
 229                 case PM_ACTION_DEFAULT:
 230                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 231                         rdev->pm.requested_clock_mode_index = 0;
 232                         rdev->pm.can_upclock = false;
 233                         break;
 234                 case PM_ACTION_NONE:
 235                 default:
 236                         DRM_ERROR("Requested mode for not defined action\n");
 237                         return;
 238                 }
 239         }
 240
 241         DRM_INFO("Requested: e: %d m: %d p: %d\n",
 242                  rdev->pm.power_state[rdev->pm.requested_power_state_index].
 243                  clock_info[rdev->pm.requested_clock_mode_index].sclk,
 244                  rdev->pm.power_state[rdev->pm.requested_power_state_index].
 245                  clock_info[rdev->pm.requested_clock_mode_index].mclk,
 246                  rdev->pm.power_state[rdev->pm.requested_power_state_index].
 247                  pcie_lanes);
 248 }
 249
 250 void r600_set_power_state(struct radeon_device *rdev, bool static_switch)
 251 {
 252         u32 sclk, mclk;
 253
 254         if ((rdev->pm.requested_clock_mode_index == rdev->pm.current_clock_mode_index) &&
 255             (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index))
 256                 return;
 257
 258         if (radeon_gui_idle(rdev)) {
 259
 260                 sclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
 261                         clock_info[rdev->pm.requested_clock_mode_index].sclk;
 262                 if (sclk > rdev->clock.default_sclk)
 263                         sclk = rdev->clock.default_sclk;
 264
 265                 mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
 266                         clock_info[rdev->pm.requested_clock_mode_index].mclk;
 267                 if (mclk > rdev->clock.default_mclk)
 268                         mclk = rdev->clock.default_mclk;
 269
 270                 /* voltage, pcie lanes, etc.*/
 271                 radeon_pm_misc(rdev);
 272
 273                 if (static_switch) {
 274
 275                         /* set engine clock */
 276                         if (sclk != rdev->pm.current_sclk) {
 277                                 radeon_set_engine_clock(rdev, sclk);
 278                                 rdev->pm.current_sclk = sclk;
 279                                 DRM_INFO("Setting: e: %d\n", sclk);
 280                         }
 281
 282                         /* set memory clock */
 283                         if (rdev->asic->set_memory_clock && (mclk != rdev->pm.current_mclk)) {
 284                                 radeon_pm_prepare(rdev);
 285                                 radeon_set_memory_clock(rdev, mclk);
 286                                 radeon_pm_finish(rdev);
 287                                 rdev->pm.current_mclk = mclk;
 288                                 DRM_INFO("Setting: m: %d\n", mclk);
 289                         }
 290
 291                 } else {
 292                         u32 position;
 293                         u32 vbl;
 294
 295                         radeon_sync_with_vblank(rdev);
 296
 297                         if (!radeon_pm_in_vbl(rdev))
 298                                 return;
 299
 300                         if (rdev->pm.active_crtcs & (1 << 0)) {
 301                                 vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END);
 302                                 position = RREG32(AVIVO_D1CRTC_STATUS_POSITION);
 303                                 position &= 0xfff;
 304                                 vbl &= 0xfff;
 305
 306                                 if (position < vbl && position > 1)
 307                                         return;
 308                         }
 309
 310                         if (rdev->pm.active_crtcs & (1 << 1)) {
 311                                 vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END);
 312                                 position = RREG32(AVIVO_D2CRTC_STATUS_POSITION);
 313                                 position &= 0xfff;
 314                                 vbl &= 0xfff;
 315
 316                                 if (position < vbl && position > 1)
 317                                         return;
 318                         }
 319
 320                         if (sclk != rdev->pm.current_sclk) {
 321                                 radeon_pm_debug_check_in_vbl(rdev, false);
 322                                 radeon_set_engine_clock(rdev, sclk);
 323                                 radeon_pm_debug_check_in_vbl(rdev, true);
 324                                 rdev->pm.current_sclk = sclk;
 325                                 DRM_INFO("Setting: e: %d\n", sclk);
 326                         }
 327
 328                         /* set memory clock */
 329                         if (rdev->asic->set_memory_clock && (mclk != rdev->pm.current_mclk)) {
 330                                 radeon_pm_debug_check_in_vbl(rdev, false);
 331                                 radeon_pm_prepare(rdev);
 332                                 radeon_set_memory_clock(rdev, mclk);
 333                                 radeon_pm_finish(rdev);
 334                                 radeon_pm_debug_check_in_vbl(rdev, true);
 335                                 rdev->pm.current_mclk = mclk;
 336                                 DRM_INFO("Setting: m: %d\n", mclk);
 337                         }
 338                 }
 339
 340                 rdev->pm.current_power_state_index = rdev->pm.requested_power_state_index;
 341                 rdev->pm.current_clock_mode_index = rdev->pm.requested_clock_mode_index;
 342         } else
 343                 DRM_INFO("GUI not idle!!!\n");
 344 }
 345
 346 void r600_pm_misc(struct radeon_device *rdev)
 347 {
 348
 349 }
 350
 351 bool r600_gui_idle(struct radeon_device *rdev)
 352 {
 353         if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
 354                 return false;
 355         else
 356                 return true;
 357 }
 358
 359 /* hpd for digital panel detect/disconnect */
 360 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 361 {
 362         bool connected = false;
 363
 364         if (ASIC_IS_DCE3(rdev)) {
 365                 switch (hpd) {
 366                 case RADEON_HPD_1:
 367                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 368                                 connected = true;
 369                         break;
 370                 case RADEON_HPD_2:
 371                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 372                                 connected = true;
 373                         break;
 374                 case RADEON_HPD_3:
 375                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 376                                 connected = true;
 377                         break;
 378                 case RADEON_HPD_4:
 379                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 380                                 connected = true;
 381                         break;
 382                         /* DCE 3.2 */
 383                 case RADEON_HPD_5:
 384                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 385                                 connected = true;
 386                         break;
 387                 case RADEON_HPD_6:
 388                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 389                                 connected = true;
 390                         break;
 391                 default:
 392                         break;
 393                 }
 394         } else {
 395                 switch (hpd) {
 396                 case RADEON_HPD_1:
 397                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 398                                 connected = true;
 399                         break;
 400                 case RADEON_HPD_2:
 401                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 402                                 connected = true;
 403                         break;
 404                 case RADEON_HPD_3:
 405                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 406                                 connected = true;
 407                         break;
 408                 default:
 409                         break;
 410                 }
 411         }
 412         return connected;
 413 }
 414
 415 void r600_hpd_set_polarity(struct radeon_device *rdev,
 416                            enum radeon_hpd_id hpd)
 417 {
 418         u32 tmp;
 419         bool connected = r600_hpd_sense(rdev, hpd);
 420
 421         if (ASIC_IS_DCE3(rdev)) {
 422                 switch (hpd) {
 423                 case RADEON_HPD_1:
 424                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 425                         if (connected)
 426                                 tmp &= ~DC_HPDx_INT_POLARITY;
 427                         else
 428                                 tmp |= DC_HPDx_INT_POLARITY;
 429                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 430                         break;
 431                 case RADEON_HPD_2:
 432                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 433                         if (connected)
 434                                 tmp &= ~DC_HPDx_INT_POLARITY;
 435                         else
 436                                 tmp |= DC_HPDx_INT_POLARITY;
 437                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 438                         break;
 439                 case RADEON_HPD_3:
 440                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 441                         if (connected)
 442                                 tmp &= ~DC_HPDx_INT_POLARITY;
 443                         else
 444                                 tmp |= DC_HPDx_INT_POLARITY;
 445                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 446                         break;
 447                 case RADEON_HPD_4:
 448                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 449                         if (connected)
 450                                 tmp &= ~DC_HPDx_INT_POLARITY;
 451                         else
 452                                 tmp |= DC_HPDx_INT_POLARITY;
 453                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 454                         break;
 455                 case RADEON_HPD_5:
 456                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 457                         if (connected)
 458                                 tmp &= ~DC_HPDx_INT_POLARITY;
 459                         else
 460                                 tmp |= DC_HPDx_INT_POLARITY;
 461                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 462                         break;
 463                         /* DCE 3.2 */
 464                 case RADEON_HPD_6:
 465                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 466                         if (connected)
 467                                 tmp &= ~DC_HPDx_INT_POLARITY;
 468                         else
 469                                 tmp |= DC_HPDx_INT_POLARITY;
 470                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 471                         break;
 472                 default:
 473                         break;
 474                 }
 475         } else {
 476                 switch (hpd) {
 477                 case RADEON_HPD_1:
 478                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 479                         if (connected)
 480                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 481                         else
 482                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 483                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 484                         break;
 485                 case RADEON_HPD_2:
 486                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 487                         if (connected)
 488                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 489                         else
 490                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 491                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 492                         break;
 493                 case RADEON_HPD_3:
 494                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 495                         if (connected)
 496                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 497                         else
 498                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 499                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 500                         break;
 501                 default:
 502                         break;
 503                 }
 504         }
 505 }
 506
 507 void r600_hpd_init(struct radeon_device *rdev)
 508 {
 509         struct drm_device *dev = rdev->ddev;
 510         struct drm_connector *connector;
 511
 512         if (ASIC_IS_DCE3(rdev)) {
 513                 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 514                 if (ASIC_IS_DCE32(rdev))
 515                         tmp |= DC_HPDx_EN;
 516
 517                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 518                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 519                         switch (radeon_connector->hpd.hpd) {
 520                         case RADEON_HPD_1:
 521                                 WREG32(DC_HPD1_CONTROL, tmp);
 522                                 rdev->irq.hpd[0] = true;
 523                                 break;
 524                         case RADEON_HPD_2:
 525                                 WREG32(DC_HPD2_CONTROL, tmp);
 526                                 rdev->irq.hpd[1] = true;
 527                                 break;
 528                         case RADEON_HPD_3:
 529                                 WREG32(DC_HPD3_CONTROL, tmp);
 530                                 rdev->irq.hpd[2] = true;
 531                                 break;
 532                         case RADEON_HPD_4:
 533                                 WREG32(DC_HPD4_CONTROL, tmp);
 534                                 rdev->irq.hpd[3] = true;
 535                                 break;
 536                                 /* DCE 3.2 */
 537                         case RADEON_HPD_5:
 538                                 WREG32(DC_HPD5_CONTROL, tmp);
 539                                 rdev->irq.hpd[4] = true;
 540                                 break;
 541                         case RADEON_HPD_6:
 542                                 WREG32(DC_HPD6_CONTROL, tmp);
 543                                 rdev->irq.hpd[5] = true;
 544                                 break;
 545                         default:
 546                                 break;
 547                         }
 548                 }
 549         } else {
 550                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 551                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 552                         switch (radeon_connector->hpd.hpd) {
 553                         case RADEON_HPD_1:
 554                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 555                                 rdev->irq.hpd[0] = true;
 556                                 break;
 557                         case RADEON_HPD_2:
 558                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 559                                 rdev->irq.hpd[1] = true;
 560                                 break;
 561                         case RADEON_HPD_3:
 562                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 563                                 rdev->irq.hpd[2] = true;
 564                                 break;
 565                         default:
 566                                 break;
 567                         }
 568                 }
 569         }
 570         if (rdev->irq.installed)
 571                 r600_irq_set(rdev);
 572 }
 573
 574 void r600_hpd_fini(struct radeon_device *rdev)
 575 {
 576         struct drm_device *dev = rdev->ddev;
 577         struct drm_connector *connector;
 578
 579         if (ASIC_IS_DCE3(rdev)) {
 580                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 581                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 582                         switch (radeon_connector->hpd.hpd) {
 583                         case RADEON_HPD_1:
 584                                 WREG32(DC_HPD1_CONTROL, 0);
 585                                 rdev->irq.hpd[0] = false;
 586                                 break;
 587                         case RADEON_HPD_2:
 588                                 WREG32(DC_HPD2_CONTROL, 0);
 589                                 rdev->irq.hpd[1] = false;
 590                                 break;
 591                         case RADEON_HPD_3:
 592                                 WREG32(DC_HPD3_CONTROL, 0);
 593                                 rdev->irq.hpd[2] = false;
 594                                 break;
 595                         case RADEON_HPD_4:
 596                                 WREG32(DC_HPD4_CONTROL, 0);
 597                                 rdev->irq.hpd[3] = false;
 598                                 break;
 599                                 /* DCE 3.2 */
 600                         case RADEON_HPD_5:
 601                                 WREG32(DC_HPD5_CONTROL, 0);
 602                                 rdev->irq.hpd[4] = false;
 603                                 break;
 604                         case RADEON_HPD_6:
 605                                 WREG32(DC_HPD6_CONTROL, 0);
 606                                 rdev->irq.hpd[5] = false;
 607                                 break;
 608                         default:
 609                                 break;
 610                         }
 611                 }
 612         } else {
 613                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 614                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 615                         switch (radeon_connector->hpd.hpd) {
 616                         case RADEON_HPD_1:
 617                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 618                                 rdev->irq.hpd[0] = false;
 619                                 break;
 620                         case RADEON_HPD_2:
 621                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 622                                 rdev->irq.hpd[1] = false;
 623                                 break;
 624                         case RADEON_HPD_3:
 625                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 626                                 rdev->irq.hpd[2] = false;
 627                                 break;
 628                         default:
 629                                 break;
 630                         }
 631                 }
 632         }
 633 }
 634
 635 /*
 636  * R600 PCIE GART
 637  */
 638 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 639 {
 640         unsigned i;
 641         u32 tmp;
 642
 643         /* flush hdp cache so updates hit vram */
 644         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 645
 646         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 647         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 648         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 649         for (i = 0; i < rdev->usec_timeout; i++) {
 650                 /* read MC_STATUS */
 651                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 652                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 653                 if (tmp == 2) {
 654                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 655                         return;
 656                 }
 657                 if (tmp) {
 658                         return;
 659                 }
 660                 udelay(1);
 661         }
 662 }
 663
 664 int r600_pcie_gart_init(struct radeon_device *rdev)
 665 {
 666         int r;
 667
 668         if (rdev->gart.table.vram.robj) {
 669                 WARN(1, "R600 PCIE GART already initialized.\n");
 670                 return 0;
 671         }
 672         /* Initialize common gart structure */
 673         r = radeon_gart_init(rdev);
 674         if (r)
 675                 return r;
 676         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 677         return radeon_gart_table_vram_alloc(rdev);
 678 }
 679
 680 int r600_pcie_gart_enable(struct radeon_device *rdev)
 681 {
 682         u32 tmp;
 683         int r, i;
 684
 685         if (rdev->gart.table.vram.robj == NULL) {
 686                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 687                 return -EINVAL;
 688         }
 689         r = radeon_gart_table_vram_pin(rdev);
 690         if (r)
 691                 return r;
 692         radeon_gart_restore(rdev);
 693
 694         /* Setup L2 cache */
 695         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 696                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 697                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 698         WREG32(VM_L2_CNTL2, 0);
 699         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 700         /* Setup TLB control */
 701         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 702                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 703                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 704                 ENABLE_WAIT_L2_QUERY;
 705         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 706         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 707         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 708         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 709         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 710         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 711         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 712         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 713         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 714         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 715         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 716         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 717         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 718         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 719         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 720         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 721         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 722         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 723                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 724         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 725                         (u32)(rdev->dummy_page.addr >> 12));
 726         for (i = 1; i < 7; i++)
 727                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 728
 729         r600_pcie_gart_tlb_flush(rdev);
 730         rdev->gart.ready = true;
 731         return 0;
 732 }
 733
 734 void r600_pcie_gart_disable(struct radeon_device *rdev)
 735 {
 736         u32 tmp;
 737         int i, r;
 738
 739         /* Disable all tables */
 740         for (i = 0; i < 7; i++)
 741                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 742
 743         /* Disable L2 cache */
 744         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 745                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 746         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 747         /* Setup L1 TLB control */
 748         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 749                 ENABLE_WAIT_L2_QUERY;
 750         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 751         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 752         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 753         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 754         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 755         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 756         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 757         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 758         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 759         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 760         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 761         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 762         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 763         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 764         if (rdev->gart.table.vram.robj) {
 765                 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
 766                 if (likely(r == 0)) {
 767                         radeon_bo_kunmap(rdev->gart.table.vram.robj);
 768                         radeon_bo_unpin(rdev->gart.table.vram.robj);
 769                         radeon_bo_unreserve(rdev->gart.table.vram.robj);
 770                 }
 771         }
 772 }
 773
 774 void r600_pcie_gart_fini(struct radeon_device *rdev)
 775 {
 776         radeon_gart_fini(rdev);
 777         r600_pcie_gart_disable(rdev);
 778         radeon_gart_table_vram_free(rdev);
 779 }
 780
 781 void r600_agp_enable(struct radeon_device *rdev)
 782 {
 783         u32 tmp;
 784         int i;
 785
 786         /* Setup L2 cache */
 787         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 788                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 789                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 790         WREG32(VM_L2_CNTL2, 0);
 791         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 792         /* Setup TLB control */
 793         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 794                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 795                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 796                 ENABLE_WAIT_L2_QUERY;
 797         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 798         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 799         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 800         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 801         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 802         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 803         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 804         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 805         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 806         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 807         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 808         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 809         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 810         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 811         for (i = 0; i < 7; i++)
 812                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 813 }
 814
 815 int r600_mc_wait_for_idle(struct radeon_device *rdev)
 816 {
 817         unsigned i;
 818         u32 tmp;
 819
 820         for (i = 0; i < rdev->usec_timeout; i++) {
 821                 /* read MC_STATUS */
 822                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
 823                 if (!tmp)
 824                         return 0;
 825                 udelay(1);
 826         }
 827         return -1;
 828 }
 829
 830 static void r600_mc_program(struct radeon_device *rdev)
 831 {
 832         struct rv515_mc_save save;
 833         u32 tmp;
 834         int i, j;
 835
 836         /* Initialize HDP */
 837         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 838                 WREG32((0x2c14 + j), 0x00000000);
 839                 WREG32((0x2c18 + j), 0x00000000);
 840                 WREG32((0x2c1c + j), 0x00000000);
 841                 WREG32((0x2c20 + j), 0x00000000);
 842                 WREG32((0x2c24 + j), 0x00000000);
 843         }
 844         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
 845
 846         rv515_mc_stop(rdev, &save);
 847         if (r600_mc_wait_for_idle(rdev)) {
 848                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 849         }
 850         /* Lockout access through VGA aperture (doesn't exist before R600) */
 851         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
 852         /* Update configuration */
 853         if (rdev->flags & RADEON_IS_AGP) {
 854                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
 855                         /* VRAM before AGP */
 856                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 857                                 rdev->mc.vram_start >> 12);
 858                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 859                                 rdev->mc.gtt_end >> 12);
 860                 } else {
 861                         /* VRAM after AGP */
 862                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 863                                 rdev->mc.gtt_start >> 12);
 864                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 865                                 rdev->mc.vram_end >> 12);
 866                 }
 867         } else {
 868                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
 869                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
 870         }
 871         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 872         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
 873         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
 874         WREG32(MC_VM_FB_LOCATION, tmp);
 875         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
 876         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
 877         WREG32(HDP_NONSURFACE_SIZE, rdev->mc.mc_vram_size | 0x3FF);
 878         if (rdev->flags & RADEON_IS_AGP) {
 879                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
 880                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
 881                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
 882         } else {
 883                 WREG32(MC_VM_AGP_BASE, 0);
 884                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
 885                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
 886         }
 887         if (r600_mc_wait_for_idle(rdev)) {
 888                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
 889         }
 890         rv515_mc_resume(rdev, &save);
 891         /* we need to own VRAM, so turn off the VGA renderer here
 892          * to stop it overwriting our objects */
 893         rv515_vga_render_disable(rdev);
 894 }
 895
 896 /**
 897  * r600_vram_gtt_location - try to find VRAM & GTT location
 898  * @rdev: radeon device structure holding all necessary informations
 899  * @mc: memory controller structure holding memory informations
 900  *
 901  * Function will place try to place VRAM at same place as in CPU (PCI)
 902  * address space as some GPU seems to have issue when we reprogram at
 903  * different address space.
 904  *
 905  * If there is not enough space to fit the unvisible VRAM after the
 906  * aperture then we limit the VRAM size to the aperture.
 907  *
 908  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
 909  * them to be in one from GPU point of view so that we can program GPU to
 910  * catch access outside them (weird GPU policy see ??).
 911  *
 912  * This function will never fails, worst case are limiting VRAM or GTT.
 913  *
 914  * Note: GTT start, end, size should be initialized before calling this
 915  * function on AGP platform.
 916  */
 917 void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 918 {
 919         u64 size_bf, size_af;
 920
 921         if (mc->mc_vram_size > 0xE0000000) {
 922                 /* leave room for at least 512M GTT */
 923                 dev_warn(rdev->dev, "limiting VRAM\n");
 924                 mc->real_vram_size = 0xE0000000;
 925                 mc->mc_vram_size = 0xE0000000;
 926         }
 927         if (rdev->flags & RADEON_IS_AGP) {
 928                 size_bf = mc->gtt_start;
 929                 size_af = 0xFFFFFFFF - mc->gtt_end + 1;
 930                 if (size_bf > size_af) {
 931                         if (mc->mc_vram_size > size_bf) {
 932                                 dev_warn(rdev->dev, "limiting VRAM\n");
 933                                 mc->real_vram_size = size_bf;
 934                                 mc->mc_vram_size = size_bf;
 935                         }
 936                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
 937                 } else {
 938                         if (mc->mc_vram_size > size_af) {
 939                                 dev_warn(rdev->dev, "limiting VRAM\n");
 940                                 mc->real_vram_size = size_af;
 941                                 mc->mc_vram_size = size_af;
 942                         }
 943                         mc->vram_start = mc->gtt_end;
 944                 }
 945                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 946                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
 947                                 mc->mc_vram_size >> 20, mc->vram_start,
 948                                 mc->vram_end, mc->real_vram_size >> 20);
 949         } else {
 950                 u64 base = 0;
 951                 if (rdev->flags & RADEON_IS_IGP)
 952                         base = (RREG32(MC_VM_FB_LOCATION) & 0xFFFF) << 24;
 953                 radeon_vram_location(rdev, &rdev->mc, base);
 954                 radeon_gtt_location(rdev, mc);
 955         }
 956 }
 957
 958 int r600_mc_init(struct radeon_device *rdev)
 959 {
 960         u32 tmp;
 961         int chansize, numchan;
 962
 963         /* Get VRAM informations */
 964         rdev->mc.vram_is_ddr = true;
 965         tmp = RREG32(RAMCFG);
 966         if (tmp & CHANSIZE_OVERRIDE) {
 967                 chansize = 16;
 968         } else if (tmp & CHANSIZE_MASK) {
 969                 chansize = 64;
 970         } else {
 971                 chansize = 32;
 972         }
 973         tmp = RREG32(CHMAP);
 974         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
 975         case 0:
 976         default:
 977                 numchan = 1;
 978                 break;
 979         case 1:
 980                 numchan = 2;
 981                 break;
 982         case 2:
 983                 numchan = 4;
 984                 break;
 985         case 3:
 986                 numchan = 8;
 987                 break;
 988         }
 989         rdev->mc.vram_width = numchan * chansize;
 990         /* Could aper size report 0 ? */
 991         rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
 992         rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
 993         /* Setup GPU memory space */
 994         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
 995         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
 996         rdev->mc.visible_vram_size = rdev->mc.aper_size;
 997         r600_vram_gtt_location(rdev, &rdev->mc);
 998
 999         if (rdev->flags & RADEON_IS_IGP)
1000                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1001         radeon_update_bandwidth_info(rdev);
1002         return 0;
1003 }
1004
1005 /* We doesn't check that the GPU really needs a reset we simply do the
1006  * reset, it's up to the caller to determine if the GPU needs one. We
1007  * might add an helper function to check that.
1008  */
1009 int r600_gpu_soft_reset(struct radeon_device *rdev)
1010 {
1011         struct rv515_mc_save save;
1012         u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) |
1013                                 S_008010_VGT_BUSY(1) | S_008010_TA03_BUSY(1) |
1014                                 S_008010_TC_BUSY(1) | S_008010_SX_BUSY(1) |
1015                                 S_008010_SH_BUSY(1) | S_008010_SPI03_BUSY(1) |
1016                                 S_008010_SMX_BUSY(1) | S_008010_SC_BUSY(1) |
1017                                 S_008010_PA_BUSY(1) | S_008010_DB03_BUSY(1) |
1018                                 S_008010_CR_BUSY(1) | S_008010_CB03_BUSY(1) |
1019                                 S_008010_GUI_ACTIVE(1);
1020         u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) | S_008014_SPI1_BUSY(1) |
1021                         S_008014_SPI2_BUSY(1) | S_008014_SPI3_BUSY(1) |
1022                         S_008014_TA0_BUSY(1) | S_008014_TA1_BUSY(1) |
1023                         S_008014_TA2_BUSY(1) | S_008014_TA3_BUSY(1) |
1024                         S_008014_DB0_BUSY(1) | S_008014_DB1_BUSY(1) |
1025                         S_008014_DB2_BUSY(1) | S_008014_DB3_BUSY(1) |
1026                         S_008014_CB0_BUSY(1) | S_008014_CB1_BUSY(1) |
1027                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
1028         u32 tmp;
1029
1030         dev_info(rdev->dev, "GPU softreset \n");
1031         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
1032                 RREG32(R_008010_GRBM_STATUS));
1033         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
1034                 RREG32(R_008014_GRBM_STATUS2));
1035         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
1036                 RREG32(R_000E50_SRBM_STATUS));
1037         rv515_mc_stop(rdev, &save);
1038         if (r600_mc_wait_for_idle(rdev)) {
1039                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1040         }
1041         /* Disable CP parsing/prefetching */
1042         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1043         /* Check if any of the rendering block is busy and reset it */
1044         if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) ||
1045             (RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
1046                 tmp = S_008020_SOFT_RESET_CR(1) |
1047                         S_008020_SOFT_RESET_DB(1) |
1048                         S_008020_SOFT_RESET_CB(1) |
1049                         S_008020_SOFT_RESET_PA(1) |
1050                         S_008020_SOFT_RESET_SC(1) |
1051                         S_008020_SOFT_RESET_SMX(1) |
1052                         S_008020_SOFT_RESET_SPI(1) |
1053                         S_008020_SOFT_RESET_SX(1) |
1054                         S_008020_SOFT_RESET_SH(1) |
1055                         S_008020_SOFT_RESET_TC(1) |
1056                         S_008020_SOFT_RESET_TA(1) |
1057                         S_008020_SOFT_RESET_VC(1) |
1058                         S_008020_SOFT_RESET_VGT(1);
1059                 dev_info(rdev->dev, "  R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1060                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1061                 RREG32(R_008020_GRBM_SOFT_RESET);
1062                 mdelay(15);
1063                 WREG32(R_008020_GRBM_SOFT_RESET, 0);
1064         }
1065         /* Reset CP (we always reset CP) */
1066         tmp = S_008020_SOFT_RESET_CP(1);
1067         dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1068         WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1069         RREG32(R_008020_GRBM_SOFT_RESET);
1070         mdelay(15);
1071         WREG32(R_008020_GRBM_SOFT_RESET, 0);
1072         /* Wait a little for things to settle down */
1073         mdelay(1);
1074         dev_info(rdev->dev, "  R_008010_GRBM_STATUS=0x%08X\n",
1075                 RREG32(R_008010_GRBM_STATUS));
1076         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2=0x%08X\n",
1077                 RREG32(R_008014_GRBM_STATUS2));
1078         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS=0x%08X\n",
1079                 RREG32(R_000E50_SRBM_STATUS));
1080         rv515_mc_resume(rdev, &save);
1081         return 0;
1082 }
1083
1084 bool r600_gpu_is_lockup(struct radeon_device *rdev)
1085 {
1086         u32 srbm_status;
1087         u32 grbm_status;
1088         u32 grbm_status2;
1089         int r;
1090
1091         srbm_status = RREG32(R_000E50_SRBM_STATUS);
1092         grbm_status = RREG32(R_008010_GRBM_STATUS);
1093         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
1094         if (!G_008010_GUI_ACTIVE(grbm_status)) {
1095                 r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
1096                 return false;
1097         }
1098         /* force CP activities */
1099         r = radeon_ring_lock(rdev, 2);
1100         if (!r) {
1101                 /* PACKET2 NOP */
1102                 radeon_ring_write(rdev, 0x80000000);
1103                 radeon_ring_write(rdev, 0x80000000);
1104                 radeon_ring_unlock_commit(rdev);
1105         }
1106         rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
1107         return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
1108 }
1109
1110 int r600_asic_reset(struct radeon_device *rdev)
1111 {
1112         return r600_gpu_soft_reset(rdev);
1113 }
1114
1115 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
1116                                              u32 num_backends,
1117                                              u32 backend_disable_mask)
1118 {
1119         u32 backend_map = 0;
1120         u32 enabled_backends_mask;
1121         u32 enabled_backends_count;
1122         u32 cur_pipe;
1123         u32 swizzle_pipe[R6XX_MAX_PIPES];
1124         u32 cur_backend;
1125         u32 i;
1126
1127         if (num_tile_pipes > R6XX_MAX_PIPES)
1128                 num_tile_pipes = R6XX_MAX_PIPES;
1129         if (num_tile_pipes < 1)
1130                 num_tile_pipes = 1;
1131         if (num_backends > R6XX_MAX_BACKENDS)
1132                 num_backends = R6XX_MAX_BACKENDS;
1133         if (num_backends < 1)
1134                 num_backends = 1;
1135
1136         enabled_backends_mask = 0;
1137         enabled_backends_count = 0;
1138         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
1139                 if (((backend_disable_mask >> i) & 1) == 0) {
1140                         enabled_backends_mask |= (1 << i);
1141                         ++enabled_backends_count;
1142                 }
1143                 if (enabled_backends_count == num_backends)
1144                         break;
1145         }
1146
1147         if (enabled_backends_count == 0) {
1148                 enabled_backends_mask = 1;
1149                 enabled_backends_count = 1;
1150         }
1151
1152         if (enabled_backends_count != num_backends)
1153                 num_backends = enabled_backends_count;
1154
1155         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
1156         switch (num_tile_pipes) {
1157         case 1:
1158                 swizzle_pipe[0] = 0;
1159                 break;
1160         case 2:
1161                 swizzle_pipe[0] = 0;
1162                 swizzle_pipe[1] = 1;
1163                 break;
1164         case 3:
1165                 swizzle_pipe[0] = 0;
1166                 swizzle_pipe[1] = 1;
1167                 swizzle_pipe[2] = 2;
1168                 break;
1169         case 4:
1170                 swizzle_pipe[0] = 0;
1171                 swizzle_pipe[1] = 1;
1172                 swizzle_pipe[2] = 2;
1173                 swizzle_pipe[3] = 3;
1174                 break;
1175         case 5:
1176                 swizzle_pipe[0] = 0;
1177                 swizzle_pipe[1] = 1;
1178                 swizzle_pipe[2] = 2;
1179                 swizzle_pipe[3] = 3;
1180                 swizzle_pipe[4] = 4;
1181                 break;
1182         case 6:
1183                 swizzle_pipe[0] = 0;
1184                 swizzle_pipe[1] = 2;
1185                 swizzle_pipe[2] = 4;
1186                 swizzle_pipe[3] = 5;
1187                 swizzle_pipe[4] = 1;
1188                 swizzle_pipe[5] = 3;
1189                 break;
1190         case 7:
1191                 swizzle_pipe[0] = 0;
1192                 swizzle_pipe[1] = 2;
1193                 swizzle_pipe[2] = 4;
1194                 swizzle_pipe[3] = 6;
1195                 swizzle_pipe[4] = 1;
1196                 swizzle_pipe[5] = 3;
1197                 swizzle_pipe[6] = 5;
1198                 break;
1199         case 8:
1200                 swizzle_pipe[0] = 0;
1201                 swizzle_pipe[1] = 2;
1202                 swizzle_pipe[2] = 4;
1203                 swizzle_pipe[3] = 6;
1204                 swizzle_pipe[4] = 1;
1205                 swizzle_pipe[5] = 3;
1206                 swizzle_pipe[6] = 5;
1207                 swizzle_pipe[7] = 7;
1208                 break;
1209         }
1210
1211         cur_backend = 0;
1212         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1213                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1214                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
1215
1216                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
1217
1218                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
1219         }
1220
1221         return backend_map;
1222 }
1223
1224 int r600_count_pipe_bits(uint32_t val)
1225 {
1226         int i, ret = 0;
1227
1228         for (i = 0; i < 32; i++) {
1229                 ret += val & 1;
1230                 val >>= 1;
1231         }
1232         return ret;
1233 }
1234
1235 void r600_gpu_init(struct radeon_device *rdev)
1236 {
1237         u32 tiling_config;
1238         u32 ramcfg;
1239         u32 backend_map;
1240         u32 cc_rb_backend_disable;
1241         u32 cc_gc_shader_pipe_config;
1242         u32 tmp;
1243         int i, j;
1244         u32 sq_config;
1245         u32 sq_gpr_resource_mgmt_1 = 0;
1246         u32 sq_gpr_resource_mgmt_2 = 0;
1247         u32 sq_thread_resource_mgmt = 0;
1248         u32 sq_stack_resource_mgmt_1 = 0;
1249         u32 sq_stack_resource_mgmt_2 = 0;
1250
1251         /* FIXME: implement */
1252         switch (rdev->family) {
1253         case CHIP_R600:
1254                 rdev->config.r600.max_pipes = 4;
1255                 rdev->config.r600.max_tile_pipes = 8;
1256                 rdev->config.r600.max_simds = 4;
1257                 rdev->config.r600.max_backends = 4;
1258                 rdev->config.r600.max_gprs = 256;
1259                 rdev->config.r600.max_threads = 192;
1260                 rdev->config.r600.max_stack_entries = 256;
1261                 rdev->config.r600.max_hw_contexts = 8;
1262                 rdev->config.r600.max_gs_threads = 16;
1263                 rdev->config.r600.sx_max_export_size = 128;
1264                 rdev->config.r600.sx_max_export_pos_size = 16;
1265                 rdev->config.r600.sx_max_export_smx_size = 128;
1266                 rdev->config.r600.sq_num_cf_insts = 2;
1267                 break;
1268         case CHIP_RV630:
1269         case CHIP_RV635:
1270                 rdev->config.r600.max_pipes = 2;
1271                 rdev->config.r600.max_tile_pipes = 2;
1272                 rdev->config.r600.max_simds = 3;
1273                 rdev->config.r600.max_backends = 1;
1274                 rdev->config.r600.max_gprs = 128;
1275                 rdev->config.r600.max_threads = 192;
1276                 rdev->config.r600.max_stack_entries = 128;
1277                 rdev->config.r600.max_hw_contexts = 8;
1278                 rdev->config.r600.max_gs_threads = 4;
1279                 rdev->config.r600.sx_max_export_size = 128;
1280                 rdev->config.r600.sx_max_export_pos_size = 16;
1281                 rdev->config.r600.sx_max_export_smx_size = 128;
1282                 rdev->config.r600.sq_num_cf_insts = 2;
1283                 break;
1284         case CHIP_RV610:
1285         case CHIP_RV620:
1286         case CHIP_RS780:
1287         case CHIP_RS880:
1288                 rdev->config.r600.max_pipes = 1;
1289                 rdev->config.r600.max_tile_pipes = 1;
1290                 rdev->config.r600.max_simds = 2;
1291                 rdev->config.r600.max_backends = 1;
1292                 rdev->config.r600.max_gprs = 128;
1293                 rdev->config.r600.max_threads = 192;
1294                 rdev->config.r600.max_stack_entries = 128;
1295                 rdev->config.r600.max_hw_contexts = 4;
1296                 rdev->config.r600.max_gs_threads = 4;
1297                 rdev->config.r600.sx_max_export_size = 128;
1298                 rdev->config.r600.sx_max_export_pos_size = 16;
1299                 rdev->config.r600.sx_max_export_smx_size = 128;
1300                 rdev->config.r600.sq_num_cf_insts = 1;
1301                 break;
1302         case CHIP_RV670:
1303                 rdev->config.r600.max_pipes = 4;
1304                 rdev->config.r600.max_tile_pipes = 4;
1305                 rdev->config.r600.max_simds = 4;
1306                 rdev->config.r600.max_backends = 4;
1307                 rdev->config.r600.max_gprs = 192;
1308                 rdev->config.r600.max_threads = 192;
1309                 rdev->config.r600.max_stack_entries = 256;
1310                 rdev->config.r600.max_hw_contexts = 8;
1311                 rdev->config.r600.max_gs_threads = 16;
1312                 rdev->config.r600.sx_max_export_size = 128;
1313                 rdev->config.r600.sx_max_export_pos_size = 16;
1314                 rdev->config.r600.sx_max_export_smx_size = 128;
1315                 rdev->config.r600.sq_num_cf_insts = 2;
1316                 break;
1317         default:
1318                 break;
1319         }
1320
1321         /* Initialize HDP */
1322         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1323                 WREG32((0x2c14 + j), 0x00000000);
1324                 WREG32((0x2c18 + j), 0x00000000);
1325                 WREG32((0x2c1c + j), 0x00000000);
1326                 WREG32((0x2c20 + j), 0x00000000);
1327                 WREG32((0x2c24 + j), 0x00000000);
1328         }
1329
1330         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1331
1332         /* Setup tiling */
1333         tiling_config = 0;
1334         ramcfg = RREG32(RAMCFG);
1335         switch (rdev->config.r600.max_tile_pipes) {
1336         case 1:
1337                 tiling_config |= PIPE_TILING(0);
1338                 break;
1339         case 2:
1340                 tiling_config |= PIPE_TILING(1);
1341                 break;
1342         case 4:
1343                 tiling_config |= PIPE_TILING(2);
1344                 break;
1345         case 8:
1346                 tiling_config |= PIPE_TILING(3);
1347                 break;
1348         default:
1349                 break;
1350         }
1351         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1352         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1353         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1354         tiling_config |= GROUP_SIZE(0);
1355         rdev->config.r600.tiling_group_size = 256;
1356         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1357         if (tmp > 3) {
1358                 tiling_config |= ROW_TILING(3);
1359                 tiling_config |= SAMPLE_SPLIT(3);
1360         } else {
1361                 tiling_config |= ROW_TILING(tmp);
1362                 tiling_config |= SAMPLE_SPLIT(tmp);
1363         }
1364         tiling_config |= BANK_SWAPS(1);
1365
1366         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1367         cc_rb_backend_disable |=
1368                 BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
1369
1370         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1371         cc_gc_shader_pipe_config |=
1372                 INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
1373         cc_gc_shader_pipe_config |=
1374                 INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
1375
1376         backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
1377                                                         (R6XX_MAX_BACKENDS -
1378                                                          r600_count_pipe_bits((cc_rb_backend_disable &
1379                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
1380                                                         (cc_rb_backend_disable >> 16));
1381
1382         tiling_config |= BACKEND_MAP(backend_map);
1383         WREG32(GB_TILING_CONFIG, tiling_config);
1384         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1385         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1386
1387         /* Setup pipes */
1388         WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1389         WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1390         WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1391
1392         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1393         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1394         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1395
1396         /* Setup some CP states */
1397         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1398         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1399
1400         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1401                              SYNC_WALKER | SYNC_ALIGNER));
1402         /* Setup various GPU states */
1403         if (rdev->family == CHIP_RV670)
1404                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1405
1406         tmp = RREG32(SX_DEBUG_1);
1407         tmp |= SMX_EVENT_RELEASE;
1408         if ((rdev->family > CHIP_R600))
1409                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1410         WREG32(SX_DEBUG_1, tmp);
1411
1412         if (((rdev->family) == CHIP_R600) ||
1413             ((rdev->family) == CHIP_RV630) ||
1414             ((rdev->family) == CHIP_RV610) ||
1415             ((rdev->family) == CHIP_RV620) ||
1416             ((rdev->family) == CHIP_RS780) ||
1417             ((rdev->family) == CHIP_RS880)) {
1418                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1419         } else {
1420                 WREG32(DB_DEBUG, 0);
1421         }
1422         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1423                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1424
1425         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1426         WREG32(VGT_NUM_INSTANCES, 0);
1427
1428         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1429         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1430
1431         tmp = RREG32(SQ_MS_FIFO_SIZES);
1432         if (((rdev->family) == CHIP_RV610) ||
1433             ((rdev->family) == CHIP_RV620) ||
1434             ((rdev->family) == CHIP_RS780) ||
1435             ((rdev->family) == CHIP_RS880)) {
1436                 tmp = (CACHE_FIFO_SIZE(0xa) |
1437                        FETCH_FIFO_HIWATER(0xa) |
1438                        DONE_FIFO_HIWATER(0xe0) |
1439                        ALU_UPDATE_FIFO_HIWATER(0x8));
1440         } else if (((rdev->family) == CHIP_R600) ||
1441                    ((rdev->family) == CHIP_RV630)) {
1442                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1443                 tmp |= DONE_FIFO_HIWATER(0x4);
1444         }
1445         WREG32(SQ_MS_FIFO_SIZES, tmp);
1446
1447         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1448          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1449          */
1450         sq_config = RREG32(SQ_CONFIG);
1451         sq_config &= ~(PS_PRIO(3) |
1452                        VS_PRIO(3) |
1453                        GS_PRIO(3) |
1454                        ES_PRIO(3));
1455         sq_config |= (DX9_CONSTS |
1456                       VC_ENABLE |
1457                       PS_PRIO(0) |
1458                       VS_PRIO(1) |
1459                       GS_PRIO(2) |
1460                       ES_PRIO(3));
1461
1462         if ((rdev->family) == CHIP_R600) {
1463                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1464                                           NUM_VS_GPRS(124) |
1465                                           NUM_CLAUSE_TEMP_GPRS(4));
1466                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1467                                           NUM_ES_GPRS(0));
1468                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1469                                            NUM_VS_THREADS(48) |
1470                                            NUM_GS_THREADS(4) |
1471                                            NUM_ES_THREADS(4));
1472                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1473                                             NUM_VS_STACK_ENTRIES(128));
1474                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1475                                             NUM_ES_STACK_ENTRIES(0));
1476         } else if (((rdev->family) == CHIP_RV610) ||
1477                    ((rdev->family) == CHIP_RV620) ||
1478                    ((rdev->family) == CHIP_RS780) ||
1479                    ((rdev->family) == CHIP_RS880)) {
1480                 /* no vertex cache */
1481                 sq_config &= ~VC_ENABLE;
1482
1483                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1484                                           NUM_VS_GPRS(44) |
1485                                           NUM_CLAUSE_TEMP_GPRS(2));
1486                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1487                                           NUM_ES_GPRS(17));
1488                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1489                                            NUM_VS_THREADS(78) |
1490                                            NUM_GS_THREADS(4) |
1491                                            NUM_ES_THREADS(31));
1492                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1493                                             NUM_VS_STACK_ENTRIES(40));
1494                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1495                                             NUM_ES_STACK_ENTRIES(16));
1496         } else if (((rdev->family) == CHIP_RV630) ||
1497                    ((rdev->family) == CHIP_RV635)) {
1498                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1499                                           NUM_VS_GPRS(44) |
1500                                           NUM_CLAUSE_TEMP_GPRS(2));
1501                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1502                                           NUM_ES_GPRS(18));
1503                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1504                                            NUM_VS_THREADS(78) |
1505                                            NUM_GS_THREADS(4) |
1506                                            NUM_ES_THREADS(31));
1507                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1508                                             NUM_VS_STACK_ENTRIES(40));
1509                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1510                                             NUM_ES_STACK_ENTRIES(16));
1511         } else if ((rdev->family) == CHIP_RV670) {
1512                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1513                                           NUM_VS_GPRS(44) |
1514                                           NUM_CLAUSE_TEMP_GPRS(2));
1515                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1516                                           NUM_ES_GPRS(17));
1517                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1518                                            NUM_VS_THREADS(78) |
1519                                            NUM_GS_THREADS(4) |
1520                                            NUM_ES_THREADS(31));
1521                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1522                                             NUM_VS_STACK_ENTRIES(64));
1523                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1524                                             NUM_ES_STACK_ENTRIES(64));
1525         }
1526
1527         WREG32(SQ_CONFIG, sq_config);
1528         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1529         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1530         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1531         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1532         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1533
1534         if (((rdev->family) == CHIP_RV610) ||
1535             ((rdev->family) == CHIP_RV620) ||
1536             ((rdev->family) == CHIP_RS780) ||
1537             ((rdev->family) == CHIP_RS880)) {
1538                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1539         } else {
1540                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1541         }
1542
1543         /* More default values. 2D/3D driver should adjust as needed */
1544         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1545                                          S1_X(0x4) | S1_Y(0xc)));
1546         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1547                                          S1_X(0x2) | S1_Y(0x2) |
1548                                          S2_X(0xa) | S2_Y(0x6) |
1549                                          S3_X(0x6) | S3_Y(0xa)));
1550         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1551                                              S1_X(0x4) | S1_Y(0xc) |
1552                                              S2_X(0x1) | S2_Y(0x6) |
1553                                              S3_X(0xa) | S3_Y(0xe)));
1554         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1555                                              S5_X(0x0) | S5_Y(0x0) |
1556                                              S6_X(0xb) | S6_Y(0x4) |
1557                                              S7_X(0x7) | S7_Y(0x8)));
1558
1559         WREG32(VGT_STRMOUT_EN, 0);
1560         tmp = rdev->config.r600.max_pipes * 16;
1561         switch (rdev->family) {
1562         case CHIP_RV610:
1563         case CHIP_RV620:
1564         case CHIP_RS780:
1565         case CHIP_RS880:
1566                 tmp += 32;
1567                 break;
1568         case CHIP_RV670:
1569                 tmp += 128;
1570                 break;
1571         default:
1572                 break;
1573         }
1574         if (tmp > 256) {
1575                 tmp = 256;
1576         }
1577         WREG32(VGT_ES_PER_GS, 128);
1578         WREG32(VGT_GS_PER_ES, tmp);
1579         WREG32(VGT_GS_PER_VS, 2);
1580         WREG32(VGT_GS_VERTEX_REUSE, 16);
1581
1582         /* more default values. 2D/3D driver should adjust as needed */
1583         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1584         WREG32(VGT_STRMOUT_EN, 0);
1585         WREG32(SX_MISC, 0);
1586         WREG32(PA_SC_MODE_CNTL, 0);
1587         WREG32(PA_SC_AA_CONFIG, 0);
1588         WREG32(PA_SC_LINE_STIPPLE, 0);
1589         WREG32(SPI_INPUT_Z, 0);
1590         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1591         WREG32(CB_COLOR7_FRAG, 0);
1592
1593         /* Clear render buffer base addresses */
1594         WREG32(CB_COLOR0_BASE, 0);
1595         WREG32(CB_COLOR1_BASE, 0);
1596         WREG32(CB_COLOR2_BASE, 0);
1597         WREG32(CB_COLOR3_BASE, 0);
1598         WREG32(CB_COLOR4_BASE, 0);
1599         WREG32(CB_COLOR5_BASE, 0);
1600         WREG32(CB_COLOR6_BASE, 0);
1601         WREG32(CB_COLOR7_BASE, 0);
1602         WREG32(CB_COLOR7_FRAG, 0);
1603
1604         switch (rdev->family) {
1605         case CHIP_RV610:
1606         case CHIP_RV620:
1607         case CHIP_RS780:
1608         case CHIP_RS880:
1609                 tmp = TC_L2_SIZE(8);
1610                 break;
1611         case CHIP_RV630:
1612         case CHIP_RV635:
1613                 tmp = TC_L2_SIZE(4);
1614                 break;
1615         case CHIP_R600:
1616                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
1617                 break;
1618         default:
1619                 tmp = TC_L2_SIZE(0);
1620                 break;
1621         }
1622         WREG32(TC_CNTL, tmp);
1623
1624         tmp = RREG32(HDP_HOST_PATH_CNTL);
1625         WREG32(HDP_HOST_PATH_CNTL, tmp);
1626
1627         tmp = RREG32(ARB_POP);
1628         tmp |= ENABLE_TC128;
1629         WREG32(ARB_POP, tmp);
1630
1631         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1632         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1633                                NUM_CLIP_SEQ(3)));
1634         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
1635 }
1636
1637
1638 /*
1639  * Indirect registers accessor
1640  */
1641 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1642 {
1643         u32 r;
1644
1645         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1646         (void)RREG32(PCIE_PORT_INDEX);
1647         r = RREG32(PCIE_PORT_DATA);
1648         return r;
1649 }
1650
1651 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1652 {
1653         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1654         (void)RREG32(PCIE_PORT_INDEX);
1655         WREG32(PCIE_PORT_DATA, (v));
1656         (void)RREG32(PCIE_PORT_DATA);
1657 }
1658
1659 /*
1660  * CP & Ring
1661  */
1662 void r600_cp_stop(struct radeon_device *rdev)
1663 {
1664         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1665 }
1666
1667 int r600_init_microcode(struct radeon_device *rdev)
1668 {
1669         struct platform_device *pdev;
1670         const char *chip_name;
1671         const char *rlc_chip_name;
1672         size_t pfp_req_size, me_req_size, rlc_req_size;
1673         char fw_name[30];
1674         int err;
1675
1676         DRM_DEBUG("\n");
1677
1678         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1679         err = IS_ERR(pdev);
1680         if (err) {
1681                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1682                 return -EINVAL;
1683         }
1684
1685         switch (rdev->family) {
1686         case CHIP_R600:
1687                 chip_name = "R600";
1688                 rlc_chip_name = "R600";
1689                 break;
1690         case CHIP_RV610:
1691                 chip_name = "RV610";
1692                 rlc_chip_name = "R600";
1693                 break;
1694         case CHIP_RV630:
1695                 chip_name = "RV630";
1696                 rlc_chip_name = "R600";
1697                 break;
1698         case CHIP_RV620:
1699                 chip_name = "RV620";
1700                 rlc_chip_name = "R600";
1701                 break;
1702         case CHIP_RV635:
1703                 chip_name = "RV635";
1704                 rlc_chip_name = "R600";
1705                 break;
1706         case CHIP_RV670:
1707                 chip_name = "RV670";
1708                 rlc_chip_name = "R600";
1709                 break;
1710         case CHIP_RS780:
1711         case CHIP_RS880:
1712                 chip_name = "RS780";
1713                 rlc_chip_name = "R600";
1714                 break;
1715         case CHIP_RV770:
1716                 chip_name = "RV770";
1717                 rlc_chip_name = "R700";
1718                 break;
1719         case CHIP_RV730:
1720         case CHIP_RV740:
1721                 chip_name = "RV730";
1722                 rlc_chip_name = "R700";
1723                 break;
1724         case CHIP_RV710:
1725                 chip_name = "RV710";
1726                 rlc_chip_name = "R700";
1727                 break;
1728         case CHIP_CEDAR:
1729                 chip_name = "CEDAR";
1730                 rlc_chip_name = "CEDAR";
1731                 break;
1732         case CHIP_REDWOOD:
1733                 chip_name = "REDWOOD";
1734                 rlc_chip_name = "REDWOOD";
1735                 break;
1736         case CHIP_JUNIPER:
1737                 chip_name = "JUNIPER";
1738                 rlc_chip_name = "JUNIPER";
1739                 break;
1740         case CHIP_CYPRESS:
1741         case CHIP_HEMLOCK:
1742                 chip_name = "CYPRESS";
1743                 rlc_chip_name = "CYPRESS";
1744                 break;
1745         default: BUG();
1746         }
1747
1748         if (rdev->family >= CHIP_CEDAR) {
1749                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
1750                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
1751                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
1752         } else if (rdev->family >= CHIP_RV770) {
1753                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
1754                 me_req_size = R700_PM4_UCODE_SIZE * 4;
1755                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
1756         } else {
1757                 pfp_req_size = PFP_UCODE_SIZE * 4;
1758                 me_req_size = PM4_UCODE_SIZE * 12;
1759                 rlc_req_size = RLC_UCODE_SIZE * 4;
1760         }
1761
1762         DRM_INFO("Loading %s Microcode\n", chip_name);
1763
1764         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1765         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1766         if (err)
1767                 goto out;
1768         if (rdev->pfp_fw->size != pfp_req_size) {
1769                 printk(KERN_ERR
1770                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1771                        rdev->pfp_fw->size, fw_name);
1772                 err = -EINVAL;
1773                 goto out;
1774         }
1775
1776         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1777         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1778         if (err)
1779                 goto out;
1780         if (rdev->me_fw->size != me_req_size) {
1781                 printk(KERN_ERR
1782                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
1783                        rdev->me_fw->size, fw_name);
1784                 err = -EINVAL;
1785         }
1786
1787         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1788         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1789         if (err)
1790                 goto out;
1791         if (rdev->rlc_fw->size != rlc_req_size) {
1792                 printk(KERN_ERR
1793                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
1794                        rdev->rlc_fw->size, fw_name);
1795                 err = -EINVAL;
1796         }
1797
1798 out:
1799         platform_device_unregister(pdev);
1800
1801         if (err) {
1802                 if (err != -EINVAL)
1803                         printk(KERN_ERR
1804                                "r600_cp: Failed to load firmware \"%s\"\n",
1805                                fw_name);
1806                 release_firmware(rdev->pfp_fw);
1807                 rdev->pfp_fw = NULL;
1808                 release_firmware(rdev->me_fw);
1809                 rdev->me_fw = NULL;
1810                 release_firmware(rdev->rlc_fw);
1811                 rdev->rlc_fw = NULL;
1812         }
1813         return err;
1814 }
1815
1816 static int r600_cp_load_microcode(struct radeon_device *rdev)
1817 {
1818         const __be32 *fw_data;
1819         int i;
1820
1821         if (!rdev->me_fw || !rdev->pfp_fw)
1822                 return -EINVAL;
1823
1824         r600_cp_stop(rdev);
1825
1826         WREG32(CP_RB_CNTL, RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1827
1828         /* Reset cp */
1829         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1830         RREG32(GRBM_SOFT_RESET);
1831         mdelay(15);
1832         WREG32(GRBM_SOFT_RESET, 0);
1833
1834         WREG32(CP_ME_RAM_WADDR, 0);
1835
1836         fw_data = (const __be32 *)rdev->me_fw->data;
1837         WREG32(CP_ME_RAM_WADDR, 0);
1838         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
1839                 WREG32(CP_ME_RAM_DATA,
1840                        be32_to_cpup(fw_data++));
1841
1842         fw_data = (const __be32 *)rdev->pfp_fw->data;
1843         WREG32(CP_PFP_UCODE_ADDR, 0);
1844         for (i = 0; i < PFP_UCODE_SIZE; i++)
1845                 WREG32(CP_PFP_UCODE_DATA,
1846                        be32_to_cpup(fw_data++));
1847
1848         WREG32(CP_PFP_UCODE_ADDR, 0);
1849         WREG32(CP_ME_RAM_WADDR, 0);
1850         WREG32(CP_ME_RAM_RADDR, 0);
1851         return 0;
1852 }
1853
1854 int r600_cp_start(struct radeon_device *rdev)
1855 {
1856         int r;
1857         uint32_t cp_me;
1858
1859         r = radeon_ring_lock(rdev, 7);
1860         if (r) {
1861                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1862                 return r;
1863         }
1864         radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1865         radeon_ring_write(rdev, 0x1);
1866         if (rdev->family >= CHIP_CEDAR) {
1867                 radeon_ring_write(rdev, 0x0);
1868                 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1869         } else if (rdev->family >= CHIP_RV770) {
1870                 radeon_ring_write(rdev, 0x0);
1871                 radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
1872         } else {
1873                 radeon_ring_write(rdev, 0x3);
1874                 radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
1875         }
1876         radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1877         radeon_ring_write(rdev, 0);
1878         radeon_ring_write(rdev, 0);
1879         radeon_ring_unlock_commit(rdev);
1880
1881         cp_me = 0xff;
1882         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
1883         return 0;
1884 }
1885
1886 int r600_cp_resume(struct radeon_device *rdev)
1887 {
1888         u32 tmp;
1889         u32 rb_bufsz;
1890         int r;
1891
1892         /* Reset cp */
1893         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1894         RREG32(GRBM_SOFT_RESET);
1895         mdelay(15);
1896         WREG32(GRBM_SOFT_RESET, 0);
1897
1898         /* Set ring buffer size */
1899         rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1900         tmp = RB_NO_UPDATE | (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1901 #ifdef __BIG_ENDIAN
1902         tmp |= BUF_SWAP_32BIT;
1903 #endif
1904         WREG32(CP_RB_CNTL, tmp);
1905         WREG32(CP_SEM_WAIT_TIMER, 0x4);
1906
1907         /* Set the write pointer delay */
1908         WREG32(CP_RB_WPTR_DELAY, 0);
1909
1910         /* Initialize the ring buffer's read and write pointers */
1911         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1912         WREG32(CP_RB_RPTR_WR, 0);
1913         WREG32(CP_RB_WPTR, 0);
1914         WREG32(CP_RB_RPTR_ADDR, rdev->cp.gpu_addr & 0xFFFFFFFF);
1915         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->cp.gpu_addr));
1916         mdelay(1);
1917         WREG32(CP_RB_CNTL, tmp);
1918
1919         WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1920         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1921
1922         rdev->cp.rptr = RREG32(CP_RB_RPTR);
1923         rdev->cp.wptr = RREG32(CP_RB_WPTR);
1924
1925         r600_cp_start(rdev);
1926         rdev->cp.ready = true;
1927         r = radeon_ring_test(rdev);
1928         if (r) {
1929                 rdev->cp.ready = false;
1930                 return r;
1931         }
1932         return 0;
1933 }
1934
1935 void r600_cp_commit(struct radeon_device *rdev)
1936 {
1937         WREG32(CP_RB_WPTR, rdev->cp.wptr);
1938         (void)RREG32(CP_RB_WPTR);
1939 }
1940
1941 void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
1942 {
1943         u32 rb_bufsz;
1944
1945         /* Align ring size */
1946         rb_bufsz = drm_order(ring_size / 8);
1947         ring_size = (1 << (rb_bufsz + 1)) * 4;
1948         rdev->cp.ring_size = ring_size;
1949         rdev->cp.align_mask = 16 - 1;
1950 }
1951
1952 void r600_cp_fini(struct radeon_device *rdev)
1953 {
1954         r600_cp_stop(rdev);
1955         radeon_ring_fini(rdev);
1956 }
1957
1958
1959 /*
1960  * GPU scratch registers helpers function.
1961  */
1962 void r600_scratch_init(struct radeon_device *rdev)
1963 {
1964         int i;
1965
1966         rdev->scratch.num_reg = 7;
1967         for (i = 0; i < rdev->scratch.num_reg; i++) {
1968                 rdev->scratch.free[i] = true;
1969                 rdev->scratch.reg[i] = SCRATCH_REG0 + (i * 4);
1970         }
1971 }
1972
1973 int r600_ring_test(struct radeon_device *rdev)
1974 {
1975         uint32_t scratch;
1976         uint32_t tmp = 0;
1977         unsigned i;
1978         int r;
1979
1980         r = radeon_scratch_get(rdev, &scratch);
1981         if (r) {
1982                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
1983                 return r;
1984         }
1985         WREG32(scratch, 0xCAFEDEAD);
1986         r = radeon_ring_lock(rdev, 3);
1987         if (r) {
1988                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1989                 radeon_scratch_free(rdev, scratch);
1990                 return r;
1991         }
1992         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1993         radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
1994         radeon_ring_write(rdev, 0xDEADBEEF);
1995         radeon_ring_unlock_commit(rdev);
1996         for (i = 0; i < rdev->usec_timeout; i++) {
1997                 tmp = RREG32(scratch);
1998                 if (tmp == 0xDEADBEEF)
1999                         break;
2000                 DRM_UDELAY(1);
2001         }
2002         if (i < rdev->usec_timeout) {
2003                 DRM_INFO("ring test succeeded in %d usecs\n", i);
2004         } else {
2005                 DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
2006                           scratch, tmp);
2007                 r = -EINVAL;
2008         }
2009         radeon_scratch_free(rdev, scratch);
2010         return r;
2011 }
2012
2013 void r600_wb_disable(struct radeon_device *rdev)
2014 {
2015         int r;
2016
2017         WREG32(SCRATCH_UMSK, 0);
2018         if (rdev->wb.wb_obj) {
2019                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
2020                 if (unlikely(r != 0))
2021                         return;
2022                 radeon_bo_kunmap(rdev->wb.wb_obj);
2023                 radeon_bo_unpin(rdev->wb.wb_obj);
2024                 radeon_bo_unreserve(rdev->wb.wb_obj);
2025         }
2026 }
2027
2028 void r600_wb_fini(struct radeon_device *rdev)
2029 {
2030         r600_wb_disable(rdev);
2031         if (rdev->wb.wb_obj) {
2032                 radeon_bo_unref(&rdev->wb.wb_obj);
2033                 rdev->wb.wb = NULL;
2034                 rdev->wb.wb_obj = NULL;
2035         }
2036 }
2037
2038 int r600_wb_enable(struct radeon_device *rdev)
2039 {
2040         int r;
2041
2042         if (rdev->wb.wb_obj == NULL) {
2043                 r = radeon_bo_create(rdev, NULL, RADEON_GPU_PAGE_SIZE, true,
2044                                 RADEON_GEM_DOMAIN_GTT, &rdev->wb.wb_obj);
2045                 if (r) {
2046                         dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
2047                         return r;
2048                 }
2049                 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
2050                 if (unlikely(r != 0)) {
2051                         r600_wb_fini(rdev);
2052                         return r;
2053                 }
2054                 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
2055                                 &rdev->wb.gpu_addr);
2056                 if (r) {
2057                         radeon_bo_unreserve(rdev->wb.wb_obj);
2058                         dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
2059                         r600_wb_fini(rdev);
2060                         return r;
2061                 }
2062                 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
2063                 radeon_bo_unreserve(rdev->wb.wb_obj);
2064                 if (r) {
2065                         dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
2066                         r600_wb_fini(rdev);
2067                         return r;
2068                 }
2069         }
2070         WREG32(SCRATCH_ADDR, (rdev->wb.gpu_addr >> 8) & 0xFFFFFFFF);
2071         WREG32(CP_RB_RPTR_ADDR, (rdev->wb.gpu_addr + 1024) & 0xFFFFFFFC);
2072         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + 1024) & 0xFF);
2073         WREG32(SCRATCH_UMSK, 0xff);
2074         return 0;
2075 }
2076
2077 void r600_fence_ring_emit(struct radeon_device *rdev,
2078                           struct radeon_fence *fence)
2079 {
2080         /* Also consider EVENT_WRITE_EOP.  it handles the interrupts + timestamps + events */
2081
2082         radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
2083         radeon_ring_write(rdev, CACHE_FLUSH_AND_INV_EVENT);
2084         /* wait for 3D idle clean */
2085         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2086         radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2087         radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2088         /* Emit fence sequence & fire IRQ */
2089         radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2090         radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2091         radeon_ring_write(rdev, fence->seq);
2092         /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2093         radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
2094         radeon_ring_write(rdev, RB_INT_STAT);
2095 }
2096
2097 int r600_copy_blit(struct radeon_device *rdev,
2098                    uint64_t src_offset, uint64_t dst_offset,
2099                    unsigned num_pages, struct radeon_fence *fence)
2100 {
2101         int r;
2102
2103         mutex_lock(&rdev->r600_blit.mutex);
2104         rdev->r600_blit.vb_ib = NULL;
2105         r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
2106         if (r) {
2107                 if (rdev->r600_blit.vb_ib)
2108                         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
2109                 mutex_unlock(&rdev->r600_blit.mutex);
2110                 return r;
2111         }
2112         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
2113         r600_blit_done_copy(rdev, fence);
2114         mutex_unlock(&rdev->r600_blit.mutex);
2115         return 0;
2116 }
2117
2118 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
2119                          uint32_t tiling_flags, uint32_t pitch,
2120                          uint32_t offset, uint32_t obj_size)
2121 {
2122         /* FIXME: implement */
2123         return 0;
2124 }
2125
2126 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
2127 {
2128         /* FIXME: implement */
2129 }
2130
2131
2132 bool r600_card_posted(struct radeon_device *rdev)
2133 {
2134         uint32_t reg;
2135
2136         /* first check CRTCs */
2137         reg = RREG32(D1CRTC_CONTROL) |
2138                 RREG32(D2CRTC_CONTROL);
2139         if (reg & CRTC_EN)
2140                 return true;
2141
2142         /* then check MEM_SIZE, in case the crtcs are off */
2143         if (RREG32(CONFIG_MEMSIZE))
2144                 return true;
2145
2146         return false;
2147 }
2148
2149 int r600_startup(struct radeon_device *rdev)
2150 {
2151         int r;
2152
2153         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2154                 r = r600_init_microcode(rdev);
2155                 if (r) {
2156                         DRM_ERROR("Failed to load firmware!\n");
2157                         return r;
2158                 }
2159         }
2160
2161         r600_mc_program(rdev);
2162         if (rdev->flags & RADEON_IS_AGP) {
2163                 r600_agp_enable(rdev);
2164         } else {
2165                 r = r600_pcie_gart_enable(rdev);
2166                 if (r)
2167                         return r;
2168         }
2169         r600_gpu_init(rdev);
2170         r = r600_blit_init(rdev);
2171         if (r) {
2172                 r600_blit_fini(rdev);
2173                 rdev->asic->copy = NULL;
2174                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2175         }
2176         /* pin copy shader into vram */
2177         if (rdev->r600_blit.shader_obj) {
2178                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
2179                 if (unlikely(r != 0))
2180                         return r;
2181                 r = radeon_bo_pin(rdev->r600_blit.shader_obj, RADEON_GEM_DOMAIN_VRAM,
2182                                 &rdev->r600_blit.shader_gpu_addr);
2183                 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
2184                 if (r) {
2185                         dev_err(rdev->dev, "(%d) pin blit object failed\n", r);
2186                         return r;
2187                 }
2188         }
2189         /* Enable IRQ */
2190         r = r600_irq_init(rdev);
2191         if (r) {
2192                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2193                 radeon_irq_kms_fini(rdev);
2194                 return r;
2195         }
2196         r600_irq_set(rdev);
2197
2198         r = radeon_ring_init(rdev, rdev->cp.ring_size);
2199         if (r)
2200                 return r;
2201         r = r600_cp_load_microcode(rdev);
2202         if (r)
2203                 return r;
2204         r = r600_cp_resume(rdev);
2205         if (r)
2206                 return r;
2207         /* write back buffer are not vital so don't worry about failure */
2208         r600_wb_enable(rdev);
2209         return 0;
2210 }
2211
2212 void r600_vga_set_state(struct radeon_device *rdev, bool state)
2213 {
2214         uint32_t temp;
2215
2216         temp = RREG32(CONFIG_CNTL);
2217         if (state == false) {
2218                 temp &= ~(1<<0);
2219                 temp |= (1<<1);
2220         } else {
2221                 temp &= ~(1<<1);
2222         }
2223         WREG32(CONFIG_CNTL, temp);
2224 }
2225
2226 int r600_resume(struct radeon_device *rdev)
2227 {
2228         int r;
2229
2230         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
2231          * posting will perform necessary task to bring back GPU into good
2232          * shape.
2233          */
2234         /* post card */
2235         atom_asic_init(rdev->mode_info.atom_context);
2236         /* Initialize clocks */
2237         r = radeon_clocks_init(rdev);
2238         if (r) {
2239                 return r;
2240         }
2241
2242         r = r600_startup(rdev);
2243         if (r) {
2244                 DRM_ERROR("r600 startup failed on resume\n");
2245                 return r;
2246         }
2247
2248         r = r600_ib_test(rdev);
2249         if (r) {
2250                 DRM_ERROR("radeon: failled testing IB (%d).\n", r);
2251                 return r;
2252         }
2253
2254         r = r600_audio_init(rdev);
2255         if (r) {
2256                 DRM_ERROR("radeon: audio resume failed\n");
2257                 return r;
2258         }
2259
2260         return r;
2261 }
2262
2263 int r600_suspend(struct radeon_device *rdev)
2264 {
2265         int r;
2266
2267         r600_audio_fini(rdev);
2268         /* FIXME: we should wait for ring to be empty */
2269         r600_cp_stop(rdev);
2270         rdev->cp.ready = false;
2271         r600_irq_suspend(rdev);
2272         r600_wb_disable(rdev);
2273         r600_pcie_gart_disable(rdev);
2274         /* unpin shaders bo */
2275         if (rdev->r600_blit.shader_obj) {
2276                 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
2277                 if (!r) {
2278                         radeon_bo_unpin(rdev->r600_blit.shader_obj);
2279                         radeon_bo_unreserve(rdev->r600_blit.shader_obj);
2280                 }
2281         }
2282         return 0;
2283 }
2284
2285 /* Plan is to move initialization in that function and use
2286  * helper function so that radeon_device_init pretty much
2287  * do nothing more than calling asic specific function. This
2288  * should also allow to remove a bunch of callback function
2289  * like vram_info.
2290  */
2291 int r600_init(struct radeon_device *rdev)
2292 {
2293         int r;
2294
2295         r = radeon_dummy_page_init(rdev);
2296         if (r)
2297                 return r;
2298         if (r600_debugfs_mc_info_init(rdev)) {
2299                 DRM_ERROR("Failed to register debugfs file for mc !\n");
2300         }
2301         /* This don't do much */
2302         r = radeon_gem_init(rdev);
2303         if (r)
2304                 return r;
2305         /* Read BIOS */
2306         if (!radeon_get_bios(rdev)) {
2307                 if (ASIC_IS_AVIVO(rdev))
2308                         return -EINVAL;
2309         }
2310         /* Must be an ATOMBIOS */
2311         if (!rdev->is_atom_bios) {
2312                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
2313                 return -EINVAL;
2314         }
2315         r = radeon_atombios_init(rdev);
2316         if (r)
2317                 return r;
2318         /* Post card if necessary */
2319         if (!r600_card_posted(rdev)) {
2320                 if (!rdev->bios) {
2321                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2322                         return -EINVAL;
2323                 }
2324                 DRM_INFO("GPU not posted. posting now...\n");
2325                 atom_asic_init(rdev->mode_info.atom_context);
2326         }
2327         /* Initialize scratch registers */
2328         r600_scratch_init(rdev);
2329         /* Initialize surface registers */
2330         radeon_surface_init(rdev);
2331         /* Initialize clocks */
2332         radeon_get_clock_info(rdev->ddev);
2333         r = radeon_clocks_init(rdev);
2334         if (r)
2335                 return r;
2336         /* Initialize power management */
2337         radeon_pm_init(rdev);
2338         /* Fence driver */
2339         r = radeon_fence_driver_init(rdev);
2340         if (r)
2341                 return r;
2342         if (rdev->flags & RADEON_IS_AGP) {
2343                 r = radeon_agp_init(rdev);
2344                 if (r)
2345                         radeon_agp_disable(rdev);
2346         }
2347         r = r600_mc_init(rdev);
2348         if (r)
2349                 return r;
2350         /* Memory manager */
2351         r = radeon_bo_init(rdev);
2352         if (r)
2353                 return r;
2354
2355         r = radeon_irq_kms_init(rdev);
2356         if (r)
2357                 return r;
2358
2359         rdev->cp.ring_obj = NULL;
2360         r600_ring_init(rdev, 1024 * 1024);
2361
2362         rdev->ih.ring_obj = NULL;
2363         r600_ih_ring_init(rdev, 64 * 1024);
2364
2365         r = r600_pcie_gart_init(rdev);
2366         if (r)
2367                 return r;
2368
2369         rdev->accel_working = true;
2370         r = r600_startup(rdev);
2371         if (r) {
2372                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2373                 r600_cp_fini(rdev);
2374                 r600_wb_fini(rdev);
2375                 r600_irq_fini(rdev);
2376                 radeon_irq_kms_fini(rdev);
2377                 r600_pcie_gart_fini(rdev);
2378                 rdev->accel_working = false;
2379         }
2380         if (rdev->accel_working) {
2381                 r = radeon_ib_pool_init(rdev);
2382                 if (r) {
2383                         dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2384                         rdev->accel_working = false;
2385                 } else {
2386                         r = r600_ib_test(rdev);
2387                         if (r) {
2388                                 dev_err(rdev->dev, "IB test failed (%d).\n", r);
2389                                 rdev->accel_working = false;
2390                         }
2391                 }
2392         }
2393
2394         r = r600_audio_init(rdev);
2395         if (r)
2396                 return r; /* TODO error handling */
2397         return 0;
2398 }
2399
2400 void r600_fini(struct radeon_device *rdev)
2401 {
2402         radeon_pm_fini(rdev);
2403         r600_audio_fini(rdev);
2404         r600_blit_fini(rdev);
2405         r600_cp_fini(rdev);
2406         r600_wb_fini(rdev);
2407         r600_irq_fini(rdev);
2408         radeon_irq_kms_fini(rdev);
2409         r600_pcie_gart_fini(rdev);
2410         radeon_agp_fini(rdev);
2411         radeon_gem_fini(rdev);
2412         radeon_fence_driver_fini(rdev);
2413         radeon_clocks_fini(rdev);
2414         radeon_bo_fini(rdev);
2415         radeon_atombios_fini(rdev);
2416         kfree(rdev->bios);
2417         rdev->bios = NULL;
2418         radeon_dummy_page_fini(rdev);
2419 }
2420
2421
2422 /*
2423  * CS stuff
2424  */
2425 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2426 {
2427         /* FIXME: implement */
2428         radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2429         radeon_ring_write(rdev, ib->gpu_addr & 0xFFFFFFFC);
2430         radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
2431         radeon_ring_write(rdev, ib->length_dw);
2432 }
2433
2434 int r600_ib_test(struct radeon_device *rdev)
2435 {
2436         struct radeon_ib *ib;
2437         uint32_t scratch;
2438         uint32_t tmp = 0;
2439         unsigned i;
2440         int r;
2441
2442         r = radeon_scratch_get(rdev, &scratch);
2443         if (r) {
2444                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
2445                 return r;
2446         }
2447         WREG32(scratch, 0xCAFEDEAD);
2448         r = radeon_ib_get(rdev, &ib);
2449         if (r) {
2450                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2451                 return r;
2452         }
2453         ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
2454         ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2455         ib->ptr[2] = 0xDEADBEEF;
2456         ib->ptr[3] = PACKET2(0);
2457         ib->ptr[4] = PACKET2(0);
2458         ib->ptr[5] = PACKET2(0);
2459         ib->ptr[6] = PACKET2(0);
2460         ib->ptr[7] = PACKET2(0);
2461         ib->ptr[8] = PACKET2(0);
2462         ib->ptr[9] = PACKET2(0);
2463         ib->ptr[10] = PACKET2(0);
2464         ib->ptr[11] = PACKET2(0);
2465         ib->ptr[12] = PACKET2(0);
2466         ib->ptr[13] = PACKET2(0);
2467         ib->ptr[14] = PACKET2(0);
2468         ib->ptr[15] = PACKET2(0);
2469         ib->length_dw = 16;
2470         r = radeon_ib_schedule(rdev, ib);
2471         if (r) {
2472                 radeon_scratch_free(rdev, scratch);
2473                 radeon_ib_free(rdev, &ib);
2474                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2475                 return r;
2476         }
2477         r = radeon_fence_wait(ib->fence, false);
2478         if (r) {
2479                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2480                 return r;
2481         }
2482         for (i = 0; i < rdev->usec_timeout; i++) {
2483                 tmp = RREG32(scratch);
2484                 if (tmp == 0xDEADBEEF)
2485                         break;
2486                 DRM_UDELAY(1);
2487         }
2488         if (i < rdev->usec_timeout) {
2489                 DRM_INFO("ib test succeeded in %u usecs\n", i);
2490         } else {
2491                 DRM_ERROR("radeon: ib test failed (sracth(0x%04X)=0x%08X)\n",
2492                           scratch, tmp);
2493                 r = -EINVAL;
2494         }
2495         radeon_scratch_free(rdev, scratch);
2496         radeon_ib_free(rdev, &ib);
2497         return r;
2498 }
2499
2500 /*
2501  * Interrupts
2502  *
2503  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
2504  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
2505  * writing to the ring and the GPU consuming, the GPU writes to the ring
2506  * and host consumes.  As the host irq handler processes interrupts, it
2507  * increments the rptr.  When the rptr catches up with the wptr, all the
2508  * current interrupts have been processed.
2509  */
2510
2511 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
2512 {
2513         u32 rb_bufsz;
2514
2515         /* Align ring size */
2516         rb_bufsz = drm_order(ring_size / 4);
2517         ring_size = (1 << rb_bufsz) * 4;
2518         rdev->ih.ring_size = ring_size;
2519         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
2520         rdev->ih.rptr = 0;
2521 }
2522
2523 static int r600_ih_ring_alloc(struct radeon_device *rdev)
2524 {
2525         int r;
2526
2527         /* Allocate ring buffer */
2528         if (rdev->ih.ring_obj == NULL) {
2529                 r = radeon_bo_create(rdev, NULL, rdev->ih.ring_size,
2530                                      true,
2531                                      RADEON_GEM_DOMAIN_GTT,
2532                                      &rdev->ih.ring_obj);
2533                 if (r) {
2534                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
2535                         return r;
2536                 }
2537                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2538                 if (unlikely(r != 0))
2539                         return r;
2540                 r = radeon_bo_pin(rdev->ih.ring_obj,
2541                                   RADEON_GEM_DOMAIN_GTT,
2542                                   &rdev->ih.gpu_addr);
2543                 if (r) {
2544                         radeon_bo_unreserve(rdev->ih.ring_obj);
2545                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
2546                         return r;
2547                 }
2548                 r = radeon_bo_kmap(rdev->ih.ring_obj,
2549                                    (void **)&rdev->ih.ring);
2550                 radeon_bo_unreserve(rdev->ih.ring_obj);
2551                 if (r) {
2552                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
2553                         return r;
2554                 }
2555         }
2556         return 0;
2557 }
2558
2559 static void r600_ih_ring_fini(struct radeon_device *rdev)
2560 {
2561         int r;
2562         if (rdev->ih.ring_obj) {
2563                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
2564                 if (likely(r == 0)) {
2565                         radeon_bo_kunmap(rdev->ih.ring_obj);
2566                         radeon_bo_unpin(rdev->ih.ring_obj);
2567                         radeon_bo_unreserve(rdev->ih.ring_obj);
2568                 }
2569                 radeon_bo_unref(&rdev->ih.ring_obj);
2570                 rdev->ih.ring = NULL;
2571                 rdev->ih.ring_obj = NULL;
2572         }
2573 }
2574
2575 void r600_rlc_stop(struct radeon_device *rdev)
2576 {
2577
2578         if ((rdev->family >= CHIP_RV770) &&
2579             (rdev->family <= CHIP_RV740)) {
2580                 /* r7xx asics need to soft reset RLC before halting */
2581                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
2582                 RREG32(SRBM_SOFT_RESET);
2583                 udelay(15000);
2584                 WREG32(SRBM_SOFT_RESET, 0);
2585                 RREG32(SRBM_SOFT_RESET);
2586         }
2587
2588         WREG32(RLC_CNTL, 0);
2589 }
2590
2591 static void r600_rlc_start(struct radeon_device *rdev)
2592 {
2593         WREG32(RLC_CNTL, RLC_ENABLE);
2594 }
2595
2596 static int r600_rlc_init(struct radeon_device *rdev)
2597 {
2598         u32 i;
2599         const __be32 *fw_data;
2600
2601         if (!rdev->rlc_fw)
2602                 return -EINVAL;
2603
2604         r600_rlc_stop(rdev);
2605
2606         WREG32(RLC_HB_BASE, 0);
2607         WREG32(RLC_HB_CNTL, 0);
2608         WREG32(RLC_HB_RPTR, 0);
2609         WREG32(RLC_HB_WPTR, 0);
2610         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
2611         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2612         WREG32(RLC_MC_CNTL, 0);
2613         WREG32(RLC_UCODE_CNTL, 0);
2614
2615         fw_data = (const __be32 *)rdev->rlc_fw->data;
2616         if (rdev->family >= CHIP_CEDAR) {
2617                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
2618                         WREG32(RLC_UCODE_ADDR, i);
2619                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2620                 }
2621         } else if (rdev->family >= CHIP_RV770) {
2622                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
2623                         WREG32(RLC_UCODE_ADDR, i);
2624                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2625                 }
2626         } else {
2627                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
2628                         WREG32(RLC_UCODE_ADDR, i);
2629                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
2630                 }
2631         }
2632         WREG32(RLC_UCODE_ADDR, 0);
2633
2634         r600_rlc_start(rdev);
2635
2636         return 0;
2637 }
2638
2639 static void r600_enable_interrupts(struct radeon_device *rdev)
2640 {
2641         u32 ih_cntl = RREG32(IH_CNTL);
2642         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2643
2644         ih_cntl |= ENABLE_INTR;
2645         ih_rb_cntl |= IH_RB_ENABLE;
2646         WREG32(IH_CNTL, ih_cntl);
2647         WREG32(IH_RB_CNTL, ih_rb_cntl);
2648         rdev->ih.enabled = true;
2649 }
2650
2651 void r600_disable_interrupts(struct radeon_device *rdev)
2652 {
2653         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
2654         u32 ih_cntl = RREG32(IH_CNTL);
2655
2656         ih_rb_cntl &= ~IH_RB_ENABLE;
2657         ih_cntl &= ~ENABLE_INTR;
2658         WREG32(IH_RB_CNTL, ih_rb_cntl);
2659         WREG32(IH_CNTL, ih_cntl);
2660         /* set rptr, wptr to 0 */
2661         WREG32(IH_RB_RPTR, 0);
2662         WREG32(IH_RB_WPTR, 0);
2663         rdev->ih.enabled = false;
2664         rdev->ih.wptr = 0;
2665         rdev->ih.rptr = 0;
2666 }
2667
2668 static void r600_disable_interrupt_state(struct radeon_device *rdev)
2669 {
2670         u32 tmp;
2671
2672         WREG32(CP_INT_CNTL, 0);
2673         WREG32(GRBM_INT_CNTL, 0);
2674         WREG32(DxMODE_INT_MASK, 0);
2675         if (ASIC_IS_DCE3(rdev)) {
2676                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
2677                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
2678                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2679                 WREG32(DC_HPD1_INT_CONTROL, tmp);
2680                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2681                 WREG32(DC_HPD2_INT_CONTROL, tmp);
2682                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2683                 WREG32(DC_HPD3_INT_CONTROL, tmp);
2684                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2685                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2686                 if (ASIC_IS_DCE32(rdev)) {
2687                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2688                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2689                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2690                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2691                 }
2692         } else {
2693                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2694                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2695                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2696                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2697                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2698                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2699                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
2700                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2701         }
2702 }
2703
2704 int r600_irq_init(struct radeon_device *rdev)
2705 {
2706         int ret = 0;
2707         int rb_bufsz;
2708         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
2709
2710         /* allocate ring */
2711         ret = r600_ih_ring_alloc(rdev);
2712         if (ret)
2713                 return ret;
2714
2715         /* disable irqs */
2716         r600_disable_interrupts(rdev);
2717
2718         /* init rlc */
2719         ret = r600_rlc_init(rdev);
2720         if (ret) {
2721                 r600_ih_ring_fini(rdev);
2722                 return ret;
2723         }
2724
2725         /* setup interrupt control */
2726         /* set dummy read address to ring address */
2727         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
2728         interrupt_cntl = RREG32(INTERRUPT_CNTL);
2729         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
2730          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
2731          */
2732         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
2733         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
2734         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
2735         WREG32(INTERRUPT_CNTL, interrupt_cntl);
2736
2737         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
2738         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
2739
2740         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
2741                       IH_WPTR_OVERFLOW_CLEAR |
2742                       (rb_bufsz << 1));
2743         /* WPTR writeback, not yet */
2744         /*ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;*/
2745         WREG32(IH_RB_WPTR_ADDR_LO, 0);
2746         WREG32(IH_RB_WPTR_ADDR_HI, 0);
2747
2748         WREG32(IH_RB_CNTL, ih_rb_cntl);
2749
2750         /* set rptr, wptr to 0 */
2751         WREG32(IH_RB_RPTR, 0);
2752         WREG32(IH_RB_WPTR, 0);
2753
2754         /* Default settings for IH_CNTL (disabled at first) */
2755         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
2756         /* RPTR_REARM only works if msi's are enabled */
2757         if (rdev->msi_enabled)
2758                 ih_cntl |= RPTR_REARM;
2759
2760 #ifdef __BIG_ENDIAN
2761         ih_cntl |= IH_MC_SWAP(IH_MC_SWAP_32BIT);
2762 #endif
2763         WREG32(IH_CNTL, ih_cntl);
2764
2765         /* force the active interrupt state to all disabled */
2766         if (rdev->family >= CHIP_CEDAR)
2767                 evergreen_disable_interrupt_state(rdev);
2768         else
2769                 r600_disable_interrupt_state(rdev);
2770
2771         /* enable irqs */
2772         r600_enable_interrupts(rdev);
2773
2774         return ret;
2775 }
2776
2777 void r600_irq_suspend(struct radeon_device *rdev)
2778 {
2779         r600_irq_disable(rdev);
2780         r600_rlc_stop(rdev);
2781 }
2782
2783 void r600_irq_fini(struct radeon_device *rdev)
2784 {
2785         r600_irq_suspend(rdev);
2786         r600_ih_ring_fini(rdev);
2787 }
2788
2789 int r600_irq_set(struct radeon_device *rdev)
2790 {
2791         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2792         u32 mode_int = 0;
2793         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2794         u32 grbm_int_cntl = 0;
2795         u32 hdmi1, hdmi2;
2796
2797         if (!rdev->irq.installed) {
2798                 WARN(1, "Can't enable IRQ/MSI because no handler is installed.\n");
2799                 return -EINVAL;
2800         }
2801         /* don't enable anything if the ih is disabled */
2802         if (!rdev->ih.enabled) {
2803                 r600_disable_interrupts(rdev);
2804                 /* force the active interrupt state to all disabled */
2805                 r600_disable_interrupt_state(rdev);
2806                 return 0;
2807         }
2808
2809         hdmi1 = RREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2810         if (ASIC_IS_DCE3(rdev)) {
2811                 hdmi2 = RREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2812                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2813                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2814                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2815                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2816                 if (ASIC_IS_DCE32(rdev)) {
2817                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2818                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2819                 }
2820         } else {
2821                 hdmi2 = RREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
2822                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2823                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2824                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2825         }
2826
2827         if (rdev->irq.sw_int) {
2828                 DRM_DEBUG("r600_irq_set: sw int\n");
2829                 cp_int_cntl |= RB_INT_ENABLE;
2830         }
2831         if (rdev->irq.crtc_vblank_int[0]) {
2832                 DRM_DEBUG("r600_irq_set: vblank 0\n");
2833                 mode_int |= D1MODE_VBLANK_INT_MASK;
2834         }
2835         if (rdev->irq.crtc_vblank_int[1]) {
2836                 DRM_DEBUG("r600_irq_set: vblank 1\n");
2837                 mode_int |= D2MODE_VBLANK_INT_MASK;
2838         }
2839         if (rdev->irq.hpd[0]) {
2840                 DRM_DEBUG("r600_irq_set: hpd 1\n");
2841                 hpd1 |= DC_HPDx_INT_EN;
2842         }
2843         if (rdev->irq.hpd[1]) {
2844                 DRM_DEBUG("r600_irq_set: hpd 2\n");
2845                 hpd2 |= DC_HPDx_INT_EN;
2846         }
2847         if (rdev->irq.hpd[2]) {
2848                 DRM_DEBUG("r600_irq_set: hpd 3\n");
2849                 hpd3 |= DC_HPDx_INT_EN;
2850         }
2851         if (rdev->irq.hpd[3]) {
2852                 DRM_DEBUG("r600_irq_set: hpd 4\n");
2853                 hpd4 |= DC_HPDx_INT_EN;
2854         }
2855         if (rdev->irq.hpd[4]) {
2856                 DRM_DEBUG("r600_irq_set: hpd 5\n");
2857                 hpd5 |= DC_HPDx_INT_EN;
2858         }
2859         if (rdev->irq.hpd[5]) {
2860                 DRM_DEBUG("r600_irq_set: hpd 6\n");
2861                 hpd6 |= DC_HPDx_INT_EN;
2862         }
2863         if (rdev->irq.hdmi[0]) {
2864                 DRM_DEBUG("r600_irq_set: hdmi 1\n");
2865                 hdmi1 |= R600_HDMI_INT_EN;
2866         }
2867         if (rdev->irq.hdmi[1]) {
2868                 DRM_DEBUG("r600_irq_set: hdmi 2\n");
2869                 hdmi2 |= R600_HDMI_INT_EN;
2870         }
2871         if (rdev->irq.gui_idle) {
2872                 DRM_DEBUG("gui idle\n");
2873                 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2874         }
2875
2876         WREG32(CP_INT_CNTL, cp_int_cntl);
2877         WREG32(DxMODE_INT_MASK, mode_int);
2878         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2879         WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
2880         if (ASIC_IS_DCE3(rdev)) {
2881                 WREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, hdmi2);
2882                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2883                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2884                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2885                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2886                 if (ASIC_IS_DCE32(rdev)) {
2887                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
2888                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
2889                 }
2890         } else {
2891                 WREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, hdmi2);
2892                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
2893                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
2894                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2895         }
2896
2897         return 0;
2898 }
2899
2900 static inline void r600_irq_ack(struct radeon_device *rdev,
2901                                 u32 *disp_int,
2902                                 u32 *disp_int_cont,
2903                                 u32 *disp_int_cont2)
2904 {
2905         u32 tmp;
2906
2907         if (ASIC_IS_DCE3(rdev)) {
2908                 *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
2909                 *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
2910                 *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2911         } else {
2912                 *disp_int = RREG32(DISP_INTERRUPT_STATUS);
2913                 *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2914                 *disp_int_cont2 = 0;
2915         }
2916
2917         if (*disp_int & LB_D1_VBLANK_INTERRUPT)
2918                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2919         if (*disp_int & LB_D1_VLINE_INTERRUPT)
2920                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2921         if (*disp_int & LB_D2_VBLANK_INTERRUPT)
2922                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
2923         if (*disp_int & LB_D2_VLINE_INTERRUPT)
2924                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
2925         if (*disp_int & DC_HPD1_INTERRUPT) {
2926                 if (ASIC_IS_DCE3(rdev)) {
2927                         tmp = RREG32(DC_HPD1_INT_CONTROL);
2928                         tmp |= DC_HPDx_INT_ACK;
2929                         WREG32(DC_HPD1_INT_CONTROL, tmp);
2930                 } else {
2931                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
2932                         tmp |= DC_HPDx_INT_ACK;
2933                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
2934                 }
2935         }
2936         if (*disp_int & DC_HPD2_INTERRUPT) {
2937                 if (ASIC_IS_DCE3(rdev)) {
2938                         tmp = RREG32(DC_HPD2_INT_CONTROL);
2939                         tmp |= DC_HPDx_INT_ACK;
2940                         WREG32(DC_HPD2_INT_CONTROL, tmp);
2941                 } else {
2942                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
2943                         tmp |= DC_HPDx_INT_ACK;
2944                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
2945                 }
2946         }
2947         if (*disp_int_cont & DC_HPD3_INTERRUPT) {
2948                 if (ASIC_IS_DCE3(rdev)) {
2949                         tmp = RREG32(DC_HPD3_INT_CONTROL);
2950                         tmp |= DC_HPDx_INT_ACK;
2951                         WREG32(DC_HPD3_INT_CONTROL, tmp);
2952                 } else {
2953                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
2954                         tmp |= DC_HPDx_INT_ACK;
2955                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2956                 }
2957         }
2958         if (*disp_int_cont & DC_HPD4_INTERRUPT) {
2959                 tmp = RREG32(DC_HPD4_INT_CONTROL);
2960                 tmp |= DC_HPDx_INT_ACK;
2961                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2962         }
2963         if (ASIC_IS_DCE32(rdev)) {
2964                 if (*disp_int_cont2 & DC_HPD5_INTERRUPT) {
2965                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2966                         tmp |= DC_HPDx_INT_ACK;
2967                         WREG32(DC_HPD5_INT_CONTROL, tmp);
2968                 }
2969                 if (*disp_int_cont2 & DC_HPD6_INTERRUPT) {
2970                         tmp = RREG32(DC_HPD5_INT_CONTROL);
2971                         tmp |= DC_HPDx_INT_ACK;
2972                         WREG32(DC_HPD6_INT_CONTROL, tmp);
2973                 }
2974         }
2975         if (RREG32(R600_HDMI_BLOCK1 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2976                 WREG32_P(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2977         }
2978         if (ASIC_IS_DCE3(rdev)) {
2979                 if (RREG32(R600_HDMI_BLOCK3 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2980                         WREG32_P(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2981                 }
2982         } else {
2983                 if (RREG32(R600_HDMI_BLOCK2 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
2984                         WREG32_P(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
2985                 }
2986         }
2987 }
2988
2989 void r600_irq_disable(struct radeon_device *rdev)
2990 {
2991         u32 disp_int, disp_int_cont, disp_int_cont2;
2992
2993         r600_disable_interrupts(rdev);
2994         /* Wait and acknowledge irq */
2995         mdelay(1);
2996         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
2997         r600_disable_interrupt_state(rdev);
2998 }
2999
3000 static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
3001 {
3002         u32 wptr, tmp;
3003
3004         /* XXX use writeback */
3005         wptr = RREG32(IH_RB_WPTR);
3006
3007         if (wptr & RB_OVERFLOW) {
3008                 /* When a ring buffer overflow happen start parsing interrupt
3009                  * from the last not overwritten vector (wptr + 16). Hopefully
3010                  * this should allow us to catchup.
3011                  */
3012                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3013                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3014                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3015                 tmp = RREG32(IH_RB_CNTL);
3016                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3017                 WREG32(IH_RB_CNTL, tmp);
3018         }
3019         return (wptr & rdev->ih.ptr_mask);
3020 }
3021
3022 /*        r600 IV Ring
3023  * Each IV ring entry is 128 bits:
3024  * [7:0]    - interrupt source id
3025  * [31:8]   - reserved
3026  * [59:32]  - interrupt source data
3027  * [127:60]  - reserved
3028  *
3029  * The basic interrupt vector entries
3030  * are decoded as follows:
3031  * src_id  src_data  description
3032  *      1         0  D1 Vblank
3033  *      1         1  D1 Vline
3034  *      5         0  D2 Vblank
3035  *      5         1  D2 Vline
3036  *     19         0  FP Hot plug detection A
3037  *     19         1  FP Hot plug detection B
3038  *     19         2  DAC A auto-detection
3039  *     19         3  DAC B auto-detection
3040  *     21         4  HDMI block A
3041  *     21         5  HDMI block B
3042  *    176         -  CP_INT RB
3043  *    177         -  CP_INT IB1
3044  *    178         -  CP_INT IB2
3045  *    181         -  EOP Interrupt
3046  *    233         -  GUI Idle
3047  *
3048  * Note, these are based on r600 and may need to be
3049  * adjusted or added to on newer asics
3050  */
3051
3052 int r600_irq_process(struct radeon_device *rdev)
3053 {
3054         u32 wptr = r600_get_ih_wptr(rdev);
3055         u32 rptr = rdev->ih.rptr;
3056         u32 src_id, src_data;
3057         u32 ring_index, disp_int, disp_int_cont, disp_int_cont2;
3058         unsigned long flags;
3059         bool queue_hotplug = false;
3060
3061         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3062         if (!rdev->ih.enabled)
3063                 return IRQ_NONE;
3064
3065         spin_lock_irqsave(&rdev->ih.lock, flags);
3066
3067         if (rptr == wptr) {
3068                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
3069                 return IRQ_NONE;
3070         }
3071         if (rdev->shutdown) {
3072                 spin_unlock_irqrestore(&rdev->ih.lock, flags);
3073                 return IRQ_NONE;
3074         }
3075
3076 restart_ih:
3077         /* display interrupts */
3078         r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2);
3079
3080         rdev->ih.wptr = wptr;
3081         while (rptr != wptr) {
3082                 /* wptr/rptr are in bytes! */
3083                 ring_index = rptr / 4;
3084                 src_id =  rdev->ih.ring[ring_index] & 0xff;
3085                 src_data = rdev->ih.ring[ring_index + 1] & 0xfffffff;
3086
3087                 switch (src_id) {
3088                 case 1: /* D1 vblank/vline */
3089                         switch (src_data) {
3090                         case 0: /* D1 vblank */
3091                                 if (disp_int & LB_D1_VBLANK_INTERRUPT) {
3092                                         drm_handle_vblank(rdev->ddev, 0);
3093                                         rdev->pm.vblank_sync = true;
3094                                         wake_up(&rdev->irq.vblank_queue);
3095                                         disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3096                                         DRM_DEBUG("IH: D1 vblank\n");
3097                                 }
3098                                 break;
3099                         case 1: /* D1 vline */
3100                                 if (disp_int & LB_D1_VLINE_INTERRUPT) {
3101                                         disp_int &= ~LB_D1_VLINE_INTERRUPT;
3102                                         DRM_DEBUG("IH: D1 vline\n");
3103                                 }
3104                                 break;
3105                         default:
3106                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3107                                 break;
3108                         }
3109                         break;
3110                 case 5: /* D2 vblank/vline */
3111                         switch (src_data) {
3112                         case 0: /* D2 vblank */
3113                                 if (disp_int & LB_D2_VBLANK_INTERRUPT) {
3114                                         drm_handle_vblank(rdev->ddev, 1);
3115                                         rdev->pm.vblank_sync = true;
3116                                         wake_up(&rdev->irq.vblank_queue);
3117                                         disp_int &= ~LB_D2_VBLANK_INTERRUPT;
3118                                         DRM_DEBUG("IH: D2 vblank\n");
3119                                 }
3120                                 break;
3121                         case 1: /* D1 vline */
3122                                 if (disp_int & LB_D2_VLINE_INTERRUPT) {
3123                                         disp_int &= ~LB_D2_VLINE_INTERRUPT;
3124                                         DRM_DEBUG("IH: D2 vline\n");
3125                                 }
3126                                 break;
3127                         default:
3128                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3129                                 break;
3130                         }
3131                         break;
3132                 case 19: /* HPD/DAC hotplug */
3133                         switch (src_data) {
3134                         case 0:
3135                                 if (disp_int & DC_HPD1_INTERRUPT) {
3136                                         disp_int &= ~DC_HPD1_INTERRUPT;
3137                                         queue_hotplug = true;
3138                                         DRM_DEBUG("IH: HPD1\n");
3139                                 }
3140                                 break;
3141                         case 1:
3142                                 if (disp_int & DC_HPD2_INTERRUPT) {
3143                                         disp_int &= ~DC_HPD2_INTERRUPT;
3144                                         queue_hotplug = true;
3145                                         DRM_DEBUG("IH: HPD2\n");
3146                                 }
3147                                 break;
3148                         case 4:
3149                                 if (disp_int_cont & DC_HPD3_INTERRUPT) {
3150                                         disp_int_cont &= ~DC_HPD3_INTERRUPT;
3151                                         queue_hotplug = true;
3152                                         DRM_DEBUG("IH: HPD3\n");
3153                                 }
3154                                 break;
3155                         case 5:
3156                                 if (disp_int_cont & DC_HPD4_INTERRUPT) {
3157                                         disp_int_cont &= ~DC_HPD4_INTERRUPT;
3158                                         queue_hotplug = true;
3159                                         DRM_DEBUG("IH: HPD4\n");
3160                                 }
3161                                 break;
3162                         case 10:
3163                                 if (disp_int_cont2 & DC_HPD5_INTERRUPT) {
3164                                         disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
3165                                         queue_hotplug = true;
3166                                         DRM_DEBUG("IH: HPD5\n");
3167                                 }
3168                                 break;
3169                         case 12:
3170                                 if (disp_int_cont2 & DC_HPD6_INTERRUPT) {
3171                                         disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
3172                                         queue_hotplug = true;
3173                                         DRM_DEBUG("IH: HPD6\n");
3174                                 }
3175                                 break;
3176                         default:
3177                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3178                                 break;
3179                         }
3180                         break;
3181                 case 21: /* HDMI */
3182                         DRM_DEBUG("IH: HDMI: 0x%x\n", src_data);
3183                         r600_audio_schedule_polling(rdev);
3184                         break;
3185                 case 176: /* CP_INT in ring buffer */
3186                 case 177: /* CP_INT in IB1 */
3187                 case 178: /* CP_INT in IB2 */
3188                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
3189                         radeon_fence_process(rdev);
3190                         break;
3191                 case 181: /* CP EOP event */
3192                         DRM_DEBUG("IH: CP EOP\n");
3193                         break;
3194                 case 233: /* GUI IDLE */
3195                         DRM_DEBUG("IH: CP EOP\n");
3196                         rdev->pm.gui_idle = true;
3197                         wake_up(&rdev->irq.idle_queue);
3198                         break;
3199                 default:
3200                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3201                         break;
3202                 }
3203
3204                 /* wptr/rptr are in bytes! */
3205                 rptr += 16;
3206                 rptr &= rdev->ih.ptr_mask;
3207         }
3208         /* make sure wptr hasn't changed while processing */
3209         wptr = r600_get_ih_wptr(rdev);
3210         if (wptr != rdev->ih.wptr)
3211                 goto restart_ih;
3212         if (queue_hotplug)
3213                 queue_work(rdev->wq, &rdev->hotplug_work);
3214         rdev->ih.rptr = rptr;
3215         WREG32(IH_RB_RPTR, rdev->ih.rptr);
3216         spin_unlock_irqrestore(&rdev->ih.lock, flags);
3217         return IRQ_HANDLED;
3218 }
3219
3220 /*
3221  * Debugfs info
3222  */
3223 #if defined(CONFIG_DEBUG_FS)
3224
3225 static int r600_debugfs_cp_ring_info(struct seq_file *m, void *data)
3226 {
3227         struct drm_info_node *node = (struct drm_info_node *) m->private;
3228         struct drm_device *dev = node->minor->dev;
3229         struct radeon_device *rdev = dev->dev_private;
3230         unsigned count, i, j;
3231
3232         radeon_ring_free_size(rdev);
3233         count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
3234         seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
3235         seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
3236         seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
3237         seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
3238         seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
3239         seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
3240         seq_printf(m, "%u dwords in ring\n", count);
3241         i = rdev->cp.rptr;
3242         for (j = 0; j <= count; j++) {
3243                 seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
3244                 i = (i + 1) & rdev->cp.ptr_mask;
3245         }
3246         return 0;
3247 }
3248
3249 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
3250 {
3251         struct drm_info_node *node = (struct drm_info_node *) m->private;
3252         struct drm_device *dev = node->minor->dev;
3253         struct radeon_device *rdev = dev->dev_private;
3254
3255         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
3256         DREG32_SYS(m, rdev, VM_L2_STATUS);
3257         return 0;
3258 }
3259
3260 static struct drm_info_list r600_mc_info_list[] = {
3261         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
3262         {"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
3263 };
3264 #endif
3265
3266 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
3267 {
3268 #if defined(CONFIG_DEBUG_FS)
3269         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
3270 #else
3271         return 0;
3272 #endif
3273 }
3274
3275 /**
3276  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
3277  * rdev: radeon device structure
3278  * bo: buffer object struct which userspace is waiting for idle
3279  *
3280  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
3281  * through ring buffer, this leads to corruption in rendering, see
3282  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
3283  * directly perform HDP flush by writing register through MMIO.
3284  */
3285 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
3286 {
3287         WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3288 }