drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "radeon_asic.h"
  29 #include "radeon_audio.h"
  30 #include <drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37
  38 /*
  39  * Indirect registers accessor
  40  */
  41 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  42 {
  43         unsigned long flags;
  44         u32 r;
  45
  46         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  47         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  48         r = RREG32(EVERGREEN_CG_IND_DATA);
  49         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  50         return r;
  51 }
  52
  53 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  54 {
  55         unsigned long flags;
  56
  57         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  58         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  59         WREG32(EVERGREEN_CG_IND_DATA, (v));
  60         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  61 }
  62
  63 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  64 {
  65         unsigned long flags;
  66         u32 r;
  67
  68         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  69         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  70         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  71         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  72         return r;
  73 }
  74
  75 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  76 {
  77         unsigned long flags;
  78
  79         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  80         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  81         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  82         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  83 }
  84
  85 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  86 {
  87         unsigned long flags;
  88         u32 r;
  89
  90         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  91         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
  92         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
  93         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  94         return r;
  95 }
  96
  97 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  98 {
  99         unsigned long flags;
 100
 101         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 102         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 103         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 104         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 105 }
 106
 107 static const u32 crtc_offsets[6] =
 108 {
 109         EVERGREEN_CRTC0_REGISTER_OFFSET,
 110         EVERGREEN_CRTC1_REGISTER_OFFSET,
 111         EVERGREEN_CRTC2_REGISTER_OFFSET,
 112         EVERGREEN_CRTC3_REGISTER_OFFSET,
 113         EVERGREEN_CRTC4_REGISTER_OFFSET,
 114         EVERGREEN_CRTC5_REGISTER_OFFSET
 115 };
 116
 117 #include "clearstate_evergreen.h"
 118
 119 static const u32 sumo_rlc_save_restore_register_list[] =
 120 {
 121         0x98fc,
 122         0x9830,
 123         0x9834,
 124         0x9838,
 125         0x9870,
 126         0x9874,
 127         0x8a14,
 128         0x8b24,
 129         0x8bcc,
 130         0x8b10,
 131         0x8d00,
 132         0x8d04,
 133         0x8c00,
 134         0x8c04,
 135         0x8c08,
 136         0x8c0c,
 137         0x8d8c,
 138         0x8c20,
 139         0x8c24,
 140         0x8c28,
 141         0x8c18,
 142         0x8c1c,
 143         0x8cf0,
 144         0x8e2c,
 145         0x8e38,
 146         0x8c30,
 147         0x9508,
 148         0x9688,
 149         0x9608,
 150         0x960c,
 151         0x9610,
 152         0x9614,
 153         0x88c4,
 154         0x88d4,
 155         0xa008,
 156         0x900c,
 157         0x9100,
 158         0x913c,
 159         0x98f8,
 160         0x98f4,
 161         0x9b7c,
 162         0x3f8c,
 163         0x8950,
 164         0x8954,
 165         0x8a18,
 166         0x8b28,
 167         0x9144,
 168         0x9148,
 169         0x914c,
 170         0x3f90,
 171         0x3f94,
 172         0x915c,
 173         0x9160,
 174         0x9178,
 175         0x917c,
 176         0x9180,
 177         0x918c,
 178         0x9190,
 179         0x9194,
 180         0x9198,
 181         0x919c,
 182         0x91a8,
 183         0x91ac,
 184         0x91b0,
 185         0x91b4,
 186         0x91b8,
 187         0x91c4,
 188         0x91c8,
 189         0x91cc,
 190         0x91d0,
 191         0x91d4,
 192         0x91e0,
 193         0x91e4,
 194         0x91ec,
 195         0x91f0,
 196         0x91f4,
 197         0x9200,
 198         0x9204,
 199         0x929c,
 200         0x9150,
 201         0x802c,
 202 };
 203
 204 static void evergreen_gpu_init(struct radeon_device *rdev);
 205 void evergreen_fini(struct radeon_device *rdev);
 206 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 207 void evergreen_program_aspm(struct radeon_device *rdev);
 208 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 209                                      int ring, u32 cp_int_cntl);
 210 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 211                                    u32 status, u32 addr);
 212 void cik_init_cp_pg_table(struct radeon_device *rdev);
 213
 214 extern u32 si_get_csb_size(struct radeon_device *rdev);
 215 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 216 extern u32 cik_get_csb_size(struct radeon_device *rdev);
 217 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 218 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 219
 220 static const u32 evergreen_golden_registers[] =
 221 {
 222         0x3f90, 0xffff0000, 0xff000000,
 223         0x9148, 0xffff0000, 0xff000000,
 224         0x3f94, 0xffff0000, 0xff000000,
 225         0x914c, 0xffff0000, 0xff000000,
 226         0x9b7c, 0xffffffff, 0x00000000,
 227         0x8a14, 0xffffffff, 0x00000007,
 228         0x8b10, 0xffffffff, 0x00000000,
 229         0x960c, 0xffffffff, 0x54763210,
 230         0x88c4, 0xffffffff, 0x000000c2,
 231         0x88d4, 0xffffffff, 0x00000010,
 232         0x8974, 0xffffffff, 0x00000000,
 233         0xc78, 0x00000080, 0x00000080,
 234         0x5eb4, 0xffffffff, 0x00000002,
 235         0x5e78, 0xffffffff, 0x001000f0,
 236         0x6104, 0x01000300, 0x00000000,
 237         0x5bc0, 0x00300000, 0x00000000,
 238         0x7030, 0xffffffff, 0x00000011,
 239         0x7c30, 0xffffffff, 0x00000011,
 240         0x10830, 0xffffffff, 0x00000011,
 241         0x11430, 0xffffffff, 0x00000011,
 242         0x12030, 0xffffffff, 0x00000011,
 243         0x12c30, 0xffffffff, 0x00000011,
 244         0xd02c, 0xffffffff, 0x08421000,
 245         0x240c, 0xffffffff, 0x00000380,
 246         0x8b24, 0xffffffff, 0x00ff0fff,
 247         0x28a4c, 0x06000000, 0x06000000,
 248         0x10c, 0x00000001, 0x00000001,
 249         0x8d00, 0xffffffff, 0x100e4848,
 250         0x8d04, 0xffffffff, 0x00164745,
 251         0x8c00, 0xffffffff, 0xe4000003,
 252         0x8c04, 0xffffffff, 0x40600060,
 253         0x8c08, 0xffffffff, 0x001c001c,
 254         0x8cf0, 0xffffffff, 0x08e00620,
 255         0x8c20, 0xffffffff, 0x00800080,
 256         0x8c24, 0xffffffff, 0x00800080,
 257         0x8c18, 0xffffffff, 0x20202078,
 258         0x8c1c, 0xffffffff, 0x00001010,
 259         0x28350, 0xffffffff, 0x00000000,
 260         0xa008, 0xffffffff, 0x00010000,
 261         0x5c4, 0xffffffff, 0x00000001,
 262         0x9508, 0xffffffff, 0x00000002,
 263         0x913c, 0x0000000f, 0x0000000a
 264 };
 265
 266 static const u32 evergreen_golden_registers2[] =
 267 {
 268         0x2f4c, 0xffffffff, 0x00000000,
 269         0x54f4, 0xffffffff, 0x00000000,
 270         0x54f0, 0xffffffff, 0x00000000,
 271         0x5498, 0xffffffff, 0x00000000,
 272         0x549c, 0xffffffff, 0x00000000,
 273         0x5494, 0xffffffff, 0x00000000,
 274         0x53cc, 0xffffffff, 0x00000000,
 275         0x53c8, 0xffffffff, 0x00000000,
 276         0x53c4, 0xffffffff, 0x00000000,
 277         0x53c0, 0xffffffff, 0x00000000,
 278         0x53bc, 0xffffffff, 0x00000000,
 279         0x53b8, 0xffffffff, 0x00000000,
 280         0x53b4, 0xffffffff, 0x00000000,
 281         0x53b0, 0xffffffff, 0x00000000
 282 };
 283
 284 static const u32 cypress_mgcg_init[] =
 285 {
 286         0x802c, 0xffffffff, 0xc0000000,
 287         0x5448, 0xffffffff, 0x00000100,
 288         0x55e4, 0xffffffff, 0x00000100,
 289         0x160c, 0xffffffff, 0x00000100,
 290         0x5644, 0xffffffff, 0x00000100,
 291         0xc164, 0xffffffff, 0x00000100,
 292         0x8a18, 0xffffffff, 0x00000100,
 293         0x897c, 0xffffffff, 0x06000100,
 294         0x8b28, 0xffffffff, 0x00000100,
 295         0x9144, 0xffffffff, 0x00000100,
 296         0x9a60, 0xffffffff, 0x00000100,
 297         0x9868, 0xffffffff, 0x00000100,
 298         0x8d58, 0xffffffff, 0x00000100,
 299         0x9510, 0xffffffff, 0x00000100,
 300         0x949c, 0xffffffff, 0x00000100,
 301         0x9654, 0xffffffff, 0x00000100,
 302         0x9030, 0xffffffff, 0x00000100,
 303         0x9034, 0xffffffff, 0x00000100,
 304         0x9038, 0xffffffff, 0x00000100,
 305         0x903c, 0xffffffff, 0x00000100,
 306         0x9040, 0xffffffff, 0x00000100,
 307         0xa200, 0xffffffff, 0x00000100,
 308         0xa204, 0xffffffff, 0x00000100,
 309         0xa208, 0xffffffff, 0x00000100,
 310         0xa20c, 0xffffffff, 0x00000100,
 311         0x971c, 0xffffffff, 0x00000100,
 312         0x977c, 0xffffffff, 0x00000100,
 313         0x3f80, 0xffffffff, 0x00000100,
 314         0xa210, 0xffffffff, 0x00000100,
 315         0xa214, 0xffffffff, 0x00000100,
 316         0x4d8, 0xffffffff, 0x00000100,
 317         0x9784, 0xffffffff, 0x00000100,
 318         0x9698, 0xffffffff, 0x00000100,
 319         0x4d4, 0xffffffff, 0x00000200,
 320         0x30cc, 0xffffffff, 0x00000100,
 321         0xd0c0, 0xffffffff, 0xff000100,
 322         0x802c, 0xffffffff, 0x40000000,
 323         0x915c, 0xffffffff, 0x00010000,
 324         0x9160, 0xffffffff, 0x00030002,
 325         0x9178, 0xffffffff, 0x00070000,
 326         0x917c, 0xffffffff, 0x00030002,
 327         0x9180, 0xffffffff, 0x00050004,
 328         0x918c, 0xffffffff, 0x00010006,
 329         0x9190, 0xffffffff, 0x00090008,
 330         0x9194, 0xffffffff, 0x00070000,
 331         0x9198, 0xffffffff, 0x00030002,
 332         0x919c, 0xffffffff, 0x00050004,
 333         0x91a8, 0xffffffff, 0x00010006,
 334         0x91ac, 0xffffffff, 0x00090008,
 335         0x91b0, 0xffffffff, 0x00070000,
 336         0x91b4, 0xffffffff, 0x00030002,
 337         0x91b8, 0xffffffff, 0x00050004,
 338         0x91c4, 0xffffffff, 0x00010006,
 339         0x91c8, 0xffffffff, 0x00090008,
 340         0x91cc, 0xffffffff, 0x00070000,
 341         0x91d0, 0xffffffff, 0x00030002,
 342         0x91d4, 0xffffffff, 0x00050004,
 343         0x91e0, 0xffffffff, 0x00010006,
 344         0x91e4, 0xffffffff, 0x00090008,
 345         0x91e8, 0xffffffff, 0x00000000,
 346         0x91ec, 0xffffffff, 0x00070000,
 347         0x91f0, 0xffffffff, 0x00030002,
 348         0x91f4, 0xffffffff, 0x00050004,
 349         0x9200, 0xffffffff, 0x00010006,
 350         0x9204, 0xffffffff, 0x00090008,
 351         0x9208, 0xffffffff, 0x00070000,
 352         0x920c, 0xffffffff, 0x00030002,
 353         0x9210, 0xffffffff, 0x00050004,
 354         0x921c, 0xffffffff, 0x00010006,
 355         0x9220, 0xffffffff, 0x00090008,
 356         0x9224, 0xffffffff, 0x00070000,
 357         0x9228, 0xffffffff, 0x00030002,
 358         0x922c, 0xffffffff, 0x00050004,
 359         0x9238, 0xffffffff, 0x00010006,
 360         0x923c, 0xffffffff, 0x00090008,
 361         0x9240, 0xffffffff, 0x00070000,
 362         0x9244, 0xffffffff, 0x00030002,
 363         0x9248, 0xffffffff, 0x00050004,
 364         0x9254, 0xffffffff, 0x00010006,
 365         0x9258, 0xffffffff, 0x00090008,
 366         0x925c, 0xffffffff, 0x00070000,
 367         0x9260, 0xffffffff, 0x00030002,
 368         0x9264, 0xffffffff, 0x00050004,
 369         0x9270, 0xffffffff, 0x00010006,
 370         0x9274, 0xffffffff, 0x00090008,
 371         0x9278, 0xffffffff, 0x00070000,
 372         0x927c, 0xffffffff, 0x00030002,
 373         0x9280, 0xffffffff, 0x00050004,
 374         0x928c, 0xffffffff, 0x00010006,
 375         0x9290, 0xffffffff, 0x00090008,
 376         0x9294, 0xffffffff, 0x00000000,
 377         0x929c, 0xffffffff, 0x00000001,
 378         0x802c, 0xffffffff, 0x40010000,
 379         0x915c, 0xffffffff, 0x00010000,
 380         0x9160, 0xffffffff, 0x00030002,
 381         0x9178, 0xffffffff, 0x00070000,
 382         0x917c, 0xffffffff, 0x00030002,
 383         0x9180, 0xffffffff, 0x00050004,
 384         0x918c, 0xffffffff, 0x00010006,
 385         0x9190, 0xffffffff, 0x00090008,
 386         0x9194, 0xffffffff, 0x00070000,
 387         0x9198, 0xffffffff, 0x00030002,
 388         0x919c, 0xffffffff, 0x00050004,
 389         0x91a8, 0xffffffff, 0x00010006,
 390         0x91ac, 0xffffffff, 0x00090008,
 391         0x91b0, 0xffffffff, 0x00070000,
 392         0x91b4, 0xffffffff, 0x00030002,
 393         0x91b8, 0xffffffff, 0x00050004,
 394         0x91c4, 0xffffffff, 0x00010006,
 395         0x91c8, 0xffffffff, 0x00090008,
 396         0x91cc, 0xffffffff, 0x00070000,
 397         0x91d0, 0xffffffff, 0x00030002,
 398         0x91d4, 0xffffffff, 0x00050004,
 399         0x91e0, 0xffffffff, 0x00010006,
 400         0x91e4, 0xffffffff, 0x00090008,
 401         0x91e8, 0xffffffff, 0x00000000,
 402         0x91ec, 0xffffffff, 0x00070000,
 403         0x91f0, 0xffffffff, 0x00030002,
 404         0x91f4, 0xffffffff, 0x00050004,
 405         0x9200, 0xffffffff, 0x00010006,
 406         0x9204, 0xffffffff, 0x00090008,
 407         0x9208, 0xffffffff, 0x00070000,
 408         0x920c, 0xffffffff, 0x00030002,
 409         0x9210, 0xffffffff, 0x00050004,
 410         0x921c, 0xffffffff, 0x00010006,
 411         0x9220, 0xffffffff, 0x00090008,
 412         0x9224, 0xffffffff, 0x00070000,
 413         0x9228, 0xffffffff, 0x00030002,
 414         0x922c, 0xffffffff, 0x00050004,
 415         0x9238, 0xffffffff, 0x00010006,
 416         0x923c, 0xffffffff, 0x00090008,
 417         0x9240, 0xffffffff, 0x00070000,
 418         0x9244, 0xffffffff, 0x00030002,
 419         0x9248, 0xffffffff, 0x00050004,
 420         0x9254, 0xffffffff, 0x00010006,
 421         0x9258, 0xffffffff, 0x00090008,
 422         0x925c, 0xffffffff, 0x00070000,
 423         0x9260, 0xffffffff, 0x00030002,
 424         0x9264, 0xffffffff, 0x00050004,
 425         0x9270, 0xffffffff, 0x00010006,
 426         0x9274, 0xffffffff, 0x00090008,
 427         0x9278, 0xffffffff, 0x00070000,
 428         0x927c, 0xffffffff, 0x00030002,
 429         0x9280, 0xffffffff, 0x00050004,
 430         0x928c, 0xffffffff, 0x00010006,
 431         0x9290, 0xffffffff, 0x00090008,
 432         0x9294, 0xffffffff, 0x00000000,
 433         0x929c, 0xffffffff, 0x00000001,
 434         0x802c, 0xffffffff, 0xc0000000
 435 };
 436
 437 static const u32 redwood_mgcg_init[] =
 438 {
 439         0x802c, 0xffffffff, 0xc0000000,
 440         0x5448, 0xffffffff, 0x00000100,
 441         0x55e4, 0xffffffff, 0x00000100,
 442         0x160c, 0xffffffff, 0x00000100,
 443         0x5644, 0xffffffff, 0x00000100,
 444         0xc164, 0xffffffff, 0x00000100,
 445         0x8a18, 0xffffffff, 0x00000100,
 446         0x897c, 0xffffffff, 0x06000100,
 447         0x8b28, 0xffffffff, 0x00000100,
 448         0x9144, 0xffffffff, 0x00000100,
 449         0x9a60, 0xffffffff, 0x00000100,
 450         0x9868, 0xffffffff, 0x00000100,
 451         0x8d58, 0xffffffff, 0x00000100,
 452         0x9510, 0xffffffff, 0x00000100,
 453         0x949c, 0xffffffff, 0x00000100,
 454         0x9654, 0xffffffff, 0x00000100,
 455         0x9030, 0xffffffff, 0x00000100,
 456         0x9034, 0xffffffff, 0x00000100,
 457         0x9038, 0xffffffff, 0x00000100,
 458         0x903c, 0xffffffff, 0x00000100,
 459         0x9040, 0xffffffff, 0x00000100,
 460         0xa200, 0xffffffff, 0x00000100,
 461         0xa204, 0xffffffff, 0x00000100,
 462         0xa208, 0xffffffff, 0x00000100,
 463         0xa20c, 0xffffffff, 0x00000100,
 464         0x971c, 0xffffffff, 0x00000100,
 465         0x977c, 0xffffffff, 0x00000100,
 466         0x3f80, 0xffffffff, 0x00000100,
 467         0xa210, 0xffffffff, 0x00000100,
 468         0xa214, 0xffffffff, 0x00000100,
 469         0x4d8, 0xffffffff, 0x00000100,
 470         0x9784, 0xffffffff, 0x00000100,
 471         0x9698, 0xffffffff, 0x00000100,
 472         0x4d4, 0xffffffff, 0x00000200,
 473         0x30cc, 0xffffffff, 0x00000100,
 474         0xd0c0, 0xffffffff, 0xff000100,
 475         0x802c, 0xffffffff, 0x40000000,
 476         0x915c, 0xffffffff, 0x00010000,
 477         0x9160, 0xffffffff, 0x00030002,
 478         0x9178, 0xffffffff, 0x00070000,
 479         0x917c, 0xffffffff, 0x00030002,
 480         0x9180, 0xffffffff, 0x00050004,
 481         0x918c, 0xffffffff, 0x00010006,
 482         0x9190, 0xffffffff, 0x00090008,
 483         0x9194, 0xffffffff, 0x00070000,
 484         0x9198, 0xffffffff, 0x00030002,
 485         0x919c, 0xffffffff, 0x00050004,
 486         0x91a8, 0xffffffff, 0x00010006,
 487         0x91ac, 0xffffffff, 0x00090008,
 488         0x91b0, 0xffffffff, 0x00070000,
 489         0x91b4, 0xffffffff, 0x00030002,
 490         0x91b8, 0xffffffff, 0x00050004,
 491         0x91c4, 0xffffffff, 0x00010006,
 492         0x91c8, 0xffffffff, 0x00090008,
 493         0x91cc, 0xffffffff, 0x00070000,
 494         0x91d0, 0xffffffff, 0x00030002,
 495         0x91d4, 0xffffffff, 0x00050004,
 496         0x91e0, 0xffffffff, 0x00010006,
 497         0x91e4, 0xffffffff, 0x00090008,
 498         0x91e8, 0xffffffff, 0x00000000,
 499         0x91ec, 0xffffffff, 0x00070000,
 500         0x91f0, 0xffffffff, 0x00030002,
 501         0x91f4, 0xffffffff, 0x00050004,
 502         0x9200, 0xffffffff, 0x00010006,
 503         0x9204, 0xffffffff, 0x00090008,
 504         0x9294, 0xffffffff, 0x00000000,
 505         0x929c, 0xffffffff, 0x00000001,
 506         0x802c, 0xffffffff, 0xc0000000
 507 };
 508
 509 static const u32 cedar_golden_registers[] =
 510 {
 511         0x3f90, 0xffff0000, 0xff000000,
 512         0x9148, 0xffff0000, 0xff000000,
 513         0x3f94, 0xffff0000, 0xff000000,
 514         0x914c, 0xffff0000, 0xff000000,
 515         0x9b7c, 0xffffffff, 0x00000000,
 516         0x8a14, 0xffffffff, 0x00000007,
 517         0x8b10, 0xffffffff, 0x00000000,
 518         0x960c, 0xffffffff, 0x54763210,
 519         0x88c4, 0xffffffff, 0x000000c2,
 520         0x88d4, 0xffffffff, 0x00000000,
 521         0x8974, 0xffffffff, 0x00000000,
 522         0xc78, 0x00000080, 0x00000080,
 523         0x5eb4, 0xffffffff, 0x00000002,
 524         0x5e78, 0xffffffff, 0x001000f0,
 525         0x6104, 0x01000300, 0x00000000,
 526         0x5bc0, 0x00300000, 0x00000000,
 527         0x7030, 0xffffffff, 0x00000011,
 528         0x7c30, 0xffffffff, 0x00000011,
 529         0x10830, 0xffffffff, 0x00000011,
 530         0x11430, 0xffffffff, 0x00000011,
 531         0xd02c, 0xffffffff, 0x08421000,
 532         0x240c, 0xffffffff, 0x00000380,
 533         0x8b24, 0xffffffff, 0x00ff0fff,
 534         0x28a4c, 0x06000000, 0x06000000,
 535         0x10c, 0x00000001, 0x00000001,
 536         0x8d00, 0xffffffff, 0x100e4848,
 537         0x8d04, 0xffffffff, 0x00164745,
 538         0x8c00, 0xffffffff, 0xe4000003,
 539         0x8c04, 0xffffffff, 0x40600060,
 540         0x8c08, 0xffffffff, 0x001c001c,
 541         0x8cf0, 0xffffffff, 0x08e00410,
 542         0x8c20, 0xffffffff, 0x00800080,
 543         0x8c24, 0xffffffff, 0x00800080,
 544         0x8c18, 0xffffffff, 0x20202078,
 545         0x8c1c, 0xffffffff, 0x00001010,
 546         0x28350, 0xffffffff, 0x00000000,
 547         0xa008, 0xffffffff, 0x00010000,
 548         0x5c4, 0xffffffff, 0x00000001,
 549         0x9508, 0xffffffff, 0x00000002
 550 };
 551
 552 static const u32 cedar_mgcg_init[] =
 553 {
 554         0x802c, 0xffffffff, 0xc0000000,
 555         0x5448, 0xffffffff, 0x00000100,
 556         0x55e4, 0xffffffff, 0x00000100,
 557         0x160c, 0xffffffff, 0x00000100,
 558         0x5644, 0xffffffff, 0x00000100,
 559         0xc164, 0xffffffff, 0x00000100,
 560         0x8a18, 0xffffffff, 0x00000100,
 561         0x897c, 0xffffffff, 0x06000100,
 562         0x8b28, 0xffffffff, 0x00000100,
 563         0x9144, 0xffffffff, 0x00000100,
 564         0x9a60, 0xffffffff, 0x00000100,
 565         0x9868, 0xffffffff, 0x00000100,
 566         0x8d58, 0xffffffff, 0x00000100,
 567         0x9510, 0xffffffff, 0x00000100,
 568         0x949c, 0xffffffff, 0x00000100,
 569         0x9654, 0xffffffff, 0x00000100,
 570         0x9030, 0xffffffff, 0x00000100,
 571         0x9034, 0xffffffff, 0x00000100,
 572         0x9038, 0xffffffff, 0x00000100,
 573         0x903c, 0xffffffff, 0x00000100,
 574         0x9040, 0xffffffff, 0x00000100,
 575         0xa200, 0xffffffff, 0x00000100,
 576         0xa204, 0xffffffff, 0x00000100,
 577         0xa208, 0xffffffff, 0x00000100,
 578         0xa20c, 0xffffffff, 0x00000100,
 579         0x971c, 0xffffffff, 0x00000100,
 580         0x977c, 0xffffffff, 0x00000100,
 581         0x3f80, 0xffffffff, 0x00000100,
 582         0xa210, 0xffffffff, 0x00000100,
 583         0xa214, 0xffffffff, 0x00000100,
 584         0x4d8, 0xffffffff, 0x00000100,
 585         0x9784, 0xffffffff, 0x00000100,
 586         0x9698, 0xffffffff, 0x00000100,
 587         0x4d4, 0xffffffff, 0x00000200,
 588         0x30cc, 0xffffffff, 0x00000100,
 589         0xd0c0, 0xffffffff, 0xff000100,
 590         0x802c, 0xffffffff, 0x40000000,
 591         0x915c, 0xffffffff, 0x00010000,
 592         0x9178, 0xffffffff, 0x00050000,
 593         0x917c, 0xffffffff, 0x00030002,
 594         0x918c, 0xffffffff, 0x00010004,
 595         0x9190, 0xffffffff, 0x00070006,
 596         0x9194, 0xffffffff, 0x00050000,
 597         0x9198, 0xffffffff, 0x00030002,
 598         0x91a8, 0xffffffff, 0x00010004,
 599         0x91ac, 0xffffffff, 0x00070006,
 600         0x91e8, 0xffffffff, 0x00000000,
 601         0x9294, 0xffffffff, 0x00000000,
 602         0x929c, 0xffffffff, 0x00000001,
 603         0x802c, 0xffffffff, 0xc0000000
 604 };
 605
 606 static const u32 juniper_mgcg_init[] =
 607 {
 608         0x802c, 0xffffffff, 0xc0000000,
 609         0x5448, 0xffffffff, 0x00000100,
 610         0x55e4, 0xffffffff, 0x00000100,
 611         0x160c, 0xffffffff, 0x00000100,
 612         0x5644, 0xffffffff, 0x00000100,
 613         0xc164, 0xffffffff, 0x00000100,
 614         0x8a18, 0xffffffff, 0x00000100,
 615         0x897c, 0xffffffff, 0x06000100,
 616         0x8b28, 0xffffffff, 0x00000100,
 617         0x9144, 0xffffffff, 0x00000100,
 618         0x9a60, 0xffffffff, 0x00000100,
 619         0x9868, 0xffffffff, 0x00000100,
 620         0x8d58, 0xffffffff, 0x00000100,
 621         0x9510, 0xffffffff, 0x00000100,
 622         0x949c, 0xffffffff, 0x00000100,
 623         0x9654, 0xffffffff, 0x00000100,
 624         0x9030, 0xffffffff, 0x00000100,
 625         0x9034, 0xffffffff, 0x00000100,
 626         0x9038, 0xffffffff, 0x00000100,
 627         0x903c, 0xffffffff, 0x00000100,
 628         0x9040, 0xffffffff, 0x00000100,
 629         0xa200, 0xffffffff, 0x00000100,
 630         0xa204, 0xffffffff, 0x00000100,
 631         0xa208, 0xffffffff, 0x00000100,
 632         0xa20c, 0xffffffff, 0x00000100,
 633         0x971c, 0xffffffff, 0x00000100,
 634         0xd0c0, 0xffffffff, 0xff000100,
 635         0x802c, 0xffffffff, 0x40000000,
 636         0x915c, 0xffffffff, 0x00010000,
 637         0x9160, 0xffffffff, 0x00030002,
 638         0x9178, 0xffffffff, 0x00070000,
 639         0x917c, 0xffffffff, 0x00030002,
 640         0x9180, 0xffffffff, 0x00050004,
 641         0x918c, 0xffffffff, 0x00010006,
 642         0x9190, 0xffffffff, 0x00090008,
 643         0x9194, 0xffffffff, 0x00070000,
 644         0x9198, 0xffffffff, 0x00030002,
 645         0x919c, 0xffffffff, 0x00050004,
 646         0x91a8, 0xffffffff, 0x00010006,
 647         0x91ac, 0xffffffff, 0x00090008,
 648         0x91b0, 0xffffffff, 0x00070000,
 649         0x91b4, 0xffffffff, 0x00030002,
 650         0x91b8, 0xffffffff, 0x00050004,
 651         0x91c4, 0xffffffff, 0x00010006,
 652         0x91c8, 0xffffffff, 0x00090008,
 653         0x91cc, 0xffffffff, 0x00070000,
 654         0x91d0, 0xffffffff, 0x00030002,
 655         0x91d4, 0xffffffff, 0x00050004,
 656         0x91e0, 0xffffffff, 0x00010006,
 657         0x91e4, 0xffffffff, 0x00090008,
 658         0x91e8, 0xffffffff, 0x00000000,
 659         0x91ec, 0xffffffff, 0x00070000,
 660         0x91f0, 0xffffffff, 0x00030002,
 661         0x91f4, 0xffffffff, 0x00050004,
 662         0x9200, 0xffffffff, 0x00010006,
 663         0x9204, 0xffffffff, 0x00090008,
 664         0x9208, 0xffffffff, 0x00070000,
 665         0x920c, 0xffffffff, 0x00030002,
 666         0x9210, 0xffffffff, 0x00050004,
 667         0x921c, 0xffffffff, 0x00010006,
 668         0x9220, 0xffffffff, 0x00090008,
 669         0x9224, 0xffffffff, 0x00070000,
 670         0x9228, 0xffffffff, 0x00030002,
 671         0x922c, 0xffffffff, 0x00050004,
 672         0x9238, 0xffffffff, 0x00010006,
 673         0x923c, 0xffffffff, 0x00090008,
 674         0x9240, 0xffffffff, 0x00070000,
 675         0x9244, 0xffffffff, 0x00030002,
 676         0x9248, 0xffffffff, 0x00050004,
 677         0x9254, 0xffffffff, 0x00010006,
 678         0x9258, 0xffffffff, 0x00090008,
 679         0x925c, 0xffffffff, 0x00070000,
 680         0x9260, 0xffffffff, 0x00030002,
 681         0x9264, 0xffffffff, 0x00050004,
 682         0x9270, 0xffffffff, 0x00010006,
 683         0x9274, 0xffffffff, 0x00090008,
 684         0x9278, 0xffffffff, 0x00070000,
 685         0x927c, 0xffffffff, 0x00030002,
 686         0x9280, 0xffffffff, 0x00050004,
 687         0x928c, 0xffffffff, 0x00010006,
 688         0x9290, 0xffffffff, 0x00090008,
 689         0x9294, 0xffffffff, 0x00000000,
 690         0x929c, 0xffffffff, 0x00000001,
 691         0x802c, 0xffffffff, 0xc0000000,
 692         0x977c, 0xffffffff, 0x00000100,
 693         0x3f80, 0xffffffff, 0x00000100,
 694         0xa210, 0xffffffff, 0x00000100,
 695         0xa214, 0xffffffff, 0x00000100,
 696         0x4d8, 0xffffffff, 0x00000100,
 697         0x9784, 0xffffffff, 0x00000100,
 698         0x9698, 0xffffffff, 0x00000100,
 699         0x4d4, 0xffffffff, 0x00000200,
 700         0x30cc, 0xffffffff, 0x00000100,
 701         0x802c, 0xffffffff, 0xc0000000
 702 };
 703
 704 static const u32 supersumo_golden_registers[] =
 705 {
 706         0x5eb4, 0xffffffff, 0x00000002,
 707         0x5c4, 0xffffffff, 0x00000001,
 708         0x7030, 0xffffffff, 0x00000011,
 709         0x7c30, 0xffffffff, 0x00000011,
 710         0x6104, 0x01000300, 0x00000000,
 711         0x5bc0, 0x00300000, 0x00000000,
 712         0x8c04, 0xffffffff, 0x40600060,
 713         0x8c08, 0xffffffff, 0x001c001c,
 714         0x8c20, 0xffffffff, 0x00800080,
 715         0x8c24, 0xffffffff, 0x00800080,
 716         0x8c18, 0xffffffff, 0x20202078,
 717         0x8c1c, 0xffffffff, 0x00001010,
 718         0x918c, 0xffffffff, 0x00010006,
 719         0x91a8, 0xffffffff, 0x00010006,
 720         0x91c4, 0xffffffff, 0x00010006,
 721         0x91e0, 0xffffffff, 0x00010006,
 722         0x9200, 0xffffffff, 0x00010006,
 723         0x9150, 0xffffffff, 0x6e944040,
 724         0x917c, 0xffffffff, 0x00030002,
 725         0x9180, 0xffffffff, 0x00050004,
 726         0x9198, 0xffffffff, 0x00030002,
 727         0x919c, 0xffffffff, 0x00050004,
 728         0x91b4, 0xffffffff, 0x00030002,
 729         0x91b8, 0xffffffff, 0x00050004,
 730         0x91d0, 0xffffffff, 0x00030002,
 731         0x91d4, 0xffffffff, 0x00050004,
 732         0x91f0, 0xffffffff, 0x00030002,
 733         0x91f4, 0xffffffff, 0x00050004,
 734         0x915c, 0xffffffff, 0x00010000,
 735         0x9160, 0xffffffff, 0x00030002,
 736         0x3f90, 0xffff0000, 0xff000000,
 737         0x9178, 0xffffffff, 0x00070000,
 738         0x9194, 0xffffffff, 0x00070000,
 739         0x91b0, 0xffffffff, 0x00070000,
 740         0x91cc, 0xffffffff, 0x00070000,
 741         0x91ec, 0xffffffff, 0x00070000,
 742         0x9148, 0xffff0000, 0xff000000,
 743         0x9190, 0xffffffff, 0x00090008,
 744         0x91ac, 0xffffffff, 0x00090008,
 745         0x91c8, 0xffffffff, 0x00090008,
 746         0x91e4, 0xffffffff, 0x00090008,
 747         0x9204, 0xffffffff, 0x00090008,
 748         0x3f94, 0xffff0000, 0xff000000,
 749         0x914c, 0xffff0000, 0xff000000,
 750         0x929c, 0xffffffff, 0x00000001,
 751         0x8a18, 0xffffffff, 0x00000100,
 752         0x8b28, 0xffffffff, 0x00000100,
 753         0x9144, 0xffffffff, 0x00000100,
 754         0x5644, 0xffffffff, 0x00000100,
 755         0x9b7c, 0xffffffff, 0x00000000,
 756         0x8030, 0xffffffff, 0x0000100a,
 757         0x8a14, 0xffffffff, 0x00000007,
 758         0x8b24, 0xffffffff, 0x00ff0fff,
 759         0x8b10, 0xffffffff, 0x00000000,
 760         0x28a4c, 0x06000000, 0x06000000,
 761         0x4d8, 0xffffffff, 0x00000100,
 762         0x913c, 0xffff000f, 0x0100000a,
 763         0x960c, 0xffffffff, 0x54763210,
 764         0x88c4, 0xffffffff, 0x000000c2,
 765         0x88d4, 0xffffffff, 0x00000010,
 766         0x8974, 0xffffffff, 0x00000000,
 767         0xc78, 0x00000080, 0x00000080,
 768         0x5e78, 0xffffffff, 0x001000f0,
 769         0xd02c, 0xffffffff, 0x08421000,
 770         0xa008, 0xffffffff, 0x00010000,
 771         0x8d00, 0xffffffff, 0x100e4848,
 772         0x8d04, 0xffffffff, 0x00164745,
 773         0x8c00, 0xffffffff, 0xe4000003,
 774         0x8cf0, 0x1fffffff, 0x08e00620,
 775         0x28350, 0xffffffff, 0x00000000,
 776         0x9508, 0xffffffff, 0x00000002
 777 };
 778
 779 static const u32 sumo_golden_registers[] =
 780 {
 781         0x900c, 0x00ffffff, 0x0017071f,
 782         0x8c18, 0xffffffff, 0x10101060,
 783         0x8c1c, 0xffffffff, 0x00001010,
 784         0x8c30, 0x0000000f, 0x00000005,
 785         0x9688, 0x0000000f, 0x00000007
 786 };
 787
 788 static const u32 wrestler_golden_registers[] =
 789 {
 790         0x5eb4, 0xffffffff, 0x00000002,
 791         0x5c4, 0xffffffff, 0x00000001,
 792         0x7030, 0xffffffff, 0x00000011,
 793         0x7c30, 0xffffffff, 0x00000011,
 794         0x6104, 0x01000300, 0x00000000,
 795         0x5bc0, 0x00300000, 0x00000000,
 796         0x918c, 0xffffffff, 0x00010006,
 797         0x91a8, 0xffffffff, 0x00010006,
 798         0x9150, 0xffffffff, 0x6e944040,
 799         0x917c, 0xffffffff, 0x00030002,
 800         0x9198, 0xffffffff, 0x00030002,
 801         0x915c, 0xffffffff, 0x00010000,
 802         0x3f90, 0xffff0000, 0xff000000,
 803         0x9178, 0xffffffff, 0x00070000,
 804         0x9194, 0xffffffff, 0x00070000,
 805         0x9148, 0xffff0000, 0xff000000,
 806         0x9190, 0xffffffff, 0x00090008,
 807         0x91ac, 0xffffffff, 0x00090008,
 808         0x3f94, 0xffff0000, 0xff000000,
 809         0x914c, 0xffff0000, 0xff000000,
 810         0x929c, 0xffffffff, 0x00000001,
 811         0x8a18, 0xffffffff, 0x00000100,
 812         0x8b28, 0xffffffff, 0x00000100,
 813         0x9144, 0xffffffff, 0x00000100,
 814         0x9b7c, 0xffffffff, 0x00000000,
 815         0x8030, 0xffffffff, 0x0000100a,
 816         0x8a14, 0xffffffff, 0x00000001,
 817         0x8b24, 0xffffffff, 0x00ff0fff,
 818         0x8b10, 0xffffffff, 0x00000000,
 819         0x28a4c, 0x06000000, 0x06000000,
 820         0x4d8, 0xffffffff, 0x00000100,
 821         0x913c, 0xffff000f, 0x0100000a,
 822         0x960c, 0xffffffff, 0x54763210,
 823         0x88c4, 0xffffffff, 0x000000c2,
 824         0x88d4, 0xffffffff, 0x00000010,
 825         0x8974, 0xffffffff, 0x00000000,
 826         0xc78, 0x00000080, 0x00000080,
 827         0x5e78, 0xffffffff, 0x001000f0,
 828         0xd02c, 0xffffffff, 0x08421000,
 829         0xa008, 0xffffffff, 0x00010000,
 830         0x8d00, 0xffffffff, 0x100e4848,
 831         0x8d04, 0xffffffff, 0x00164745,
 832         0x8c00, 0xffffffff, 0xe4000003,
 833         0x8cf0, 0x1fffffff, 0x08e00410,
 834         0x28350, 0xffffffff, 0x00000000,
 835         0x9508, 0xffffffff, 0x00000002,
 836         0x900c, 0xffffffff, 0x0017071f,
 837         0x8c18, 0xffffffff, 0x10101060,
 838         0x8c1c, 0xffffffff, 0x00001010
 839 };
 840
 841 static const u32 barts_golden_registers[] =
 842 {
 843         0x5eb4, 0xffffffff, 0x00000002,
 844         0x5e78, 0x8f311ff1, 0x001000f0,
 845         0x3f90, 0xffff0000, 0xff000000,
 846         0x9148, 0xffff0000, 0xff000000,
 847         0x3f94, 0xffff0000, 0xff000000,
 848         0x914c, 0xffff0000, 0xff000000,
 849         0xc78, 0x00000080, 0x00000080,
 850         0xbd4, 0x70073777, 0x00010001,
 851         0xd02c, 0xbfffff1f, 0x08421000,
 852         0xd0b8, 0x03773777, 0x02011003,
 853         0x5bc0, 0x00200000, 0x50100000,
 854         0x98f8, 0x33773777, 0x02011003,
 855         0x98fc, 0xffffffff, 0x76543210,
 856         0x7030, 0x31000311, 0x00000011,
 857         0x2f48, 0x00000007, 0x02011003,
 858         0x6b28, 0x00000010, 0x00000012,
 859         0x7728, 0x00000010, 0x00000012,
 860         0x10328, 0x00000010, 0x00000012,
 861         0x10f28, 0x00000010, 0x00000012,
 862         0x11b28, 0x00000010, 0x00000012,
 863         0x12728, 0x00000010, 0x00000012,
 864         0x240c, 0x000007ff, 0x00000380,
 865         0x8a14, 0xf000001f, 0x00000007,
 866         0x8b24, 0x3fff3fff, 0x00ff0fff,
 867         0x8b10, 0x0000ff0f, 0x00000000,
 868         0x28a4c, 0x07ffffff, 0x06000000,
 869         0x10c, 0x00000001, 0x00010003,
 870         0xa02c, 0xffffffff, 0x0000009b,
 871         0x913c, 0x0000000f, 0x0100000a,
 872         0x8d00, 0xffff7f7f, 0x100e4848,
 873         0x8d04, 0x00ffffff, 0x00164745,
 874         0x8c00, 0xfffc0003, 0xe4000003,
 875         0x8c04, 0xf8ff00ff, 0x40600060,
 876         0x8c08, 0x00ff00ff, 0x001c001c,
 877         0x8cf0, 0x1fff1fff, 0x08e00620,
 878         0x8c20, 0x0fff0fff, 0x00800080,
 879         0x8c24, 0x0fff0fff, 0x00800080,
 880         0x8c18, 0xffffffff, 0x20202078,
 881         0x8c1c, 0x0000ffff, 0x00001010,
 882         0x28350, 0x00000f01, 0x00000000,
 883         0x9508, 0x3700001f, 0x00000002,
 884         0x960c, 0xffffffff, 0x54763210,
 885         0x88c4, 0x001f3ae3, 0x000000c2,
 886         0x88d4, 0x0000001f, 0x00000010,
 887         0x8974, 0xffffffff, 0x00000000
 888 };
 889
 890 static const u32 turks_golden_registers[] =
 891 {
 892         0x5eb4, 0xffffffff, 0x00000002,
 893         0x5e78, 0x8f311ff1, 0x001000f0,
 894         0x8c8, 0x00003000, 0x00001070,
 895         0x8cc, 0x000fffff, 0x00040035,
 896         0x3f90, 0xffff0000, 0xfff00000,
 897         0x9148, 0xffff0000, 0xfff00000,
 898         0x3f94, 0xffff0000, 0xfff00000,
 899         0x914c, 0xffff0000, 0xfff00000,
 900         0xc78, 0x00000080, 0x00000080,
 901         0xbd4, 0x00073007, 0x00010002,
 902         0xd02c, 0xbfffff1f, 0x08421000,
 903         0xd0b8, 0x03773777, 0x02010002,
 904         0x5bc0, 0x00200000, 0x50100000,
 905         0x98f8, 0x33773777, 0x00010002,
 906         0x98fc, 0xffffffff, 0x33221100,
 907         0x7030, 0x31000311, 0x00000011,
 908         0x2f48, 0x33773777, 0x00010002,
 909         0x6b28, 0x00000010, 0x00000012,
 910         0x7728, 0x00000010, 0x00000012,
 911         0x10328, 0x00000010, 0x00000012,
 912         0x10f28, 0x00000010, 0x00000012,
 913         0x11b28, 0x00000010, 0x00000012,
 914         0x12728, 0x00000010, 0x00000012,
 915         0x240c, 0x000007ff, 0x00000380,
 916         0x8a14, 0xf000001f, 0x00000007,
 917         0x8b24, 0x3fff3fff, 0x00ff0fff,
 918         0x8b10, 0x0000ff0f, 0x00000000,
 919         0x28a4c, 0x07ffffff, 0x06000000,
 920         0x10c, 0x00000001, 0x00010003,
 921         0xa02c, 0xffffffff, 0x0000009b,
 922         0x913c, 0x0000000f, 0x0100000a,
 923         0x8d00, 0xffff7f7f, 0x100e4848,
 924         0x8d04, 0x00ffffff, 0x00164745,
 925         0x8c00, 0xfffc0003, 0xe4000003,
 926         0x8c04, 0xf8ff00ff, 0x40600060,
 927         0x8c08, 0x00ff00ff, 0x001c001c,
 928         0x8cf0, 0x1fff1fff, 0x08e00410,
 929         0x8c20, 0x0fff0fff, 0x00800080,
 930         0x8c24, 0x0fff0fff, 0x00800080,
 931         0x8c18, 0xffffffff, 0x20202078,
 932         0x8c1c, 0x0000ffff, 0x00001010,
 933         0x28350, 0x00000f01, 0x00000000,
 934         0x9508, 0x3700001f, 0x00000002,
 935         0x960c, 0xffffffff, 0x54763210,
 936         0x88c4, 0x001f3ae3, 0x000000c2,
 937         0x88d4, 0x0000001f, 0x00000010,
 938         0x8974, 0xffffffff, 0x00000000
 939 };
 940
 941 static const u32 caicos_golden_registers[] =
 942 {
 943         0x5eb4, 0xffffffff, 0x00000002,
 944         0x5e78, 0x8f311ff1, 0x001000f0,
 945         0x8c8, 0x00003420, 0x00001450,
 946         0x8cc, 0x000fffff, 0x00040035,
 947         0x3f90, 0xffff0000, 0xfffc0000,
 948         0x9148, 0xffff0000, 0xfffc0000,
 949         0x3f94, 0xffff0000, 0xfffc0000,
 950         0x914c, 0xffff0000, 0xfffc0000,
 951         0xc78, 0x00000080, 0x00000080,
 952         0xbd4, 0x00073007, 0x00010001,
 953         0xd02c, 0xbfffff1f, 0x08421000,
 954         0xd0b8, 0x03773777, 0x02010001,
 955         0x5bc0, 0x00200000, 0x50100000,
 956         0x98f8, 0x33773777, 0x02010001,
 957         0x98fc, 0xffffffff, 0x33221100,
 958         0x7030, 0x31000311, 0x00000011,
 959         0x2f48, 0x33773777, 0x02010001,
 960         0x6b28, 0x00000010, 0x00000012,
 961         0x7728, 0x00000010, 0x00000012,
 962         0x10328, 0x00000010, 0x00000012,
 963         0x10f28, 0x00000010, 0x00000012,
 964         0x11b28, 0x00000010, 0x00000012,
 965         0x12728, 0x00000010, 0x00000012,
 966         0x240c, 0x000007ff, 0x00000380,
 967         0x8a14, 0xf000001f, 0x00000001,
 968         0x8b24, 0x3fff3fff, 0x00ff0fff,
 969         0x8b10, 0x0000ff0f, 0x00000000,
 970         0x28a4c, 0x07ffffff, 0x06000000,
 971         0x10c, 0x00000001, 0x00010003,
 972         0xa02c, 0xffffffff, 0x0000009b,
 973         0x913c, 0x0000000f, 0x0100000a,
 974         0x8d00, 0xffff7f7f, 0x100e4848,
 975         0x8d04, 0x00ffffff, 0x00164745,
 976         0x8c00, 0xfffc0003, 0xe4000003,
 977         0x8c04, 0xf8ff00ff, 0x40600060,
 978         0x8c08, 0x00ff00ff, 0x001c001c,
 979         0x8cf0, 0x1fff1fff, 0x08e00410,
 980         0x8c20, 0x0fff0fff, 0x00800080,
 981         0x8c24, 0x0fff0fff, 0x00800080,
 982         0x8c18, 0xffffffff, 0x20202078,
 983         0x8c1c, 0x0000ffff, 0x00001010,
 984         0x28350, 0x00000f01, 0x00000000,
 985         0x9508, 0x3700001f, 0x00000002,
 986         0x960c, 0xffffffff, 0x54763210,
 987         0x88c4, 0x001f3ae3, 0x000000c2,
 988         0x88d4, 0x0000001f, 0x00000010,
 989         0x8974, 0xffffffff, 0x00000000
 990 };
 991
 992 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 993 {
 994         switch (rdev->family) {
 995         case CHIP_CYPRESS:
 996         case CHIP_HEMLOCK:
 997                 radeon_program_register_sequence(rdev,
 998                                                  evergreen_golden_registers,
 999                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000                 radeon_program_register_sequence(rdev,
1001                                                  evergreen_golden_registers2,
1002                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003                 radeon_program_register_sequence(rdev,
1004                                                  cypress_mgcg_init,
1005                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006                 break;
1007         case CHIP_JUNIPER:
1008                 radeon_program_register_sequence(rdev,
1009                                                  evergreen_golden_registers,
1010                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011                 radeon_program_register_sequence(rdev,
1012                                                  evergreen_golden_registers2,
1013                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014                 radeon_program_register_sequence(rdev,
1015                                                  juniper_mgcg_init,
1016                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017                 break;
1018         case CHIP_REDWOOD:
1019                 radeon_program_register_sequence(rdev,
1020                                                  evergreen_golden_registers,
1021                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022                 radeon_program_register_sequence(rdev,
1023                                                  evergreen_golden_registers2,
1024                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025                 radeon_program_register_sequence(rdev,
1026                                                  redwood_mgcg_init,
1027                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028                 break;
1029         case CHIP_CEDAR:
1030                 radeon_program_register_sequence(rdev,
1031                                                  cedar_golden_registers,
1032                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1033                 radeon_program_register_sequence(rdev,
1034                                                  evergreen_golden_registers2,
1035                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036                 radeon_program_register_sequence(rdev,
1037                                                  cedar_mgcg_init,
1038                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039                 break;
1040         case CHIP_PALM:
1041                 radeon_program_register_sequence(rdev,
1042                                                  wrestler_golden_registers,
1043                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044                 break;
1045         case CHIP_SUMO:
1046                 radeon_program_register_sequence(rdev,
1047                                                  supersumo_golden_registers,
1048                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049                 break;
1050         case CHIP_SUMO2:
1051                 radeon_program_register_sequence(rdev,
1052                                                  supersumo_golden_registers,
1053                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054                 radeon_program_register_sequence(rdev,
1055                                                  sumo_golden_registers,
1056                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1057                 break;
1058         case CHIP_BARTS:
1059                 radeon_program_register_sequence(rdev,
1060                                                  barts_golden_registers,
1061                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1062                 break;
1063         case CHIP_TURKS:
1064                 radeon_program_register_sequence(rdev,
1065                                                  turks_golden_registers,
1066                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1067                 break;
1068         case CHIP_CAICOS:
1069                 radeon_program_register_sequence(rdev,
1070                                                  caicos_golden_registers,
1071                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1072                 break;
1073         default:
1074                 break;
1075         }
1076 }
1077
1078 /**
1079  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080  *
1081  * @rdev: radeon_device pointer
1082  * @reg: register offset in bytes
1083  * @val: register value
1084  *
1085  * Returns 0 for success or -EINVAL for an invalid register
1086  *
1087  */
1088 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089                                         u32 reg, u32 *val)
1090 {
1091         switch (reg) {
1092         case GRBM_STATUS:
1093         case GRBM_STATUS_SE0:
1094         case GRBM_STATUS_SE1:
1095         case SRBM_STATUS:
1096         case SRBM_STATUS2:
1097         case DMA_STATUS_REG:
1098         case UVD_STATUS:
1099                 *val = RREG32(reg);
1100                 return 0;
1101         default:
1102                 return -EINVAL;
1103         }
1104 }
1105
1106 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107                              unsigned *bankh, unsigned *mtaspect,
1108                              unsigned *tile_split)
1109 {
1110         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114         switch (*bankw) {
1115         default:
1116         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120         }
1121         switch (*bankh) {
1122         default:
1123         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127         }
1128         switch (*mtaspect) {
1129         default:
1130         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134         }
1135 }
1136
1137 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138                               u32 cntl_reg, u32 status_reg)
1139 {
1140         int r, i;
1141         struct atom_clock_dividers dividers;
1142
1143         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1144                                            clock, false, &dividers);
1145         if (r)
1146                 return r;
1147
1148         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150         for (i = 0; i < 100; i++) {
1151                 if (RREG32(status_reg) & DCLK_STATUS)
1152                         break;
1153                 mdelay(10);
1154         }
1155         if (i == 100)
1156                 return -ETIMEDOUT;
1157
1158         return 0;
1159 }
1160
1161 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162 {
1163         int r = 0;
1164         u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167         if (r)
1168                 goto done;
1169         cg_scratch &= 0xffff0000;
1170         cg_scratch |= vclk / 100; /* Mhz */
1171
1172         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173         if (r)
1174                 goto done;
1175         cg_scratch &= 0x0000ffff;
1176         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178 done:
1179         WREG32(CG_SCRATCH1, cg_scratch);
1180
1181         return r;
1182 }
1183
1184 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185 {
1186         /* start off with something large */
1187         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1188         int r;
1189
1190         /* bypass vclk and dclk with bclk */
1191         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195         /* put PLL in bypass mode */
1196         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198         if (!vclk || !dclk) {
1199                 /* keep the Bypass mode, put PLL to sleep */
1200                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201                 return 0;
1202         }
1203
1204         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205                                           16384, 0x03FFFFFF, 0, 128, 5,
1206                                           &fb_div, &vclk_div, &dclk_div);
1207         if (r)
1208                 return r;
1209
1210         /* set VCO_MODE to 1 */
1211         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213         /* toggle UPLL_SLEEP to 1 then back to 0 */
1214         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217         /* deassert UPLL_RESET */
1218         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220         mdelay(1);
1221
1222         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1223         if (r)
1224                 return r;
1225
1226         /* assert UPLL_RESET again */
1227         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229         /* disable spread spectrum. */
1230         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232         /* set feedback divider */
1233         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1234
1235         /* set ref divider to 0 */
1236         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
1238         if (fb_div < 307200)
1239                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240         else
1241                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243         /* set PDIV_A and PDIV_B */
1244         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1245                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1246                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248         /* give the PLL some time to settle */
1249         mdelay(15);
1250
1251         /* deassert PLL_RESET */
1252         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254         mdelay(15);
1255
1256         /* switch from bypass mode to normal mode */
1257         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
1259         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1260         if (r)
1261                 return r;
1262
1263         /* switch VCLK and DCLK selection */
1264         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268         mdelay(100);
1269
1270         return 0;
1271 }
1272
1273 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274 {
1275         int readrq;
1276         u16 v;
1277
1278         readrq = pcie_get_readrq(rdev->pdev);
1279         v = ffs(readrq) - 8;
1280         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281          * to avoid hangs or perfomance issues
1282          */
1283         if ((v == 0) || (v == 6) || (v == 7))
1284                 pcie_set_readrq(rdev->pdev, 512);
1285 }
1286
1287 void dce4_program_fmt(struct drm_encoder *encoder)
1288 {
1289         struct drm_device *dev = encoder->dev;
1290         struct radeon_device *rdev = dev->dev_private;
1291         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294         int bpc = 0;
1295         u32 tmp = 0;
1296         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1297
1298         if (connector) {
1299                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1300                 bpc = radeon_get_monitor_bpc(connector);
1301                 dither = radeon_connector->dither;
1302         }
1303
1304         /* LVDS/eDP FMT is set up by atom */
1305         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306                 return;
1307
1308         /* not needed for analog */
1309         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311                 return;
1312
1313         if (bpc == 0)
1314                 return;
1315
1316         switch (bpc) {
1317         case 6:
1318                 if (dither == RADEON_FMT_DITHER_ENABLE)
1319                         /* XXX sort out optimal dither settings */
1320                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321                                 FMT_SPATIAL_DITHER_EN);
1322                 else
1323                         tmp |= FMT_TRUNCATE_EN;
1324                 break;
1325         case 8:
1326                 if (dither == RADEON_FMT_DITHER_ENABLE)
1327                         /* XXX sort out optimal dither settings */
1328                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329                                 FMT_RGB_RANDOM_ENABLE |
1330                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331                 else
1332                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333                 break;
1334         case 10:
1335         default:
1336                 /* not needed */
1337                 break;
1338         }
1339
1340         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341 }
1342
1343 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344 {
1345         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346                 return true;
1347         else
1348                 return false;
1349 }
1350
1351 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352 {
1353         u32 pos1, pos2;
1354
1355         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358         if (pos1 != pos2)
1359                 return true;
1360         else
1361                 return false;
1362 }
1363
1364 /**
1365  * dce4_wait_for_vblank - vblank wait asic callback.
1366  *
1367  * @rdev: radeon_device pointer
1368  * @crtc: crtc to wait for vblank on
1369  *
1370  * Wait for vblank on the requested crtc (evergreen+).
1371  */
1372 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373 {
1374         unsigned i = 0;
1375
1376         if (crtc >= rdev->num_crtc)
1377                 return;
1378
1379         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380                 return;
1381
1382         /* depending on when we hit vblank, we may be close to active; if so,
1383          * wait for another frame.
1384          */
1385         while (dce4_is_in_vblank(rdev, crtc)) {
1386                 if (i++ % 100 == 0) {
1387                         if (!dce4_is_counter_moving(rdev, crtc))
1388                                 break;
1389                 }
1390         }
1391
1392         while (!dce4_is_in_vblank(rdev, crtc)) {
1393                 if (i++ % 100 == 0) {
1394                         if (!dce4_is_counter_moving(rdev, crtc))
1395                                 break;
1396                 }
1397         }
1398 }
1399
1400 /**
1401  * evergreen_page_flip - pageflip callback.
1402  *
1403  * @rdev: radeon_device pointer
1404  * @crtc_id: crtc to cleanup pageflip on
1405  * @crtc_base: new address of the crtc (GPU MC address)
1406  *
1407  * Does the actual pageflip (evergreen+).
1408  * During vblank we take the crtc lock and wait for the update_pending
1409  * bit to go high, when it does, we release the lock, and allow the
1410  * double buffered update to take place.
1411  * Returns the current update pending status.
1412  */
1413 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1414 {
1415         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1416         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1417         int i;
1418
1419         /* Lock the graphics update lock */
1420         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1421         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1422
1423         /* update the scanout addresses */
1424         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1425                upper_32_bits(crtc_base));
1426         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1427                (u32)crtc_base);
1428
1429         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430                upper_32_bits(crtc_base));
1431         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432                (u32)crtc_base);
1433
1434         /* Wait for update_pending to go high. */
1435         for (i = 0; i < rdev->usec_timeout; i++) {
1436                 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1437                         break;
1438                 udelay(1);
1439         }
1440         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1441
1442         /* Unlock the lock, so double-buffering can take place inside vblank */
1443         tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1444         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1445 }
1446
1447 /**
1448  * evergreen_page_flip_pending - check if page flip is still pending
1449  *
1450  * @rdev: radeon_device pointer
1451  * @crtc_id: crtc to check
1452  *
1453  * Returns the current update pending status.
1454  */
1455 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1456 {
1457         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1458
1459         /* Return current update_pending status: */
1460         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1461                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1462 }
1463
1464 /* get temperature in millidegrees */
1465 int evergreen_get_temp(struct radeon_device *rdev)
1466 {
1467         u32 temp, toffset;
1468         int actual_temp = 0;
1469
1470         if (rdev->family == CHIP_JUNIPER) {
1471                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1472                         TOFFSET_SHIFT;
1473                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1474                         TS0_ADC_DOUT_SHIFT;
1475
1476                 if (toffset & 0x100)
1477                         actual_temp = temp / 2 - (0x200 - toffset);
1478                 else
1479                         actual_temp = temp / 2 + toffset;
1480
1481                 actual_temp = actual_temp * 1000;
1482
1483         } else {
1484                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1485                         ASIC_T_SHIFT;
1486
1487                 if (temp & 0x400)
1488                         actual_temp = -256;
1489                 else if (temp & 0x200)
1490                         actual_temp = 255;
1491                 else if (temp & 0x100) {
1492                         actual_temp = temp & 0x1ff;
1493                         actual_temp |= ~0x1ff;
1494                 } else
1495                         actual_temp = temp & 0xff;
1496
1497                 actual_temp = (actual_temp * 1000) / 2;
1498         }
1499
1500         return actual_temp;
1501 }
1502
1503 int sumo_get_temp(struct radeon_device *rdev)
1504 {
1505         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1506         int actual_temp = temp - 49;
1507
1508         return actual_temp * 1000;
1509 }
1510
1511 /**
1512  * sumo_pm_init_profile - Initialize power profiles callback.
1513  *
1514  * @rdev: radeon_device pointer
1515  *
1516  * Initialize the power states used in profile mode
1517  * (sumo, trinity, SI).
1518  * Used for profile mode only.
1519  */
1520 void sumo_pm_init_profile(struct radeon_device *rdev)
1521 {
1522         int idx;
1523
1524         /* default */
1525         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1526         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1527         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1528         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1529
1530         /* low,mid sh/mh */
1531         if (rdev->flags & RADEON_IS_MOBILITY)
1532                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1533         else
1534                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1535
1536         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1537         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1538         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1539         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1540
1541         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1542         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1543         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1544         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1545
1546         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1547         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1548         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1549         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1550
1551         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1552         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1553         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1554         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1555
1556         /* high sh/mh */
1557         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1558         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1559         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1560         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1561         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1562                 rdev->pm.power_state[idx].num_clock_modes - 1;
1563
1564         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1565         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1566         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1567         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1568                 rdev->pm.power_state[idx].num_clock_modes - 1;
1569 }
1570
1571 /**
1572  * btc_pm_init_profile - Initialize power profiles callback.
1573  *
1574  * @rdev: radeon_device pointer
1575  *
1576  * Initialize the power states used in profile mode
1577  * (BTC, cayman).
1578  * Used for profile mode only.
1579  */
1580 void btc_pm_init_profile(struct radeon_device *rdev)
1581 {
1582         int idx;
1583
1584         /* default */
1585         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1586         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1587         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1588         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1589         /* starting with BTC, there is one state that is used for both
1590          * MH and SH.  Difference is that we always use the high clock index for
1591          * mclk.
1592          */
1593         if (rdev->flags & RADEON_IS_MOBILITY)
1594                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1595         else
1596                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1597         /* low sh */
1598         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1599         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1600         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1601         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1602         /* mid sh */
1603         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1604         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1605         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1606         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1607         /* high sh */
1608         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1609         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1610         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1611         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1612         /* low mh */
1613         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1614         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1615         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1616         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1617         /* mid mh */
1618         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1619         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1620         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1621         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1622         /* high mh */
1623         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1624         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1625         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1626         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1627 }
1628
1629 /**
1630  * evergreen_pm_misc - set additional pm hw parameters callback.
1631  *
1632  * @rdev: radeon_device pointer
1633  *
1634  * Set non-clock parameters associated with a power state
1635  * (voltage, etc.) (evergreen+).
1636  */
1637 void evergreen_pm_misc(struct radeon_device *rdev)
1638 {
1639         int req_ps_idx = rdev->pm.requested_power_state_index;
1640         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1641         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1642         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1643
1644         if (voltage->type == VOLTAGE_SW) {
1645                 /* 0xff0x are flags rather then an actual voltage */
1646                 if ((voltage->voltage & 0xff00) == 0xff00)
1647                         return;
1648                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1649                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1650                         rdev->pm.current_vddc = voltage->voltage;
1651                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1652                 }
1653
1654                 /* starting with BTC, there is one state that is used for both
1655                  * MH and SH.  Difference is that we always use the high clock index for
1656                  * mclk and vddci.
1657                  */
1658                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1659                     (rdev->family >= CHIP_BARTS) &&
1660                     rdev->pm.active_crtc_count &&
1661                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1662                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1663                         voltage = &rdev->pm.power_state[req_ps_idx].
1664                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1665
1666                 /* 0xff0x are flags rather then an actual voltage */
1667                 if ((voltage->vddci & 0xff00) == 0xff00)
1668                         return;
1669                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1670                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1671                         rdev->pm.current_vddci = voltage->vddci;
1672                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1673                 }
1674         }
1675 }
1676
1677 /**
1678  * evergreen_pm_prepare - pre-power state change callback.
1679  *
1680  * @rdev: radeon_device pointer
1681  *
1682  * Prepare for a power state change (evergreen+).
1683  */
1684 void evergreen_pm_prepare(struct radeon_device *rdev)
1685 {
1686         struct drm_device *ddev = rdev->ddev;
1687         struct drm_crtc *crtc;
1688         struct radeon_crtc *radeon_crtc;
1689         u32 tmp;
1690
1691         /* disable any active CRTCs */
1692         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1693                 radeon_crtc = to_radeon_crtc(crtc);
1694                 if (radeon_crtc->enabled) {
1695                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1696                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1697                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1698                 }
1699         }
1700 }
1701
1702 /**
1703  * evergreen_pm_finish - post-power state change callback.
1704  *
1705  * @rdev: radeon_device pointer
1706  *
1707  * Clean up after a power state change (evergreen+).
1708  */
1709 void evergreen_pm_finish(struct radeon_device *rdev)
1710 {
1711         struct drm_device *ddev = rdev->ddev;
1712         struct drm_crtc *crtc;
1713         struct radeon_crtc *radeon_crtc;
1714         u32 tmp;
1715
1716         /* enable any active CRTCs */
1717         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1718                 radeon_crtc = to_radeon_crtc(crtc);
1719                 if (radeon_crtc->enabled) {
1720                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1721                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1722                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1723                 }
1724         }
1725 }
1726
1727 /**
1728  * evergreen_hpd_sense - hpd sense callback.
1729  *
1730  * @rdev: radeon_device pointer
1731  * @hpd: hpd (hotplug detect) pin
1732  *
1733  * Checks if a digital monitor is connected (evergreen+).
1734  * Returns true if connected, false if not connected.
1735  */
1736 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1737 {
1738         bool connected = false;
1739
1740         switch (hpd) {
1741         case RADEON_HPD_1:
1742                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1743                         connected = true;
1744                 break;
1745         case RADEON_HPD_2:
1746                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1747                         connected = true;
1748                 break;
1749         case RADEON_HPD_3:
1750                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1751                         connected = true;
1752                 break;
1753         case RADEON_HPD_4:
1754                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1755                         connected = true;
1756                 break;
1757         case RADEON_HPD_5:
1758                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1759                         connected = true;
1760                 break;
1761         case RADEON_HPD_6:
1762                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1763                         connected = true;
1764                 break;
1765         default:
1766                 break;
1767         }
1768
1769         return connected;
1770 }
1771
1772 /**
1773  * evergreen_hpd_set_polarity - hpd set polarity callback.
1774  *
1775  * @rdev: radeon_device pointer
1776  * @hpd: hpd (hotplug detect) pin
1777  *
1778  * Set the polarity of the hpd pin (evergreen+).
1779  */
1780 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1781                                 enum radeon_hpd_id hpd)
1782 {
1783         u32 tmp;
1784         bool connected = evergreen_hpd_sense(rdev, hpd);
1785
1786         switch (hpd) {
1787         case RADEON_HPD_1:
1788                 tmp = RREG32(DC_HPD1_INT_CONTROL);
1789                 if (connected)
1790                         tmp &= ~DC_HPDx_INT_POLARITY;
1791                 else
1792                         tmp |= DC_HPDx_INT_POLARITY;
1793                 WREG32(DC_HPD1_INT_CONTROL, tmp);
1794                 break;
1795         case RADEON_HPD_2:
1796                 tmp = RREG32(DC_HPD2_INT_CONTROL);
1797                 if (connected)
1798                         tmp &= ~DC_HPDx_INT_POLARITY;
1799                 else
1800                         tmp |= DC_HPDx_INT_POLARITY;
1801                 WREG32(DC_HPD2_INT_CONTROL, tmp);
1802                 break;
1803         case RADEON_HPD_3:
1804                 tmp = RREG32(DC_HPD3_INT_CONTROL);
1805                 if (connected)
1806                         tmp &= ~DC_HPDx_INT_POLARITY;
1807                 else
1808                         tmp |= DC_HPDx_INT_POLARITY;
1809                 WREG32(DC_HPD3_INT_CONTROL, tmp);
1810                 break;
1811         case RADEON_HPD_4:
1812                 tmp = RREG32(DC_HPD4_INT_CONTROL);
1813                 if (connected)
1814                         tmp &= ~DC_HPDx_INT_POLARITY;
1815                 else
1816                         tmp |= DC_HPDx_INT_POLARITY;
1817                 WREG32(DC_HPD4_INT_CONTROL, tmp);
1818                 break;
1819         case RADEON_HPD_5:
1820                 tmp = RREG32(DC_HPD5_INT_CONTROL);
1821                 if (connected)
1822                         tmp &= ~DC_HPDx_INT_POLARITY;
1823                 else
1824                         tmp |= DC_HPDx_INT_POLARITY;
1825                 WREG32(DC_HPD5_INT_CONTROL, tmp);
1826                         break;
1827         case RADEON_HPD_6:
1828                 tmp = RREG32(DC_HPD6_INT_CONTROL);
1829                 if (connected)
1830                         tmp &= ~DC_HPDx_INT_POLARITY;
1831                 else
1832                         tmp |= DC_HPDx_INT_POLARITY;
1833                 WREG32(DC_HPD6_INT_CONTROL, tmp);
1834                 break;
1835         default:
1836                 break;
1837         }
1838 }
1839
1840 /**
1841  * evergreen_hpd_init - hpd setup callback.
1842  *
1843  * @rdev: radeon_device pointer
1844  *
1845  * Setup the hpd pins used by the card (evergreen+).
1846  * Enable the pin, set the polarity, and enable the hpd interrupts.
1847  */
1848 void evergreen_hpd_init(struct radeon_device *rdev)
1849 {
1850         struct drm_device *dev = rdev->ddev;
1851         struct drm_connector *connector;
1852         unsigned enabled = 0;
1853         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1854                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1855
1856         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1857                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1858
1859                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1860                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1861                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1862                          * aux dp channel on imac and help (but not completely fix)
1863                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1864                          * also avoid interrupt storms during dpms.
1865                          */
1866                         continue;
1867                 }
1868                 switch (radeon_connector->hpd.hpd) {
1869                 case RADEON_HPD_1:
1870                         WREG32(DC_HPD1_CONTROL, tmp);
1871                         break;
1872                 case RADEON_HPD_2:
1873                         WREG32(DC_HPD2_CONTROL, tmp);
1874                         break;
1875                 case RADEON_HPD_3:
1876                         WREG32(DC_HPD3_CONTROL, tmp);
1877                         break;
1878                 case RADEON_HPD_4:
1879                         WREG32(DC_HPD4_CONTROL, tmp);
1880                         break;
1881                 case RADEON_HPD_5:
1882                         WREG32(DC_HPD5_CONTROL, tmp);
1883                         break;
1884                 case RADEON_HPD_6:
1885                         WREG32(DC_HPD6_CONTROL, tmp);
1886                         break;
1887                 default:
1888                         break;
1889                 }
1890                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1891                 enabled |= 1 << radeon_connector->hpd.hpd;
1892         }
1893         radeon_irq_kms_enable_hpd(rdev, enabled);
1894 }
1895
1896 /**
1897  * evergreen_hpd_fini - hpd tear down callback.
1898  *
1899  * @rdev: radeon_device pointer
1900  *
1901  * Tear down the hpd pins used by the card (evergreen+).
1902  * Disable the hpd interrupts.
1903  */
1904 void evergreen_hpd_fini(struct radeon_device *rdev)
1905 {
1906         struct drm_device *dev = rdev->ddev;
1907         struct drm_connector *connector;
1908         unsigned disabled = 0;
1909
1910         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1911                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1912                 switch (radeon_connector->hpd.hpd) {
1913                 case RADEON_HPD_1:
1914                         WREG32(DC_HPD1_CONTROL, 0);
1915                         break;
1916                 case RADEON_HPD_2:
1917                         WREG32(DC_HPD2_CONTROL, 0);
1918                         break;
1919                 case RADEON_HPD_3:
1920                         WREG32(DC_HPD3_CONTROL, 0);
1921                         break;
1922                 case RADEON_HPD_4:
1923                         WREG32(DC_HPD4_CONTROL, 0);
1924                         break;
1925                 case RADEON_HPD_5:
1926                         WREG32(DC_HPD5_CONTROL, 0);
1927                         break;
1928                 case RADEON_HPD_6:
1929                         WREG32(DC_HPD6_CONTROL, 0);
1930                         break;
1931                 default:
1932                         break;
1933                 }
1934                 disabled |= 1 << radeon_connector->hpd.hpd;
1935         }
1936         radeon_irq_kms_disable_hpd(rdev, disabled);
1937 }
1938
1939 /* watermark setup */
1940
1941 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1942                                         struct radeon_crtc *radeon_crtc,
1943                                         struct drm_display_mode *mode,
1944                                         struct drm_display_mode *other_mode)
1945 {
1946         u32 tmp, buffer_alloc, i;
1947         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1948         /*
1949          * Line Buffer Setup
1950          * There are 3 line buffers, each one shared by 2 display controllers.
1951          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1952          * the display controllers.  The paritioning is done via one of four
1953          * preset allocations specified in bits 2:0:
1954          * first display controller
1955          *  0 - first half of lb (3840 * 2)
1956          *  1 - first 3/4 of lb (5760 * 2)
1957          *  2 - whole lb (7680 * 2), other crtc must be disabled
1958          *  3 - first 1/4 of lb (1920 * 2)
1959          * second display controller
1960          *  4 - second half of lb (3840 * 2)
1961          *  5 - second 3/4 of lb (5760 * 2)
1962          *  6 - whole lb (7680 * 2), other crtc must be disabled
1963          *  7 - last 1/4 of lb (1920 * 2)
1964          */
1965         /* this can get tricky if we have two large displays on a paired group
1966          * of crtcs.  Ideally for multiple large displays we'd assign them to
1967          * non-linked crtcs for maximum line buffer allocation.
1968          */
1969         if (radeon_crtc->base.enabled && mode) {
1970                 if (other_mode) {
1971                         tmp = 0; /* 1/2 */
1972                         buffer_alloc = 1;
1973                 } else {
1974                         tmp = 2; /* whole */
1975                         buffer_alloc = 2;
1976                 }
1977         } else {
1978                 tmp = 0;
1979                 buffer_alloc = 0;
1980         }
1981
1982         /* second controller of the pair uses second half of the lb */
1983         if (radeon_crtc->crtc_id % 2)
1984                 tmp += 4;
1985         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1986
1987         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1988                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1989                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1990                 for (i = 0; i < rdev->usec_timeout; i++) {
1991                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1992                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1993                                 break;
1994                         udelay(1);
1995                 }
1996         }
1997
1998         if (radeon_crtc->base.enabled && mode) {
1999                 switch (tmp) {
2000                 case 0:
2001                 case 4:
2002                 default:
2003                         if (ASIC_IS_DCE5(rdev))
2004                                 return 4096 * 2;
2005                         else
2006                                 return 3840 * 2;
2007                 case 1:
2008                 case 5:
2009                         if (ASIC_IS_DCE5(rdev))
2010                                 return 6144 * 2;
2011                         else
2012                                 return 5760 * 2;
2013                 case 2:
2014                 case 6:
2015                         if (ASIC_IS_DCE5(rdev))
2016                                 return 8192 * 2;
2017                         else
2018                                 return 7680 * 2;
2019                 case 3:
2020                 case 7:
2021                         if (ASIC_IS_DCE5(rdev))
2022                                 return 2048 * 2;
2023                         else
2024                                 return 1920 * 2;
2025                 }
2026         }
2027
2028         /* controller not enabled, so no lb used */
2029         return 0;
2030 }
2031
2032 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
2033 {
2034         u32 tmp = RREG32(MC_SHARED_CHMAP);
2035
2036         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2037         case 0:
2038         default:
2039                 return 1;
2040         case 1:
2041                 return 2;
2042         case 2:
2043                 return 4;
2044         case 3:
2045                 return 8;
2046         }
2047 }
2048
2049 struct evergreen_wm_params {
2050         u32 dram_channels; /* number of dram channels */
2051         u32 yclk;          /* bandwidth per dram data pin in kHz */
2052         u32 sclk;          /* engine clock in kHz */
2053         u32 disp_clk;      /* display clock in kHz */
2054         u32 src_width;     /* viewport width */
2055         u32 active_time;   /* active display time in ns */
2056         u32 blank_time;    /* blank time in ns */
2057         bool interlaced;    /* mode is interlaced */
2058         fixed20_12 vsc;    /* vertical scale ratio */
2059         u32 num_heads;     /* number of active crtcs */
2060         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2061         u32 lb_size;       /* line buffer allocated to pipe */
2062         u32 vtaps;         /* vertical scaler taps */
2063 };
2064
2065 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2066 {
2067         /* Calculate DRAM Bandwidth and the part allocated to display. */
2068         fixed20_12 dram_efficiency; /* 0.7 */
2069         fixed20_12 yclk, dram_channels, bandwidth;
2070         fixed20_12 a;
2071
2072         a.full = dfixed_const(1000);
2073         yclk.full = dfixed_const(wm->yclk);
2074         yclk.full = dfixed_div(yclk, a);
2075         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2076         a.full = dfixed_const(10);
2077         dram_efficiency.full = dfixed_const(7);
2078         dram_efficiency.full = dfixed_div(dram_efficiency, a);
2079         bandwidth.full = dfixed_mul(dram_channels, yclk);
2080         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2081
2082         return dfixed_trunc(bandwidth);
2083 }
2084
2085 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2086 {
2087         /* Calculate DRAM Bandwidth and the part allocated to display. */
2088         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2089         fixed20_12 yclk, dram_channels, bandwidth;
2090         fixed20_12 a;
2091
2092         a.full = dfixed_const(1000);
2093         yclk.full = dfixed_const(wm->yclk);
2094         yclk.full = dfixed_div(yclk, a);
2095         dram_channels.full = dfixed_const(wm->dram_channels * 4);
2096         a.full = dfixed_const(10);
2097         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2098         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2099         bandwidth.full = dfixed_mul(dram_channels, yclk);
2100         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2101
2102         return dfixed_trunc(bandwidth);
2103 }
2104
2105 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2106 {
2107         /* Calculate the display Data return Bandwidth */
2108         fixed20_12 return_efficiency; /* 0.8 */
2109         fixed20_12 sclk, bandwidth;
2110         fixed20_12 a;
2111
2112         a.full = dfixed_const(1000);
2113         sclk.full = dfixed_const(wm->sclk);
2114         sclk.full = dfixed_div(sclk, a);
2115         a.full = dfixed_const(10);
2116         return_efficiency.full = dfixed_const(8);
2117         return_efficiency.full = dfixed_div(return_efficiency, a);
2118         a.full = dfixed_const(32);
2119         bandwidth.full = dfixed_mul(a, sclk);
2120         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2121
2122         return dfixed_trunc(bandwidth);
2123 }
2124
2125 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2126 {
2127         /* Calculate the DMIF Request Bandwidth */
2128         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2129         fixed20_12 disp_clk, bandwidth;
2130         fixed20_12 a;
2131
2132         a.full = dfixed_const(1000);
2133         disp_clk.full = dfixed_const(wm->disp_clk);
2134         disp_clk.full = dfixed_div(disp_clk, a);
2135         a.full = dfixed_const(10);
2136         disp_clk_request_efficiency.full = dfixed_const(8);
2137         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2138         a.full = dfixed_const(32);
2139         bandwidth.full = dfixed_mul(a, disp_clk);
2140         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2141
2142         return dfixed_trunc(bandwidth);
2143 }
2144
2145 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2146 {
2147         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2148         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2149         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2150         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2151
2152         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2153 }
2154
2155 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2156 {
2157         /* Calculate the display mode Average Bandwidth
2158          * DisplayMode should contain the source and destination dimensions,
2159          * timing, etc.
2160          */
2161         fixed20_12 bpp;
2162         fixed20_12 line_time;
2163         fixed20_12 src_width;
2164         fixed20_12 bandwidth;
2165         fixed20_12 a;
2166
2167         a.full = dfixed_const(1000);
2168         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2169         line_time.full = dfixed_div(line_time, a);
2170         bpp.full = dfixed_const(wm->bytes_per_pixel);
2171         src_width.full = dfixed_const(wm->src_width);
2172         bandwidth.full = dfixed_mul(src_width, bpp);
2173         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2174         bandwidth.full = dfixed_div(bandwidth, line_time);
2175
2176         return dfixed_trunc(bandwidth);
2177 }
2178
2179 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2180 {
2181         /* First calcualte the latency in ns */
2182         u32 mc_latency = 2000; /* 2000 ns. */
2183         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2184         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2185         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2186         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2187         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2188                 (wm->num_heads * cursor_line_pair_return_time);
2189         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2190         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2191         fixed20_12 a, b, c;
2192
2193         if (wm->num_heads == 0)
2194                 return 0;
2195
2196         a.full = dfixed_const(2);
2197         b.full = dfixed_const(1);
2198         if ((wm->vsc.full > a.full) ||
2199             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2200             (wm->vtaps >= 5) ||
2201             ((wm->vsc.full >= a.full) && wm->interlaced))
2202                 max_src_lines_per_dst_line = 4;
2203         else
2204                 max_src_lines_per_dst_line = 2;
2205
2206         a.full = dfixed_const(available_bandwidth);
2207         b.full = dfixed_const(wm->num_heads);
2208         a.full = dfixed_div(a, b);
2209
2210         b.full = dfixed_const(1000);
2211         c.full = dfixed_const(wm->disp_clk);
2212         b.full = dfixed_div(c, b);
2213         c.full = dfixed_const(wm->bytes_per_pixel);
2214         b.full = dfixed_mul(b, c);
2215
2216         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2217
2218         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2219         b.full = dfixed_const(1000);
2220         c.full = dfixed_const(lb_fill_bw);
2221         b.full = dfixed_div(c, b);
2222         a.full = dfixed_div(a, b);
2223         line_fill_time = dfixed_trunc(a);
2224
2225         if (line_fill_time < wm->active_time)
2226                 return latency;
2227         else
2228                 return latency + (line_fill_time - wm->active_time);
2229
2230 }
2231
2232 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2233 {
2234         if (evergreen_average_bandwidth(wm) <=
2235             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2236                 return true;
2237         else
2238                 return false;
2239 };
2240
2241 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2242 {
2243         if (evergreen_average_bandwidth(wm) <=
2244             (evergreen_available_bandwidth(wm) / wm->num_heads))
2245                 return true;
2246         else
2247                 return false;
2248 };
2249
2250 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2251 {
2252         u32 lb_partitions = wm->lb_size / wm->src_width;
2253         u32 line_time = wm->active_time + wm->blank_time;
2254         u32 latency_tolerant_lines;
2255         u32 latency_hiding;
2256         fixed20_12 a;
2257
2258         a.full = dfixed_const(1);
2259         if (wm->vsc.full > a.full)
2260                 latency_tolerant_lines = 1;
2261         else {
2262                 if (lb_partitions <= (wm->vtaps + 1))
2263                         latency_tolerant_lines = 1;
2264                 else
2265                         latency_tolerant_lines = 2;
2266         }
2267
2268         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2269
2270         if (evergreen_latency_watermark(wm) <= latency_hiding)
2271                 return true;
2272         else
2273                 return false;
2274 }
2275
2276 static void evergreen_program_watermarks(struct radeon_device *rdev,
2277                                          struct radeon_crtc *radeon_crtc,
2278                                          u32 lb_size, u32 num_heads)
2279 {
2280         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2281         struct evergreen_wm_params wm_low, wm_high;
2282         u32 dram_channels;
2283         u32 pixel_period;
2284         u32 line_time = 0;
2285         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2286         u32 priority_a_mark = 0, priority_b_mark = 0;
2287         u32 priority_a_cnt = PRIORITY_OFF;
2288         u32 priority_b_cnt = PRIORITY_OFF;
2289         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2290         u32 tmp, arb_control3;
2291         fixed20_12 a, b, c;
2292
2293         if (radeon_crtc->base.enabled && num_heads && mode) {
2294                 pixel_period = 1000000 / (u32)mode->clock;
2295                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2296                 priority_a_cnt = 0;
2297                 priority_b_cnt = 0;
2298                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2299
2300                 /* watermark for high clocks */
2301                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2302                         wm_high.yclk =
2303                                 radeon_dpm_get_mclk(rdev, false) * 10;
2304                         wm_high.sclk =
2305                                 radeon_dpm_get_sclk(rdev, false) * 10;
2306                 } else {
2307                         wm_high.yclk = rdev->pm.current_mclk * 10;
2308                         wm_high.sclk = rdev->pm.current_sclk * 10;
2309                 }
2310
2311                 wm_high.disp_clk = mode->clock;
2312                 wm_high.src_width = mode->crtc_hdisplay;
2313                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2314                 wm_high.blank_time = line_time - wm_high.active_time;
2315                 wm_high.interlaced = false;
2316                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2317                         wm_high.interlaced = true;
2318                 wm_high.vsc = radeon_crtc->vsc;
2319                 wm_high.vtaps = 1;
2320                 if (radeon_crtc->rmx_type != RMX_OFF)
2321                         wm_high.vtaps = 2;
2322                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2323                 wm_high.lb_size = lb_size;
2324                 wm_high.dram_channels = dram_channels;
2325                 wm_high.num_heads = num_heads;
2326
2327                 /* watermark for low clocks */
2328                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2329                         wm_low.yclk =
2330                                 radeon_dpm_get_mclk(rdev, true) * 10;
2331                         wm_low.sclk =
2332                                 radeon_dpm_get_sclk(rdev, true) * 10;
2333                 } else {
2334                         wm_low.yclk = rdev->pm.current_mclk * 10;
2335                         wm_low.sclk = rdev->pm.current_sclk * 10;
2336                 }
2337
2338                 wm_low.disp_clk = mode->clock;
2339                 wm_low.src_width = mode->crtc_hdisplay;
2340                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2341                 wm_low.blank_time = line_time - wm_low.active_time;
2342                 wm_low.interlaced = false;
2343                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2344                         wm_low.interlaced = true;
2345                 wm_low.vsc = radeon_crtc->vsc;
2346                 wm_low.vtaps = 1;
2347                 if (radeon_crtc->rmx_type != RMX_OFF)
2348                         wm_low.vtaps = 2;
2349                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2350                 wm_low.lb_size = lb_size;
2351                 wm_low.dram_channels = dram_channels;
2352                 wm_low.num_heads = num_heads;
2353
2354                 /* set for high clocks */
2355                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2356                 /* set for low clocks */
2357                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2358
2359                 /* possibly force display priority to high */
2360                 /* should really do this at mode validation time... */
2361                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2362                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2363                     !evergreen_check_latency_hiding(&wm_high) ||
2364                     (rdev->disp_priority == 2)) {
2365                         DRM_DEBUG_KMS("force priority a to high\n");
2366                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2367                 }
2368                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2369                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2370                     !evergreen_check_latency_hiding(&wm_low) ||
2371                     (rdev->disp_priority == 2)) {
2372                         DRM_DEBUG_KMS("force priority b to high\n");
2373                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2374                 }
2375
2376                 a.full = dfixed_const(1000);
2377                 b.full = dfixed_const(mode->clock);
2378                 b.full = dfixed_div(b, a);
2379                 c.full = dfixed_const(latency_watermark_a);
2380                 c.full = dfixed_mul(c, b);
2381                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2382                 c.full = dfixed_div(c, a);
2383                 a.full = dfixed_const(16);
2384                 c.full = dfixed_div(c, a);
2385                 priority_a_mark = dfixed_trunc(c);
2386                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2387
2388                 a.full = dfixed_const(1000);
2389                 b.full = dfixed_const(mode->clock);
2390                 b.full = dfixed_div(b, a);
2391                 c.full = dfixed_const(latency_watermark_b);
2392                 c.full = dfixed_mul(c, b);
2393                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2394                 c.full = dfixed_div(c, a);
2395                 a.full = dfixed_const(16);
2396                 c.full = dfixed_div(c, a);
2397                 priority_b_mark = dfixed_trunc(c);
2398                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2399         }
2400
2401         /* select wm A */
2402         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2403         tmp = arb_control3;
2404         tmp &= ~LATENCY_WATERMARK_MASK(3);
2405         tmp |= LATENCY_WATERMARK_MASK(1);
2406         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2407         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2408                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2409                 LATENCY_HIGH_WATERMARK(line_time)));
2410         /* select wm B */
2411         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2412         tmp &= ~LATENCY_WATERMARK_MASK(3);
2413         tmp |= LATENCY_WATERMARK_MASK(2);
2414         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2415         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2416                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2417                 LATENCY_HIGH_WATERMARK(line_time)));
2418         /* restore original selection */
2419         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2420
2421         /* write the priority marks */
2422         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2423         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2424
2425         /* save values for DPM */
2426         radeon_crtc->line_time = line_time;
2427         radeon_crtc->wm_high = latency_watermark_a;
2428         radeon_crtc->wm_low = latency_watermark_b;
2429 }
2430
2431 /**
2432  * evergreen_bandwidth_update - update display watermarks callback.
2433  *
2434  * @rdev: radeon_device pointer
2435  *
2436  * Update the display watermarks based on the requested mode(s)
2437  * (evergreen+).
2438  */
2439 void evergreen_bandwidth_update(struct radeon_device *rdev)
2440 {
2441         struct drm_display_mode *mode0 = NULL;
2442         struct drm_display_mode *mode1 = NULL;
2443         u32 num_heads = 0, lb_size;
2444         int i;
2445
2446         if (!rdev->mode_info.mode_config_initialized)
2447                 return;
2448
2449         radeon_update_display_priority(rdev);
2450
2451         for (i = 0; i < rdev->num_crtc; i++) {
2452                 if (rdev->mode_info.crtcs[i]->base.enabled)
2453                         num_heads++;
2454         }
2455         for (i = 0; i < rdev->num_crtc; i += 2) {
2456                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2457                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2458                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2459                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2460                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2461                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2462         }
2463 }
2464
2465 /**
2466  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2467  *
2468  * @rdev: radeon_device pointer
2469  *
2470  * Wait for the MC (memory controller) to be idle.
2471  * (evergreen+).
2472  * Returns 0 if the MC is idle, -1 if not.
2473  */
2474 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2475 {
2476         unsigned i;
2477         u32 tmp;
2478
2479         for (i = 0; i < rdev->usec_timeout; i++) {
2480                 /* read MC_STATUS */
2481                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2482                 if (!tmp)
2483                         return 0;
2484                 udelay(1);
2485         }
2486         return -1;
2487 }
2488
2489 /*
2490  * GART
2491  */
2492 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2493 {
2494         unsigned i;
2495         u32 tmp;
2496
2497         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2498
2499         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2500         for (i = 0; i < rdev->usec_timeout; i++) {
2501                 /* read MC_STATUS */
2502                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2503                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2504                 if (tmp == 2) {
2505                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2506                         return;
2507                 }
2508                 if (tmp) {
2509                         return;
2510                 }
2511                 udelay(1);
2512         }
2513 }
2514
2515 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2516 {
2517         u32 tmp;
2518         int r;
2519
2520         if (rdev->gart.robj == NULL) {
2521                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2522                 return -EINVAL;
2523         }
2524         r = radeon_gart_table_vram_pin(rdev);
2525         if (r)
2526                 return r;
2527         /* Setup L2 cache */
2528         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2529                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2530                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2531         WREG32(VM_L2_CNTL2, 0);
2532         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2533         /* Setup TLB control */
2534         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2535                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2536                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2537                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2538         if (rdev->flags & RADEON_IS_IGP) {
2539                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2540                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2541                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2542         } else {
2543                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2544                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2545                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2546                 if ((rdev->family == CHIP_JUNIPER) ||
2547                     (rdev->family == CHIP_CYPRESS) ||
2548                     (rdev->family == CHIP_HEMLOCK) ||
2549                     (rdev->family == CHIP_BARTS))
2550                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2551         }
2552         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2553         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2554         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2555         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2556         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2557         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2558         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2559         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2560                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2561         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2562                         (u32)(rdev->dummy_page.addr >> 12));
2563         WREG32(VM_CONTEXT1_CNTL, 0);
2564
2565         evergreen_pcie_gart_tlb_flush(rdev);
2566         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2567                  (unsigned)(rdev->mc.gtt_size >> 20),
2568                  (unsigned long long)rdev->gart.table_addr);
2569         rdev->gart.ready = true;
2570         return 0;
2571 }
2572
2573 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2574 {
2575         u32 tmp;
2576
2577         /* Disable all tables */
2578         WREG32(VM_CONTEXT0_CNTL, 0);
2579         WREG32(VM_CONTEXT1_CNTL, 0);
2580
2581         /* Setup L2 cache */
2582         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2583                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2584         WREG32(VM_L2_CNTL2, 0);
2585         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2586         /* Setup TLB control */
2587         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2588         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2589         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2590         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2591         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2592         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2593         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2594         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2595         radeon_gart_table_vram_unpin(rdev);
2596 }
2597
2598 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2599 {
2600         evergreen_pcie_gart_disable(rdev);
2601         radeon_gart_table_vram_free(rdev);
2602         radeon_gart_fini(rdev);
2603 }
2604
2605
2606 static void evergreen_agp_enable(struct radeon_device *rdev)
2607 {
2608         u32 tmp;
2609
2610         /* Setup L2 cache */
2611         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2612                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2613                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2614         WREG32(VM_L2_CNTL2, 0);
2615         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2616         /* Setup TLB control */
2617         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2618                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2619                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2620                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2621         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2622         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2623         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2624         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2625         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2626         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2627         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2628         WREG32(VM_CONTEXT0_CNTL, 0);
2629         WREG32(VM_CONTEXT1_CNTL, 0);
2630 }
2631
2632 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2633 {
2634         u32 crtc_enabled, tmp, frame_count, blackout;
2635         int i, j;
2636
2637         if (!ASIC_IS_NODCE(rdev)) {
2638                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2639                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2640
2641                 /* disable VGA render */
2642                 WREG32(VGA_RENDER_CONTROL, 0);
2643         }
2644         /* blank the display controllers */
2645         for (i = 0; i < rdev->num_crtc; i++) {
2646                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2647                 if (crtc_enabled) {
2648                         save->crtc_enabled[i] = true;
2649                         if (ASIC_IS_DCE6(rdev)) {
2650                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2651                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2652                                         radeon_wait_for_vblank(rdev, i);
2653                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2654                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2655                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2656                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2657                                 }
2658                         } else {
2659                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2660                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2661                                         radeon_wait_for_vblank(rdev, i);
2662                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2663                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2664                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2665                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2666                                 }
2667                         }
2668                         /* wait for the next frame */
2669                         frame_count = radeon_get_vblank_counter(rdev, i);
2670                         for (j = 0; j < rdev->usec_timeout; j++) {
2671                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2672                                         break;
2673                                 udelay(1);
2674                         }
2675
2676                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2677                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2678                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2679                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2680                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2681                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2682                         save->crtc_enabled[i] = false;
2683                         /* ***** */
2684                 } else {
2685                         save->crtc_enabled[i] = false;
2686                 }
2687         }
2688
2689         radeon_mc_wait_for_idle(rdev);
2690
2691         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2692         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2693                 /* Block CPU access */
2694                 WREG32(BIF_FB_EN, 0);
2695                 /* blackout the MC */
2696                 blackout &= ~BLACKOUT_MODE_MASK;
2697                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2698         }
2699         /* wait for the MC to settle */
2700         udelay(100);
2701
2702         /* lock double buffered regs */
2703         for (i = 0; i < rdev->num_crtc; i++) {
2704                 if (save->crtc_enabled[i]) {
2705                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2706                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2707                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2708                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2709                         }
2710                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2711                         if (!(tmp & 1)) {
2712                                 tmp |= 1;
2713                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2714                         }
2715                 }
2716         }
2717 }
2718
2719 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2720 {
2721         u32 tmp, frame_count;
2722         int i, j;
2723
2724         /* update crtc base addresses */
2725         for (i = 0; i < rdev->num_crtc; i++) {
2726                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2727                        upper_32_bits(rdev->mc.vram_start));
2728                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2729                        upper_32_bits(rdev->mc.vram_start));
2730                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2731                        (u32)rdev->mc.vram_start);
2732                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2733                        (u32)rdev->mc.vram_start);
2734         }
2735
2736         if (!ASIC_IS_NODCE(rdev)) {
2737                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2738                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2739         }
2740
2741         /* unlock regs and wait for update */
2742         for (i = 0; i < rdev->num_crtc; i++) {
2743                 if (save->crtc_enabled[i]) {
2744                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2745                         if ((tmp & 0x7) != 3) {
2746                                 tmp &= ~0x7;
2747                                 tmp |= 0x3;
2748                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2749                         }
2750                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2751                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2752                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2753                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2754                         }
2755                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2756                         if (tmp & 1) {
2757                                 tmp &= ~1;
2758                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2759                         }
2760                         for (j = 0; j < rdev->usec_timeout; j++) {
2761                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2762                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2763                                         break;
2764                                 udelay(1);
2765                         }
2766                 }
2767         }
2768
2769         /* unblackout the MC */
2770         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2771         tmp &= ~BLACKOUT_MODE_MASK;
2772         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2773         /* allow CPU access */
2774         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2775
2776         for (i = 0; i < rdev->num_crtc; i++) {
2777                 if (save->crtc_enabled[i]) {
2778                         if (ASIC_IS_DCE6(rdev)) {
2779                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2780                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2781                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2782                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2783                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2784                         } else {
2785                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2786                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2787                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2788                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2789                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2790                         }
2791                         /* wait for the next frame */
2792                         frame_count = radeon_get_vblank_counter(rdev, i);
2793                         for (j = 0; j < rdev->usec_timeout; j++) {
2794                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2795                                         break;
2796                                 udelay(1);
2797                         }
2798                 }
2799         }
2800         if (!ASIC_IS_NODCE(rdev)) {
2801                 /* Unlock vga access */
2802                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2803                 mdelay(1);
2804                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2805         }
2806 }
2807
2808 void evergreen_mc_program(struct radeon_device *rdev)
2809 {
2810         struct evergreen_mc_save save;
2811         u32 tmp;
2812         int i, j;
2813
2814         /* Initialize HDP */
2815         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2816                 WREG32((0x2c14 + j), 0x00000000);
2817                 WREG32((0x2c18 + j), 0x00000000);
2818                 WREG32((0x2c1c + j), 0x00000000);
2819                 WREG32((0x2c20 + j), 0x00000000);
2820                 WREG32((0x2c24 + j), 0x00000000);
2821         }
2822         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2823
2824         evergreen_mc_stop(rdev, &save);
2825         if (evergreen_mc_wait_for_idle(rdev)) {
2826                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2827         }
2828         /* Lockout access through VGA aperture*/
2829         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2830         /* Update configuration */
2831         if (rdev->flags & RADEON_IS_AGP) {
2832                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2833                         /* VRAM before AGP */
2834                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2835                                 rdev->mc.vram_start >> 12);
2836                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2837                                 rdev->mc.gtt_end >> 12);
2838                 } else {
2839                         /* VRAM after AGP */
2840                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2841                                 rdev->mc.gtt_start >> 12);
2842                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2843                                 rdev->mc.vram_end >> 12);
2844                 }
2845         } else {
2846                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2847                         rdev->mc.vram_start >> 12);
2848                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2849                         rdev->mc.vram_end >> 12);
2850         }
2851         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2852         /* llano/ontario only */
2853         if ((rdev->family == CHIP_PALM) ||
2854             (rdev->family == CHIP_SUMO) ||
2855             (rdev->family == CHIP_SUMO2)) {
2856                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2857                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2858                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2859                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2860         }
2861         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2862         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2863         WREG32(MC_VM_FB_LOCATION, tmp);
2864         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2865         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2866         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2867         if (rdev->flags & RADEON_IS_AGP) {
2868                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2869                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2870                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2871         } else {
2872                 WREG32(MC_VM_AGP_BASE, 0);
2873                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2874                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2875         }
2876         if (evergreen_mc_wait_for_idle(rdev)) {
2877                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2878         }
2879         evergreen_mc_resume(rdev, &save);
2880         /* we need to own VRAM, so turn off the VGA renderer here
2881          * to stop it overwriting our objects */
2882         rv515_vga_render_disable(rdev);
2883 }
2884
2885 /*
2886  * CP.
2887  */
2888 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2889 {
2890         struct radeon_ring *ring = &rdev->ring[ib->ring];
2891         u32 next_rptr;
2892
2893         /* set to DX10/11 mode */
2894         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2895         radeon_ring_write(ring, 1);
2896
2897         if (ring->rptr_save_reg) {
2898                 next_rptr = ring->wptr + 3 + 4;
2899                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2900                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2901                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2902                 radeon_ring_write(ring, next_rptr);
2903         } else if (rdev->wb.enabled) {
2904                 next_rptr = ring->wptr + 5 + 4;
2905                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2906                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2907                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2908                 radeon_ring_write(ring, next_rptr);
2909                 radeon_ring_write(ring, 0);
2910         }
2911
2912         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2913         radeon_ring_write(ring,
2914 #ifdef __BIG_ENDIAN
2915                           (2 << 0) |
2916 #endif
2917                           (ib->gpu_addr & 0xFFFFFFFC));
2918         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2919         radeon_ring_write(ring, ib->length_dw);
2920 }
2921
2922
2923 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2924 {
2925         const __be32 *fw_data;
2926         int i;
2927
2928         if (!rdev->me_fw || !rdev->pfp_fw)
2929                 return -EINVAL;
2930
2931         r700_cp_stop(rdev);
2932         WREG32(CP_RB_CNTL,
2933 #ifdef __BIG_ENDIAN
2934                BUF_SWAP_32BIT |
2935 #endif
2936                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2937
2938         fw_data = (const __be32 *)rdev->pfp_fw->data;
2939         WREG32(CP_PFP_UCODE_ADDR, 0);
2940         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2941                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2942         WREG32(CP_PFP_UCODE_ADDR, 0);
2943
2944         fw_data = (const __be32 *)rdev->me_fw->data;
2945         WREG32(CP_ME_RAM_WADDR, 0);
2946         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2947                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2948
2949         WREG32(CP_PFP_UCODE_ADDR, 0);
2950         WREG32(CP_ME_RAM_WADDR, 0);
2951         WREG32(CP_ME_RAM_RADDR, 0);
2952         return 0;
2953 }
2954
2955 static int evergreen_cp_start(struct radeon_device *rdev)
2956 {
2957         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2958         int r, i;
2959         uint32_t cp_me;
2960
2961         r = radeon_ring_lock(rdev, ring, 7);
2962         if (r) {
2963                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2964                 return r;
2965         }
2966         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2967         radeon_ring_write(ring, 0x1);
2968         radeon_ring_write(ring, 0x0);
2969         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2970         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2971         radeon_ring_write(ring, 0);
2972         radeon_ring_write(ring, 0);
2973         radeon_ring_unlock_commit(rdev, ring, false);
2974
2975         cp_me = 0xff;
2976         WREG32(CP_ME_CNTL, cp_me);
2977
2978         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2979         if (r) {
2980                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2981                 return r;
2982         }
2983
2984         /* setup clear context state */
2985         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2986         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2987
2988         for (i = 0; i < evergreen_default_size; i++)
2989                 radeon_ring_write(ring, evergreen_default_state[i]);
2990
2991         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2992         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2993
2994         /* set clear context state */
2995         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2996         radeon_ring_write(ring, 0);
2997
2998         /* SQ_VTX_BASE_VTX_LOC */
2999         radeon_ring_write(ring, 0xc0026f00);
3000         radeon_ring_write(ring, 0x00000000);
3001         radeon_ring_write(ring, 0x00000000);
3002         radeon_ring_write(ring, 0x00000000);
3003
3004         /* Clear consts */
3005         radeon_ring_write(ring, 0xc0036f00);
3006         radeon_ring_write(ring, 0x00000bc4);
3007         radeon_ring_write(ring, 0xffffffff);
3008         radeon_ring_write(ring, 0xffffffff);
3009         radeon_ring_write(ring, 0xffffffff);
3010
3011         radeon_ring_write(ring, 0xc0026900);
3012         radeon_ring_write(ring, 0x00000316);
3013         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3014         radeon_ring_write(ring, 0x00000010); /*  */
3015
3016         radeon_ring_unlock_commit(rdev, ring, false);
3017
3018         return 0;
3019 }
3020
3021 static int evergreen_cp_resume(struct radeon_device *rdev)
3022 {
3023         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3024         u32 tmp;
3025         u32 rb_bufsz;
3026         int r;
3027
3028         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3029         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3030                                  SOFT_RESET_PA |
3031                                  SOFT_RESET_SH |
3032                                  SOFT_RESET_VGT |
3033                                  SOFT_RESET_SPI |
3034                                  SOFT_RESET_SX));
3035         RREG32(GRBM_SOFT_RESET);
3036         mdelay(15);
3037         WREG32(GRBM_SOFT_RESET, 0);
3038         RREG32(GRBM_SOFT_RESET);
3039
3040         /* Set ring buffer size */
3041         rb_bufsz = order_base_2(ring->ring_size / 8);
3042         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3043 #ifdef __BIG_ENDIAN
3044         tmp |= BUF_SWAP_32BIT;
3045 #endif
3046         WREG32(CP_RB_CNTL, tmp);
3047         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3048         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3049
3050         /* Set the write pointer delay */
3051         WREG32(CP_RB_WPTR_DELAY, 0);
3052
3053         /* Initialize the ring buffer's read and write pointers */
3054         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3055         WREG32(CP_RB_RPTR_WR, 0);
3056         ring->wptr = 0;
3057         WREG32(CP_RB_WPTR, ring->wptr);
3058
3059         /* set the wb address whether it's enabled or not */
3060         WREG32(CP_RB_RPTR_ADDR,
3061                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3062         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3063         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3064
3065         if (rdev->wb.enabled)
3066                 WREG32(SCRATCH_UMSK, 0xff);
3067         else {
3068                 tmp |= RB_NO_UPDATE;
3069                 WREG32(SCRATCH_UMSK, 0);
3070         }
3071
3072         mdelay(1);
3073         WREG32(CP_RB_CNTL, tmp);
3074
3075         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3076         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3077
3078         evergreen_cp_start(rdev);
3079         ring->ready = true;
3080         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3081         if (r) {
3082                 ring->ready = false;
3083                 return r;
3084         }
3085         return 0;
3086 }
3087
3088 /*
3089  * Core functions
3090  */
3091 static void evergreen_gpu_init(struct radeon_device *rdev)
3092 {
3093         u32 gb_addr_config;
3094         u32 mc_shared_chmap, mc_arb_ramcfg;
3095         u32 sx_debug_1;
3096         u32 smx_dc_ctl0;
3097         u32 sq_config;
3098         u32 sq_lds_resource_mgmt;
3099         u32 sq_gpr_resource_mgmt_1;
3100         u32 sq_gpr_resource_mgmt_2;
3101         u32 sq_gpr_resource_mgmt_3;
3102         u32 sq_thread_resource_mgmt;
3103         u32 sq_thread_resource_mgmt_2;
3104         u32 sq_stack_resource_mgmt_1;
3105         u32 sq_stack_resource_mgmt_2;
3106         u32 sq_stack_resource_mgmt_3;
3107         u32 vgt_cache_invalidation;
3108         u32 hdp_host_path_cntl, tmp;
3109         u32 disabled_rb_mask;
3110         int i, j, ps_thread_count;
3111
3112         switch (rdev->family) {
3113         case CHIP_CYPRESS:
3114         case CHIP_HEMLOCK:
3115                 rdev->config.evergreen.num_ses = 2;
3116                 rdev->config.evergreen.max_pipes = 4;
3117                 rdev->config.evergreen.max_tile_pipes = 8;
3118                 rdev->config.evergreen.max_simds = 10;
3119                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3120                 rdev->config.evergreen.max_gprs = 256;
3121                 rdev->config.evergreen.max_threads = 248;
3122                 rdev->config.evergreen.max_gs_threads = 32;
3123                 rdev->config.evergreen.max_stack_entries = 512;
3124                 rdev->config.evergreen.sx_num_of_sets = 4;
3125                 rdev->config.evergreen.sx_max_export_size = 256;
3126                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3127                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3128                 rdev->config.evergreen.max_hw_contexts = 8;
3129                 rdev->config.evergreen.sq_num_cf_insts = 2;
3130
3131                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3132                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3133                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3134                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3135                 break;
3136         case CHIP_JUNIPER:
3137                 rdev->config.evergreen.num_ses = 1;
3138                 rdev->config.evergreen.max_pipes = 4;
3139                 rdev->config.evergreen.max_tile_pipes = 4;
3140                 rdev->config.evergreen.max_simds = 10;
3141                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3142                 rdev->config.evergreen.max_gprs = 256;
3143                 rdev->config.evergreen.max_threads = 248;
3144                 rdev->config.evergreen.max_gs_threads = 32;
3145                 rdev->config.evergreen.max_stack_entries = 512;
3146                 rdev->config.evergreen.sx_num_of_sets = 4;
3147                 rdev->config.evergreen.sx_max_export_size = 256;
3148                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3149                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3150                 rdev->config.evergreen.max_hw_contexts = 8;
3151                 rdev->config.evergreen.sq_num_cf_insts = 2;
3152
3153                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3154                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3155                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3156                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3157                 break;
3158         case CHIP_REDWOOD:
3159                 rdev->config.evergreen.num_ses = 1;
3160                 rdev->config.evergreen.max_pipes = 4;
3161                 rdev->config.evergreen.max_tile_pipes = 4;
3162                 rdev->config.evergreen.max_simds = 5;
3163                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3164                 rdev->config.evergreen.max_gprs = 256;
3165                 rdev->config.evergreen.max_threads = 248;
3166                 rdev->config.evergreen.max_gs_threads = 32;
3167                 rdev->config.evergreen.max_stack_entries = 256;
3168                 rdev->config.evergreen.sx_num_of_sets = 4;
3169                 rdev->config.evergreen.sx_max_export_size = 256;
3170                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3171                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3172                 rdev->config.evergreen.max_hw_contexts = 8;
3173                 rdev->config.evergreen.sq_num_cf_insts = 2;
3174
3175                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3176                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3177                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3178                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3179                 break;
3180         case CHIP_CEDAR:
3181         default:
3182                 rdev->config.evergreen.num_ses = 1;
3183                 rdev->config.evergreen.max_pipes = 2;
3184                 rdev->config.evergreen.max_tile_pipes = 2;
3185                 rdev->config.evergreen.max_simds = 2;
3186                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3187                 rdev->config.evergreen.max_gprs = 256;
3188                 rdev->config.evergreen.max_threads = 192;
3189                 rdev->config.evergreen.max_gs_threads = 16;
3190                 rdev->config.evergreen.max_stack_entries = 256;
3191                 rdev->config.evergreen.sx_num_of_sets = 4;
3192                 rdev->config.evergreen.sx_max_export_size = 128;
3193                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3194                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3195                 rdev->config.evergreen.max_hw_contexts = 4;
3196                 rdev->config.evergreen.sq_num_cf_insts = 1;
3197
3198                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3199                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3200                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3201                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3202                 break;
3203         case CHIP_PALM:
3204                 rdev->config.evergreen.num_ses = 1;
3205                 rdev->config.evergreen.max_pipes = 2;
3206                 rdev->config.evergreen.max_tile_pipes = 2;
3207                 rdev->config.evergreen.max_simds = 2;
3208                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3209                 rdev->config.evergreen.max_gprs = 256;
3210                 rdev->config.evergreen.max_threads = 192;
3211                 rdev->config.evergreen.max_gs_threads = 16;
3212                 rdev->config.evergreen.max_stack_entries = 256;
3213                 rdev->config.evergreen.sx_num_of_sets = 4;
3214                 rdev->config.evergreen.sx_max_export_size = 128;
3215                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3216                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3217                 rdev->config.evergreen.max_hw_contexts = 4;
3218                 rdev->config.evergreen.sq_num_cf_insts = 1;
3219
3220                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3221                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3222                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3223                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3224                 break;
3225         case CHIP_SUMO:
3226                 rdev->config.evergreen.num_ses = 1;
3227                 rdev->config.evergreen.max_pipes = 4;
3228                 rdev->config.evergreen.max_tile_pipes = 4;
3229                 if (rdev->pdev->device == 0x9648)
3230                         rdev->config.evergreen.max_simds = 3;
3231                 else if ((rdev->pdev->device == 0x9647) ||
3232                          (rdev->pdev->device == 0x964a))
3233                         rdev->config.evergreen.max_simds = 4;
3234                 else
3235                         rdev->config.evergreen.max_simds = 5;
3236                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3237                 rdev->config.evergreen.max_gprs = 256;
3238                 rdev->config.evergreen.max_threads = 248;
3239                 rdev->config.evergreen.max_gs_threads = 32;
3240                 rdev->config.evergreen.max_stack_entries = 256;
3241                 rdev->config.evergreen.sx_num_of_sets = 4;
3242                 rdev->config.evergreen.sx_max_export_size = 256;
3243                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3244                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3245                 rdev->config.evergreen.max_hw_contexts = 8;
3246                 rdev->config.evergreen.sq_num_cf_insts = 2;
3247
3248                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3249                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3250                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3251                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3252                 break;
3253         case CHIP_SUMO2:
3254                 rdev->config.evergreen.num_ses = 1;
3255                 rdev->config.evergreen.max_pipes = 4;
3256                 rdev->config.evergreen.max_tile_pipes = 4;
3257                 rdev->config.evergreen.max_simds = 2;
3258                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3259                 rdev->config.evergreen.max_gprs = 256;
3260                 rdev->config.evergreen.max_threads = 248;
3261                 rdev->config.evergreen.max_gs_threads = 32;
3262                 rdev->config.evergreen.max_stack_entries = 512;
3263                 rdev->config.evergreen.sx_num_of_sets = 4;
3264                 rdev->config.evergreen.sx_max_export_size = 256;
3265                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3266                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3267                 rdev->config.evergreen.max_hw_contexts = 4;
3268                 rdev->config.evergreen.sq_num_cf_insts = 2;
3269
3270                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3271                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3272                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3273                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3274                 break;
3275         case CHIP_BARTS:
3276                 rdev->config.evergreen.num_ses = 2;
3277                 rdev->config.evergreen.max_pipes = 4;
3278                 rdev->config.evergreen.max_tile_pipes = 8;
3279                 rdev->config.evergreen.max_simds = 7;
3280                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3281                 rdev->config.evergreen.max_gprs = 256;
3282                 rdev->config.evergreen.max_threads = 248;
3283                 rdev->config.evergreen.max_gs_threads = 32;
3284                 rdev->config.evergreen.max_stack_entries = 512;
3285                 rdev->config.evergreen.sx_num_of_sets = 4;
3286                 rdev->config.evergreen.sx_max_export_size = 256;
3287                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3288                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3289                 rdev->config.evergreen.max_hw_contexts = 8;
3290                 rdev->config.evergreen.sq_num_cf_insts = 2;
3291
3292                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3293                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3294                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3295                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3296                 break;
3297         case CHIP_TURKS:
3298                 rdev->config.evergreen.num_ses = 1;
3299                 rdev->config.evergreen.max_pipes = 4;
3300                 rdev->config.evergreen.max_tile_pipes = 4;
3301                 rdev->config.evergreen.max_simds = 6;
3302                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3303                 rdev->config.evergreen.max_gprs = 256;
3304                 rdev->config.evergreen.max_threads = 248;
3305                 rdev->config.evergreen.max_gs_threads = 32;
3306                 rdev->config.evergreen.max_stack_entries = 256;
3307                 rdev->config.evergreen.sx_num_of_sets = 4;
3308                 rdev->config.evergreen.sx_max_export_size = 256;
3309                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3310                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3311                 rdev->config.evergreen.max_hw_contexts = 8;
3312                 rdev->config.evergreen.sq_num_cf_insts = 2;
3313
3314                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3315                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3316                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3317                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3318                 break;
3319         case CHIP_CAICOS:
3320                 rdev->config.evergreen.num_ses = 1;
3321                 rdev->config.evergreen.max_pipes = 2;
3322                 rdev->config.evergreen.max_tile_pipes = 2;
3323                 rdev->config.evergreen.max_simds = 2;
3324                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3325                 rdev->config.evergreen.max_gprs = 256;
3326                 rdev->config.evergreen.max_threads = 192;
3327                 rdev->config.evergreen.max_gs_threads = 16;
3328                 rdev->config.evergreen.max_stack_entries = 256;
3329                 rdev->config.evergreen.sx_num_of_sets = 4;
3330                 rdev->config.evergreen.sx_max_export_size = 128;
3331                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3332                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3333                 rdev->config.evergreen.max_hw_contexts = 4;
3334                 rdev->config.evergreen.sq_num_cf_insts = 1;
3335
3336                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3337                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3338                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3339                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3340                 break;
3341         }
3342
3343         /* Initialize HDP */
3344         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3345                 WREG32((0x2c14 + j), 0x00000000);
3346                 WREG32((0x2c18 + j), 0x00000000);
3347                 WREG32((0x2c1c + j), 0x00000000);
3348                 WREG32((0x2c20 + j), 0x00000000);
3349                 WREG32((0x2c24 + j), 0x00000000);
3350         }
3351
3352         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3353         WREG32(SRBM_INT_CNTL, 0x1);
3354         WREG32(SRBM_INT_ACK, 0x1);
3355
3356         evergreen_fix_pci_max_read_req_size(rdev);
3357
3358         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3359         if ((rdev->family == CHIP_PALM) ||
3360             (rdev->family == CHIP_SUMO) ||
3361             (rdev->family == CHIP_SUMO2))
3362                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3363         else
3364                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3365
3366         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3367          * not have bank info, so create a custom tiling dword.
3368          * bits 3:0   num_pipes
3369          * bits 7:4   num_banks
3370          * bits 11:8  group_size
3371          * bits 15:12 row_size
3372          */
3373         rdev->config.evergreen.tile_config = 0;
3374         switch (rdev->config.evergreen.max_tile_pipes) {
3375         case 1:
3376         default:
3377                 rdev->config.evergreen.tile_config |= (0 << 0);
3378                 break;
3379         case 2:
3380                 rdev->config.evergreen.tile_config |= (1 << 0);
3381                 break;
3382         case 4:
3383                 rdev->config.evergreen.tile_config |= (2 << 0);
3384                 break;
3385         case 8:
3386                 rdev->config.evergreen.tile_config |= (3 << 0);
3387                 break;
3388         }
3389         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3390         if (rdev->flags & RADEON_IS_IGP)
3391                 rdev->config.evergreen.tile_config |= 1 << 4;
3392         else {
3393                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3394                 case 0: /* four banks */
3395                         rdev->config.evergreen.tile_config |= 0 << 4;
3396                         break;
3397                 case 1: /* eight banks */
3398                         rdev->config.evergreen.tile_config |= 1 << 4;
3399                         break;
3400                 case 2: /* sixteen banks */
3401                 default:
3402                         rdev->config.evergreen.tile_config |= 2 << 4;
3403                         break;
3404                 }
3405         }
3406         rdev->config.evergreen.tile_config |= 0 << 8;
3407         rdev->config.evergreen.tile_config |=
3408                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3409
3410         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3411                 u32 efuse_straps_4;
3412                 u32 efuse_straps_3;
3413
3414                 efuse_straps_4 = RREG32_RCU(0x204);
3415                 efuse_straps_3 = RREG32_RCU(0x203);
3416                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3417                       ((efuse_straps_3 & 0xf0000000) >> 28));
3418         } else {
3419                 tmp = 0;
3420                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3421                         u32 rb_disable_bitmap;
3422
3423                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3424                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3425                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3426                         tmp <<= 4;
3427                         tmp |= rb_disable_bitmap;
3428                 }
3429         }
3430         /* enabled rb are just the one not disabled :) */
3431         disabled_rb_mask = tmp;
3432         tmp = 0;
3433         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3434                 tmp |= (1 << i);
3435         /* if all the backends are disabled, fix it up here */
3436         if ((disabled_rb_mask & tmp) == tmp) {
3437                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3438                         disabled_rb_mask &= ~(1 << i);
3439         }
3440
3441         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3442                 u32 simd_disable_bitmap;
3443
3444                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3445                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3446                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3447                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3448                 tmp <<= 16;
3449                 tmp |= simd_disable_bitmap;
3450         }
3451         rdev->config.evergreen.active_simds = hweight32(~tmp);
3452
3453         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3454         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3455
3456         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3457         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3458         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3459         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3460         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3461         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3462         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3463
3464         if ((rdev->config.evergreen.max_backends == 1) &&
3465             (rdev->flags & RADEON_IS_IGP)) {
3466                 if ((disabled_rb_mask & 3) == 1) {
3467                         /* RB0 disabled, RB1 enabled */
3468                         tmp = 0x11111111;
3469                 } else {
3470                         /* RB1 disabled, RB0 enabled */
3471                         tmp = 0x00000000;
3472                 }
3473         } else {
3474                 tmp = gb_addr_config & NUM_PIPES_MASK;
3475                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3476                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3477         }
3478         WREG32(GB_BACKEND_MAP, tmp);
3479
3480         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3481         WREG32(CGTS_TCC_DISABLE, 0);
3482         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3483         WREG32(CGTS_USER_TCC_DISABLE, 0);
3484
3485         /* set HW defaults for 3D engine */
3486         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3487                                      ROQ_IB2_START(0x2b)));
3488
3489         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3490
3491         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3492                              SYNC_GRADIENT |
3493                              SYNC_WALKER |
3494                              SYNC_ALIGNER));
3495
3496         sx_debug_1 = RREG32(SX_DEBUG_1);
3497         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3498         WREG32(SX_DEBUG_1, sx_debug_1);
3499
3500
3501         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3502         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3503         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3504         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3505
3506         if (rdev->family <= CHIP_SUMO2)
3507                 WREG32(SMX_SAR_CTL0, 0x00010000);
3508
3509         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3510                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3511                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3512
3513         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3514                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3515                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3516
3517         WREG32(VGT_NUM_INSTANCES, 1);
3518         WREG32(SPI_CONFIG_CNTL, 0);
3519         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3520         WREG32(CP_PERFMON_CNTL, 0);
3521
3522         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3523                                   FETCH_FIFO_HIWATER(0x4) |
3524                                   DONE_FIFO_HIWATER(0xe0) |
3525                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3526
3527         sq_config = RREG32(SQ_CONFIG);
3528         sq_config &= ~(PS_PRIO(3) |
3529                        VS_PRIO(3) |
3530                        GS_PRIO(3) |
3531                        ES_PRIO(3));
3532         sq_config |= (VC_ENABLE |
3533                       EXPORT_SRC_C |
3534                       PS_PRIO(0) |
3535                       VS_PRIO(1) |
3536                       GS_PRIO(2) |
3537                       ES_PRIO(3));
3538
3539         switch (rdev->family) {
3540         case CHIP_CEDAR:
3541         case CHIP_PALM:
3542         case CHIP_SUMO:
3543         case CHIP_SUMO2:
3544         case CHIP_CAICOS:
3545                 /* no vertex cache */
3546                 sq_config &= ~VC_ENABLE;
3547                 break;
3548         default:
3549                 break;
3550         }
3551
3552         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3553
3554         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3555         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3556         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3557         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3558         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3559         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3560         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3561
3562         switch (rdev->family) {
3563         case CHIP_CEDAR:
3564         case CHIP_PALM:
3565         case CHIP_SUMO:
3566         case CHIP_SUMO2:
3567                 ps_thread_count = 96;
3568                 break;
3569         default:
3570                 ps_thread_count = 128;
3571                 break;
3572         }
3573
3574         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3575         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3576         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3577         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3578         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3579         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3580
3581         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3582         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3583         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3584         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3585         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3586         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3587
3588         WREG32(SQ_CONFIG, sq_config);
3589         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3590         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3591         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3592         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3593         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3594         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3595         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3596         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3597         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3598         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3599
3600         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3601                                           FORCE_EOV_MAX_REZ_CNT(255)));
3602
3603         switch (rdev->family) {
3604         case CHIP_CEDAR:
3605         case CHIP_PALM:
3606         case CHIP_SUMO:
3607         case CHIP_SUMO2:
3608         case CHIP_CAICOS:
3609                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3610                 break;
3611         default:
3612                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3613                 break;
3614         }
3615         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3616         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3617
3618         WREG32(VGT_GS_VERTEX_REUSE, 16);
3619         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3620         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3621
3622         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3623         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3624
3625         WREG32(CB_PERF_CTR0_SEL_0, 0);
3626         WREG32(CB_PERF_CTR0_SEL_1, 0);
3627         WREG32(CB_PERF_CTR1_SEL_0, 0);
3628         WREG32(CB_PERF_CTR1_SEL_1, 0);
3629         WREG32(CB_PERF_CTR2_SEL_0, 0);
3630         WREG32(CB_PERF_CTR2_SEL_1, 0);
3631         WREG32(CB_PERF_CTR3_SEL_0, 0);
3632         WREG32(CB_PERF_CTR3_SEL_1, 0);
3633
3634         /* clear render buffer base addresses */
3635         WREG32(CB_COLOR0_BASE, 0);
3636         WREG32(CB_COLOR1_BASE, 0);
3637         WREG32(CB_COLOR2_BASE, 0);
3638         WREG32(CB_COLOR3_BASE, 0);
3639         WREG32(CB_COLOR4_BASE, 0);
3640         WREG32(CB_COLOR5_BASE, 0);
3641         WREG32(CB_COLOR6_BASE, 0);
3642         WREG32(CB_COLOR7_BASE, 0);
3643         WREG32(CB_COLOR8_BASE, 0);
3644         WREG32(CB_COLOR9_BASE, 0);
3645         WREG32(CB_COLOR10_BASE, 0);
3646         WREG32(CB_COLOR11_BASE, 0);
3647
3648         /* set the shader const cache sizes to 0 */
3649         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3650                 WREG32(i, 0);
3651         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3652                 WREG32(i, 0);
3653
3654         tmp = RREG32(HDP_MISC_CNTL);
3655         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3656         WREG32(HDP_MISC_CNTL, tmp);
3657
3658         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3659         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3660
3661         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3662
3663         udelay(50);
3664
3665 }
3666
3667 int evergreen_mc_init(struct radeon_device *rdev)
3668 {
3669         u32 tmp;
3670         int chansize, numchan;
3671
3672         /* Get VRAM informations */
3673         rdev->mc.vram_is_ddr = true;
3674         if ((rdev->family == CHIP_PALM) ||
3675             (rdev->family == CHIP_SUMO) ||
3676             (rdev->family == CHIP_SUMO2))
3677                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3678         else
3679                 tmp = RREG32(MC_ARB_RAMCFG);
3680         if (tmp & CHANSIZE_OVERRIDE) {
3681                 chansize = 16;
3682         } else if (tmp & CHANSIZE_MASK) {
3683                 chansize = 64;
3684         } else {
3685                 chansize = 32;
3686         }
3687         tmp = RREG32(MC_SHARED_CHMAP);
3688         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3689         case 0:
3690         default:
3691                 numchan = 1;
3692                 break;
3693         case 1:
3694                 numchan = 2;
3695                 break;
3696         case 2:
3697                 numchan = 4;
3698                 break;
3699         case 3:
3700                 numchan = 8;
3701                 break;
3702         }
3703         rdev->mc.vram_width = numchan * chansize;
3704         /* Could aper size report 0 ? */
3705         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3706         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3707         /* Setup GPU memory space */
3708         if ((rdev->family == CHIP_PALM) ||
3709             (rdev->family == CHIP_SUMO) ||
3710             (rdev->family == CHIP_SUMO2)) {
3711                 /* size in bytes on fusion */
3712                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3713                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3714         } else {
3715                 /* size in MB on evergreen/cayman/tn */
3716                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3717                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3718         }
3719         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3720         r700_vram_gtt_location(rdev, &rdev->mc);
3721         radeon_update_bandwidth_info(rdev);
3722
3723         return 0;
3724 }
3725
3726 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3727 {
3728         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3729                 RREG32(GRBM_STATUS));
3730         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3731                 RREG32(GRBM_STATUS_SE0));
3732         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3733                 RREG32(GRBM_STATUS_SE1));
3734         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3735                 RREG32(SRBM_STATUS));
3736         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3737                 RREG32(SRBM_STATUS2));
3738         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3739                 RREG32(CP_STALLED_STAT1));
3740         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3741                 RREG32(CP_STALLED_STAT2));
3742         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3743                 RREG32(CP_BUSY_STAT));
3744         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3745                 RREG32(CP_STAT));
3746         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3747                 RREG32(DMA_STATUS_REG));
3748         if (rdev->family >= CHIP_CAYMAN) {
3749                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3750                          RREG32(DMA_STATUS_REG + 0x800));
3751         }
3752 }
3753
3754 bool evergreen_is_display_hung(struct radeon_device *rdev)
3755 {
3756         u32 crtc_hung = 0;
3757         u32 crtc_status[6];
3758         u32 i, j, tmp;
3759
3760         for (i = 0; i < rdev->num_crtc; i++) {
3761                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3762                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3763                         crtc_hung |= (1 << i);
3764                 }
3765         }
3766
3767         for (j = 0; j < 10; j++) {
3768                 for (i = 0; i < rdev->num_crtc; i++) {
3769                         if (crtc_hung & (1 << i)) {
3770                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3771                                 if (tmp != crtc_status[i])
3772                                         crtc_hung &= ~(1 << i);
3773                         }
3774                 }
3775                 if (crtc_hung == 0)
3776                         return false;
3777                 udelay(100);
3778         }
3779
3780         return true;
3781 }
3782
3783 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3784 {
3785         u32 reset_mask = 0;
3786         u32 tmp;
3787
3788         /* GRBM_STATUS */
3789         tmp = RREG32(GRBM_STATUS);
3790         if (tmp & (PA_BUSY | SC_BUSY |
3791                    SH_BUSY | SX_BUSY |
3792                    TA_BUSY | VGT_BUSY |
3793                    DB_BUSY | CB_BUSY |
3794                    SPI_BUSY | VGT_BUSY_NO_DMA))
3795                 reset_mask |= RADEON_RESET_GFX;
3796
3797         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3798                    CP_BUSY | CP_COHERENCY_BUSY))
3799                 reset_mask |= RADEON_RESET_CP;
3800
3801         if (tmp & GRBM_EE_BUSY)
3802                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3803
3804         /* DMA_STATUS_REG */
3805         tmp = RREG32(DMA_STATUS_REG);
3806         if (!(tmp & DMA_IDLE))
3807                 reset_mask |= RADEON_RESET_DMA;
3808
3809         /* SRBM_STATUS2 */
3810         tmp = RREG32(SRBM_STATUS2);
3811         if (tmp & DMA_BUSY)
3812                 reset_mask |= RADEON_RESET_DMA;
3813
3814         /* SRBM_STATUS */
3815         tmp = RREG32(SRBM_STATUS);
3816         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3817                 reset_mask |= RADEON_RESET_RLC;
3818
3819         if (tmp & IH_BUSY)
3820                 reset_mask |= RADEON_RESET_IH;
3821
3822         if (tmp & SEM_BUSY)
3823                 reset_mask |= RADEON_RESET_SEM;
3824
3825         if (tmp & GRBM_RQ_PENDING)
3826                 reset_mask |= RADEON_RESET_GRBM;
3827
3828         if (tmp & VMC_BUSY)
3829                 reset_mask |= RADEON_RESET_VMC;
3830
3831         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3832                    MCC_BUSY | MCD_BUSY))
3833                 reset_mask |= RADEON_RESET_MC;
3834
3835         if (evergreen_is_display_hung(rdev))
3836                 reset_mask |= RADEON_RESET_DISPLAY;
3837
3838         /* VM_L2_STATUS */
3839         tmp = RREG32(VM_L2_STATUS);
3840         if (tmp & L2_BUSY)
3841                 reset_mask |= RADEON_RESET_VMC;
3842
3843         /* Skip MC reset as it's mostly likely not hung, just busy */
3844         if (reset_mask & RADEON_RESET_MC) {
3845                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3846                 reset_mask &= ~RADEON_RESET_MC;
3847         }
3848
3849         return reset_mask;
3850 }
3851
3852 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3853 {
3854         struct evergreen_mc_save save;
3855         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3856         u32 tmp;
3857
3858         if (reset_mask == 0)
3859                 return;
3860
3861         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3862
3863         evergreen_print_gpu_status_regs(rdev);
3864
3865         /* Disable CP parsing/prefetching */
3866         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3867
3868         if (reset_mask & RADEON_RESET_DMA) {
3869                 /* Disable DMA */
3870                 tmp = RREG32(DMA_RB_CNTL);
3871                 tmp &= ~DMA_RB_ENABLE;
3872                 WREG32(DMA_RB_CNTL, tmp);
3873         }
3874
3875         udelay(50);
3876
3877         evergreen_mc_stop(rdev, &save);
3878         if (evergreen_mc_wait_for_idle(rdev)) {
3879                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3880         }
3881
3882         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3883                 grbm_soft_reset |= SOFT_RESET_DB |
3884                         SOFT_RESET_CB |
3885                         SOFT_RESET_PA |
3886                         SOFT_RESET_SC |
3887                         SOFT_RESET_SPI |
3888                         SOFT_RESET_SX |
3889                         SOFT_RESET_SH |
3890                         SOFT_RESET_TC |
3891                         SOFT_RESET_TA |
3892                         SOFT_RESET_VC |
3893                         SOFT_RESET_VGT;
3894         }
3895
3896         if (reset_mask & RADEON_RESET_CP) {
3897                 grbm_soft_reset |= SOFT_RESET_CP |
3898                         SOFT_RESET_VGT;
3899
3900                 srbm_soft_reset |= SOFT_RESET_GRBM;
3901         }
3902
3903         if (reset_mask & RADEON_RESET_DMA)
3904                 srbm_soft_reset |= SOFT_RESET_DMA;
3905
3906         if (reset_mask & RADEON_RESET_DISPLAY)
3907                 srbm_soft_reset |= SOFT_RESET_DC;
3908
3909         if (reset_mask & RADEON_RESET_RLC)
3910                 srbm_soft_reset |= SOFT_RESET_RLC;
3911
3912         if (reset_mask & RADEON_RESET_SEM)
3913                 srbm_soft_reset |= SOFT_RESET_SEM;
3914
3915         if (reset_mask & RADEON_RESET_IH)
3916                 srbm_soft_reset |= SOFT_RESET_IH;
3917
3918         if (reset_mask & RADEON_RESET_GRBM)
3919                 srbm_soft_reset |= SOFT_RESET_GRBM;
3920
3921         if (reset_mask & RADEON_RESET_VMC)
3922                 srbm_soft_reset |= SOFT_RESET_VMC;
3923
3924         if (!(rdev->flags & RADEON_IS_IGP)) {
3925                 if (reset_mask & RADEON_RESET_MC)
3926                         srbm_soft_reset |= SOFT_RESET_MC;
3927         }
3928
3929         if (grbm_soft_reset) {
3930                 tmp = RREG32(GRBM_SOFT_RESET);
3931                 tmp |= grbm_soft_reset;
3932                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3933                 WREG32(GRBM_SOFT_RESET, tmp);
3934                 tmp = RREG32(GRBM_SOFT_RESET);
3935
3936                 udelay(50);
3937
3938                 tmp &= ~grbm_soft_reset;
3939                 WREG32(GRBM_SOFT_RESET, tmp);
3940                 tmp = RREG32(GRBM_SOFT_RESET);
3941         }
3942
3943         if (srbm_soft_reset) {
3944                 tmp = RREG32(SRBM_SOFT_RESET);
3945                 tmp |= srbm_soft_reset;
3946                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3947                 WREG32(SRBM_SOFT_RESET, tmp);
3948                 tmp = RREG32(SRBM_SOFT_RESET);
3949
3950                 udelay(50);
3951
3952                 tmp &= ~srbm_soft_reset;
3953                 WREG32(SRBM_SOFT_RESET, tmp);
3954                 tmp = RREG32(SRBM_SOFT_RESET);
3955         }
3956
3957         /* Wait a little for things to settle down */
3958         udelay(50);
3959
3960         evergreen_mc_resume(rdev, &save);
3961         udelay(50);
3962
3963         evergreen_print_gpu_status_regs(rdev);
3964 }
3965
3966 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3967 {
3968         struct evergreen_mc_save save;
3969         u32 tmp, i;
3970
3971         dev_info(rdev->dev, "GPU pci config reset\n");
3972
3973         /* disable dpm? */
3974
3975         /* Disable CP parsing/prefetching */
3976         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3977         udelay(50);
3978         /* Disable DMA */
3979         tmp = RREG32(DMA_RB_CNTL);
3980         tmp &= ~DMA_RB_ENABLE;
3981         WREG32(DMA_RB_CNTL, tmp);
3982         /* XXX other engines? */
3983
3984         /* halt the rlc */
3985         r600_rlc_stop(rdev);
3986
3987         udelay(50);
3988
3989         /* set mclk/sclk to bypass */
3990         rv770_set_clk_bypass_mode(rdev);
3991         /* disable BM */
3992         pci_clear_master(rdev->pdev);
3993         /* disable mem access */
3994         evergreen_mc_stop(rdev, &save);
3995         if (evergreen_mc_wait_for_idle(rdev)) {
3996                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3997         }
3998         /* reset */
3999         radeon_pci_config_reset(rdev);
4000         /* wait for asic to come out of reset */
4001         for (i = 0; i < rdev->usec_timeout; i++) {
4002                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4003                         break;
4004                 udelay(1);
4005         }
4006 }
4007
4008 int evergreen_asic_reset(struct radeon_device *rdev)
4009 {
4010         u32 reset_mask;
4011
4012         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4013
4014         if (reset_mask)
4015                 r600_set_bios_scratch_engine_hung(rdev, true);
4016
4017         /* try soft reset */
4018         evergreen_gpu_soft_reset(rdev, reset_mask);
4019
4020         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4021
4022         /* try pci config reset */
4023         if (reset_mask && radeon_hard_reset)
4024                 evergreen_gpu_pci_config_reset(rdev);
4025
4026         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4027
4028         if (!reset_mask)
4029                 r600_set_bios_scratch_engine_hung(rdev, false);
4030
4031         return 0;
4032 }
4033
4034 /**
4035  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4036  *
4037  * @rdev: radeon_device pointer
4038  * @ring: radeon_ring structure holding ring information
4039  *
4040  * Check if the GFX engine is locked up.
4041  * Returns true if the engine appears to be locked up, false if not.
4042  */
4043 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4044 {
4045         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4046
4047         if (!(reset_mask & (RADEON_RESET_GFX |
4048                             RADEON_RESET_COMPUTE |
4049                             RADEON_RESET_CP))) {
4050                 radeon_ring_lockup_update(rdev, ring);
4051                 return false;
4052         }
4053         return radeon_ring_test_lockup(rdev, ring);
4054 }
4055
4056 /*
4057  * RLC
4058  */
4059 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4060 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4061
4062 void sumo_rlc_fini(struct radeon_device *rdev)
4063 {
4064         int r;
4065
4066         /* save restore block */
4067         if (rdev->rlc.save_restore_obj) {
4068                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4069                 if (unlikely(r != 0))
4070                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4071                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4072                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4073
4074                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4075                 rdev->rlc.save_restore_obj = NULL;
4076         }
4077
4078         /* clear state block */
4079         if (rdev->rlc.clear_state_obj) {
4080                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4081                 if (unlikely(r != 0))
4082                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4083                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4084                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4085
4086                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4087                 rdev->rlc.clear_state_obj = NULL;
4088         }
4089
4090         /* clear state block */
4091         if (rdev->rlc.cp_table_obj) {
4092                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4093                 if (unlikely(r != 0))
4094                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4095                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4096                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4097
4098                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4099                 rdev->rlc.cp_table_obj = NULL;
4100         }
4101 }
4102
4103 #define CP_ME_TABLE_SIZE    96
4104
4105 int sumo_rlc_init(struct radeon_device *rdev)
4106 {
4107         const u32 *src_ptr;
4108         volatile u32 *dst_ptr;
4109         u32 dws, data, i, j, k, reg_num;
4110         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4111         u64 reg_list_mc_addr;
4112         const struct cs_section_def *cs_data;
4113         int r;
4114
4115         src_ptr = rdev->rlc.reg_list;
4116         dws = rdev->rlc.reg_list_size;
4117         if (rdev->family >= CHIP_BONAIRE) {
4118                 dws += (5 * 16) + 48 + 48 + 64;
4119         }
4120         cs_data = rdev->rlc.cs_data;
4121
4122         if (src_ptr) {
4123                 /* save restore block */
4124                 if (rdev->rlc.save_restore_obj == NULL) {
4125                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4126                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4127                                              NULL, &rdev->rlc.save_restore_obj);
4128                         if (r) {
4129                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4130                                 return r;
4131                         }
4132                 }
4133
4134                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4135                 if (unlikely(r != 0)) {
4136                         sumo_rlc_fini(rdev);
4137                         return r;
4138                 }
4139                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4140                                   &rdev->rlc.save_restore_gpu_addr);
4141                 if (r) {
4142                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4143                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4144                         sumo_rlc_fini(rdev);
4145                         return r;
4146                 }
4147
4148                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4149                 if (r) {
4150                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4151                         sumo_rlc_fini(rdev);
4152                         return r;
4153                 }
4154                 /* write the sr buffer */
4155                 dst_ptr = rdev->rlc.sr_ptr;
4156                 if (rdev->family >= CHIP_TAHITI) {
4157                         /* SI */
4158                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4159                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4160                 } else {
4161                         /* ON/LN/TN */
4162                         /* format:
4163                          * dw0: (reg2 << 16) | reg1
4164                          * dw1: reg1 save space
4165                          * dw2: reg2 save space
4166                          */
4167                         for (i = 0; i < dws; i++) {
4168                                 data = src_ptr[i] >> 2;
4169                                 i++;
4170                                 if (i < dws)
4171                                         data |= (src_ptr[i] >> 2) << 16;
4172                                 j = (((i - 1) * 3) / 2);
4173                                 dst_ptr[j] = cpu_to_le32(data);
4174                         }
4175                         j = ((i * 3) / 2);
4176                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4177                 }
4178                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4179                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4180         }
4181
4182         if (cs_data) {
4183                 /* clear state block */
4184                 if (rdev->family >= CHIP_BONAIRE) {
4185                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4186                 } else if (rdev->family >= CHIP_TAHITI) {
4187                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4188                         dws = rdev->rlc.clear_state_size + (256 / 4);
4189                 } else {
4190                         reg_list_num = 0;
4191                         dws = 0;
4192                         for (i = 0; cs_data[i].section != NULL; i++) {
4193                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4194                                         reg_list_num++;
4195                                         dws += cs_data[i].section[j].reg_count;
4196                                 }
4197                         }
4198                         reg_list_blk_index = (3 * reg_list_num + 2);
4199                         dws += reg_list_blk_index;
4200                         rdev->rlc.clear_state_size = dws;
4201                 }
4202
4203                 if (rdev->rlc.clear_state_obj == NULL) {
4204                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4205                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4206                                              NULL, &rdev->rlc.clear_state_obj);
4207                         if (r) {
4208                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4209                                 sumo_rlc_fini(rdev);
4210                                 return r;
4211                         }
4212                 }
4213                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4214                 if (unlikely(r != 0)) {
4215                         sumo_rlc_fini(rdev);
4216                         return r;
4217                 }
4218                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4219                                   &rdev->rlc.clear_state_gpu_addr);
4220                 if (r) {
4221                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4222                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4223                         sumo_rlc_fini(rdev);
4224                         return r;
4225                 }
4226
4227                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4228                 if (r) {
4229                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4230                         sumo_rlc_fini(rdev);
4231                         return r;
4232                 }
4233                 /* set up the cs buffer */
4234                 dst_ptr = rdev->rlc.cs_ptr;
4235                 if (rdev->family >= CHIP_BONAIRE) {
4236                         cik_get_csb_buffer(rdev, dst_ptr);
4237                 } else if (rdev->family >= CHIP_TAHITI) {
4238                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4239                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4240                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4241                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4242                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4243                 } else {
4244                         reg_list_hdr_blk_index = 0;
4245                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4246                         data = upper_32_bits(reg_list_mc_addr);
4247                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4248                         reg_list_hdr_blk_index++;
4249                         for (i = 0; cs_data[i].section != NULL; i++) {
4250                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4251                                         reg_num = cs_data[i].section[j].reg_count;
4252                                         data = reg_list_mc_addr & 0xffffffff;
4253                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4254                                         reg_list_hdr_blk_index++;
4255
4256                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4257                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4258                                         reg_list_hdr_blk_index++;
4259
4260                                         data = 0x08000000 | (reg_num * 4);
4261                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4262                                         reg_list_hdr_blk_index++;
4263
4264                                         for (k = 0; k < reg_num; k++) {
4265                                                 data = cs_data[i].section[j].extent[k];
4266                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4267                                         }
4268                                         reg_list_mc_addr += reg_num * 4;
4269                                         reg_list_blk_index += reg_num;
4270                                 }
4271                         }
4272                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4273                 }
4274                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4275                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4276         }
4277
4278         if (rdev->rlc.cp_table_size) {
4279                 if (rdev->rlc.cp_table_obj == NULL) {
4280                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4281                                              PAGE_SIZE, true,
4282                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4283                                              NULL, &rdev->rlc.cp_table_obj);
4284                         if (r) {
4285                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4286                                 sumo_rlc_fini(rdev);
4287                                 return r;
4288                         }
4289                 }
4290
4291                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4292                 if (unlikely(r != 0)) {
4293                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4294                         sumo_rlc_fini(rdev);
4295                         return r;
4296                 }
4297                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4298                                   &rdev->rlc.cp_table_gpu_addr);
4299                 if (r) {
4300                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4301                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4302                         sumo_rlc_fini(rdev);
4303                         return r;
4304                 }
4305                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4306                 if (r) {
4307                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4308                         sumo_rlc_fini(rdev);
4309                         return r;
4310                 }
4311
4312                 cik_init_cp_pg_table(rdev);
4313
4314                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4315                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4316
4317         }
4318
4319         return 0;
4320 }
4321
4322 static void evergreen_rlc_start(struct radeon_device *rdev)
4323 {
4324         u32 mask = RLC_ENABLE;
4325
4326         if (rdev->flags & RADEON_IS_IGP) {
4327                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4328         }
4329
4330         WREG32(RLC_CNTL, mask);
4331 }
4332
4333 int evergreen_rlc_resume(struct radeon_device *rdev)
4334 {
4335         u32 i;
4336         const __be32 *fw_data;
4337
4338         if (!rdev->rlc_fw)
4339                 return -EINVAL;
4340
4341         r600_rlc_stop(rdev);
4342
4343         WREG32(RLC_HB_CNTL, 0);
4344
4345         if (rdev->flags & RADEON_IS_IGP) {
4346                 if (rdev->family == CHIP_ARUBA) {
4347                         u32 always_on_bitmap =
4348                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4349                         /* find out the number of active simds */
4350                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4351                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4352                         tmp = hweight32(~tmp);
4353                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4354                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4355                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4356                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4357                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4358                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4359                         }
4360                 } else {
4361                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4362                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4363                 }
4364                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4365                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4366         } else {
4367                 WREG32(RLC_HB_BASE, 0);
4368                 WREG32(RLC_HB_RPTR, 0);
4369                 WREG32(RLC_HB_WPTR, 0);
4370                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4371                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4372         }
4373         WREG32(RLC_MC_CNTL, 0);
4374         WREG32(RLC_UCODE_CNTL, 0);
4375
4376         fw_data = (const __be32 *)rdev->rlc_fw->data;
4377         if (rdev->family >= CHIP_ARUBA) {
4378                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4379                         WREG32(RLC_UCODE_ADDR, i);
4380                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4381                 }
4382         } else if (rdev->family >= CHIP_CAYMAN) {
4383                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4384                         WREG32(RLC_UCODE_ADDR, i);
4385                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4386                 }
4387         } else {
4388                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4389                         WREG32(RLC_UCODE_ADDR, i);
4390                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4391                 }
4392         }
4393         WREG32(RLC_UCODE_ADDR, 0);
4394
4395         evergreen_rlc_start(rdev);
4396
4397         return 0;
4398 }
4399
4400 /* Interrupts */
4401
4402 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4403 {
4404         if (crtc >= rdev->num_crtc)
4405                 return 0;
4406         else
4407                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4408 }
4409
4410 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4411 {
4412         u32 tmp;
4413
4414         if (rdev->family >= CHIP_CAYMAN) {
4415                 cayman_cp_int_cntl_setup(rdev, 0,
4416                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4417                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4418                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4419                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4420                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4421         } else
4422                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4423         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4424         WREG32(DMA_CNTL, tmp);
4425         WREG32(GRBM_INT_CNTL, 0);
4426         WREG32(SRBM_INT_CNTL, 0);
4427         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4428         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4429         if (rdev->num_crtc >= 4) {
4430                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4431                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4432         }
4433         if (rdev->num_crtc >= 6) {
4434                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4435                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4436         }
4437
4438         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4439         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4440         if (rdev->num_crtc >= 4) {
4441                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4442                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4443         }
4444         if (rdev->num_crtc >= 6) {
4445                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4446                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4447         }
4448
4449         /* only one DAC on DCE5 */
4450         if (!ASIC_IS_DCE5(rdev))
4451                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4452         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4453
4454         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4455         WREG32(DC_HPD1_INT_CONTROL, tmp);
4456         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4457         WREG32(DC_HPD2_INT_CONTROL, tmp);
4458         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4459         WREG32(DC_HPD3_INT_CONTROL, tmp);
4460         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4461         WREG32(DC_HPD4_INT_CONTROL, tmp);
4462         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4463         WREG32(DC_HPD5_INT_CONTROL, tmp);
4464         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4465         WREG32(DC_HPD6_INT_CONTROL, tmp);
4466
4467 }
4468
4469 int evergreen_irq_set(struct radeon_device *rdev)
4470 {
4471         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4472         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4473         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4474         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4475         u32 grbm_int_cntl = 0;
4476         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4477         u32 dma_cntl, dma_cntl1 = 0;
4478         u32 thermal_int = 0;
4479
4480         if (!rdev->irq.installed) {
4481                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4482                 return -EINVAL;
4483         }
4484         /* don't enable anything if the ih is disabled */
4485         if (!rdev->ih.enabled) {
4486                 r600_disable_interrupts(rdev);
4487                 /* force the active interrupt state to all disabled */
4488                 evergreen_disable_interrupt_state(rdev);
4489                 return 0;
4490         }
4491
4492         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4493         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4494         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4495         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4496         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4497         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4498         if (rdev->family == CHIP_ARUBA)
4499                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4500                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4501         else
4502                 thermal_int = RREG32(CG_THERMAL_INT) &
4503                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4504
4505         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4506         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4507         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4508         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4509         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4510         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4511
4512         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4513
4514         if (rdev->family >= CHIP_CAYMAN) {
4515                 /* enable CP interrupts on all rings */
4516                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4517                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4518                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4519                 }
4520                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4521                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4522                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4523                 }
4524                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4525                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4526                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4527                 }
4528         } else {
4529                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4530                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4531                         cp_int_cntl |= RB_INT_ENABLE;
4532                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4533                 }
4534         }
4535
4536         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4537                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4538                 dma_cntl |= TRAP_ENABLE;
4539         }
4540
4541         if (rdev->family >= CHIP_CAYMAN) {
4542                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4543                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4544                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4545                         dma_cntl1 |= TRAP_ENABLE;
4546                 }
4547         }
4548
4549         if (rdev->irq.dpm_thermal) {
4550                 DRM_DEBUG("dpm thermal\n");
4551                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4552         }
4553
4554         if (rdev->irq.crtc_vblank_int[0] ||
4555             atomic_read(&rdev->irq.pflip[0])) {
4556                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4557                 crtc1 |= VBLANK_INT_MASK;
4558         }
4559         if (rdev->irq.crtc_vblank_int[1] ||
4560             atomic_read(&rdev->irq.pflip[1])) {
4561                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4562                 crtc2 |= VBLANK_INT_MASK;
4563         }
4564         if (rdev->irq.crtc_vblank_int[2] ||
4565             atomic_read(&rdev->irq.pflip[2])) {
4566                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4567                 crtc3 |= VBLANK_INT_MASK;
4568         }
4569         if (rdev->irq.crtc_vblank_int[3] ||
4570             atomic_read(&rdev->irq.pflip[3])) {
4571                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4572                 crtc4 |= VBLANK_INT_MASK;
4573         }
4574         if (rdev->irq.crtc_vblank_int[4] ||
4575             atomic_read(&rdev->irq.pflip[4])) {
4576                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4577                 crtc5 |= VBLANK_INT_MASK;
4578         }
4579         if (rdev->irq.crtc_vblank_int[5] ||
4580             atomic_read(&rdev->irq.pflip[5])) {
4581                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4582                 crtc6 |= VBLANK_INT_MASK;
4583         }
4584         if (rdev->irq.hpd[0]) {
4585                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4586                 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4587         }
4588         if (rdev->irq.hpd[1]) {
4589                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4590                 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4591         }
4592         if (rdev->irq.hpd[2]) {
4593                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4594                 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4595         }
4596         if (rdev->irq.hpd[3]) {
4597                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4598                 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4599         }
4600         if (rdev->irq.hpd[4]) {
4601                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4602                 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4603         }
4604         if (rdev->irq.hpd[5]) {
4605                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4606                 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4607         }
4608         if (rdev->irq.afmt[0]) {
4609                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4610                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4611         }
4612         if (rdev->irq.afmt[1]) {
4613                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4614                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4615         }
4616         if (rdev->irq.afmt[2]) {
4617                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4618                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4619         }
4620         if (rdev->irq.afmt[3]) {
4621                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4622                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4623         }
4624         if (rdev->irq.afmt[4]) {
4625                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4626                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4627         }
4628         if (rdev->irq.afmt[5]) {
4629                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4630                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4631         }
4632
4633         if (rdev->family >= CHIP_CAYMAN) {
4634                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4635                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4636                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4637         } else
4638                 WREG32(CP_INT_CNTL, cp_int_cntl);
4639
4640         WREG32(DMA_CNTL, dma_cntl);
4641
4642         if (rdev->family >= CHIP_CAYMAN)
4643                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4644
4645         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4646
4647         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4648         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4649         if (rdev->num_crtc >= 4) {
4650                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4651                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4652         }
4653         if (rdev->num_crtc >= 6) {
4654                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4655                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4656         }
4657
4658         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4659                GRPH_PFLIP_INT_MASK);
4660         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4661                GRPH_PFLIP_INT_MASK);
4662         if (rdev->num_crtc >= 4) {
4663                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4664                        GRPH_PFLIP_INT_MASK);
4665                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4666                        GRPH_PFLIP_INT_MASK);
4667         }
4668         if (rdev->num_crtc >= 6) {
4669                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4670                        GRPH_PFLIP_INT_MASK);
4671                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4672                        GRPH_PFLIP_INT_MASK);
4673         }
4674
4675         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4676         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4677         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4678         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4679         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4680         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4681         if (rdev->family == CHIP_ARUBA)
4682                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4683         else
4684                 WREG32(CG_THERMAL_INT, thermal_int);
4685
4686         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4687         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4688         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4689         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4690         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4691         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4692
4693         /* posting read */
4694         RREG32(SRBM_STATUS);
4695
4696         return 0;
4697 }
4698
4699 static void evergreen_irq_ack(struct radeon_device *rdev)
4700 {
4701         u32 tmp;
4702
4703         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4704         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4705         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4706         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4707         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4708         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4709         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4710         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4711         if (rdev->num_crtc >= 4) {
4712                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4713                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4714         }
4715         if (rdev->num_crtc >= 6) {
4716                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4717                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4718         }
4719
4720         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4721         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4722         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4723         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4724         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4725         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4726
4727         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4728                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4729         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4730                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4731         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4732                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4733         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4734                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4735         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4736                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4737         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4738                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4739
4740         if (rdev->num_crtc >= 4) {
4741                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4742                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4743                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4744                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4745                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4746                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4747                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4748                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4749                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4750                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4751                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4752                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4753         }
4754
4755         if (rdev->num_crtc >= 6) {
4756                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4757                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4758                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4759                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4760                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4761                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4762                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4763                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4764                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4765                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4766                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4767                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4768         }
4769
4770         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4771                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4772                 tmp |= DC_HPDx_INT_ACK;
4773                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4774         }
4775         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4776                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4777                 tmp |= DC_HPDx_INT_ACK;
4778                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4779         }
4780         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4781                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4782                 tmp |= DC_HPDx_INT_ACK;
4783                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4784         }
4785         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4786                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4787                 tmp |= DC_HPDx_INT_ACK;
4788                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4789         }
4790         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4791                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4792                 tmp |= DC_HPDx_INT_ACK;
4793                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4794         }
4795         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4796                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4797                 tmp |= DC_HPDx_INT_ACK;
4798                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4799         }
4800
4801         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4802                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4803                 tmp |= DC_HPDx_RX_INT_ACK;
4804                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4805         }
4806         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4807                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4808                 tmp |= DC_HPDx_RX_INT_ACK;
4809                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4810         }
4811         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4812                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4813                 tmp |= DC_HPDx_RX_INT_ACK;
4814                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4815         }
4816         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4817                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4818                 tmp |= DC_HPDx_RX_INT_ACK;
4819                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4820         }
4821         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4822                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4823                 tmp |= DC_HPDx_RX_INT_ACK;
4824                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4825         }
4826         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4827                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4828                 tmp |= DC_HPDx_RX_INT_ACK;
4829                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4830         }
4831
4832         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4833                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4834                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4835                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4836         }
4837         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4838                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4839                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4840                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4841         }
4842         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4843                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4844                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4845                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4846         }
4847         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4848                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4849                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4850                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4851         }
4852         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4853                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4854                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4855                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4856         }
4857         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4858                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4859                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4860                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4861         }
4862 }
4863
4864 static void evergreen_irq_disable(struct radeon_device *rdev)
4865 {
4866         r600_disable_interrupts(rdev);
4867         /* Wait and acknowledge irq */
4868         mdelay(1);
4869         evergreen_irq_ack(rdev);
4870         evergreen_disable_interrupt_state(rdev);
4871 }
4872
4873 void evergreen_irq_suspend(struct radeon_device *rdev)
4874 {
4875         evergreen_irq_disable(rdev);
4876         r600_rlc_stop(rdev);
4877 }
4878
4879 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4880 {
4881         u32 wptr, tmp;
4882
4883         if (rdev->wb.enabled)
4884                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4885         else
4886                 wptr = RREG32(IH_RB_WPTR);
4887
4888         if (wptr & RB_OVERFLOW) {
4889                 wptr &= ~RB_OVERFLOW;
4890                 /* When a ring buffer overflow happen start parsing interrupt
4891                  * from the last not overwritten vector (wptr + 16). Hopefully
4892                  * this should allow us to catchup.
4893                  */
4894                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4895                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4896                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4897                 tmp = RREG32(IH_RB_CNTL);
4898                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4899                 WREG32(IH_RB_CNTL, tmp);
4900         }
4901         return (wptr & rdev->ih.ptr_mask);
4902 }
4903
4904 int evergreen_irq_process(struct radeon_device *rdev)
4905 {
4906         u32 wptr;
4907         u32 rptr;
4908         u32 src_id, src_data;
4909         u32 ring_index;
4910         bool queue_hotplug = false;
4911         bool queue_hdmi = false;
4912         bool queue_dp = false;
4913         bool queue_thermal = false;
4914         u32 status, addr;
4915
4916         if (!rdev->ih.enabled || rdev->shutdown)
4917                 return IRQ_NONE;
4918
4919         wptr = evergreen_get_ih_wptr(rdev);
4920
4921 restart_ih:
4922         /* is somebody else already processing irqs? */
4923         if (atomic_xchg(&rdev->ih.lock, 1))
4924                 return IRQ_NONE;
4925
4926         rptr = rdev->ih.rptr;
4927         DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4928
4929         /* Order reading of wptr vs. reading of IH ring data */
4930         rmb();
4931
4932         /* display interrupts */
4933         evergreen_irq_ack(rdev);
4934
4935         while (rptr != wptr) {
4936                 /* wptr/rptr are in bytes! */
4937                 ring_index = rptr / 4;
4938                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4939                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4940
4941                 switch (src_id) {
4942                 case 1: /* D1 vblank/vline */
4943                         switch (src_data) {
4944                         case 0: /* D1 vblank */
4945                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
4946                                         DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
4947
4948                                 if (rdev->irq.crtc_vblank_int[0]) {
4949                                         drm_handle_vblank(rdev->ddev, 0);
4950                                         rdev->pm.vblank_sync = true;
4951                                         wake_up(&rdev->irq.vblank_queue);
4952                                 }
4953                                 if (atomic_read(&rdev->irq.pflip[0]))
4954                                         radeon_crtc_handle_vblank(rdev, 0);
4955                                 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4956                                 DRM_DEBUG("IH: D1 vblank\n");
4957
4958                                 break;
4959                         case 1: /* D1 vline */
4960                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
4961                                         DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
4962
4963                                 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4964                                 DRM_DEBUG("IH: D1 vline\n");
4965
4966                                 break;
4967                         default:
4968                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4969                                 break;
4970                         }
4971                         break;
4972                 case 2: /* D2 vblank/vline */
4973                         switch (src_data) {
4974                         case 0: /* D2 vblank */
4975                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
4976                                         DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
4977
4978                                 if (rdev->irq.crtc_vblank_int[1]) {
4979                                         drm_handle_vblank(rdev->ddev, 1);
4980                                         rdev->pm.vblank_sync = true;
4981                                         wake_up(&rdev->irq.vblank_queue);
4982                                 }
4983                                 if (atomic_read(&rdev->irq.pflip[1]))
4984                                         radeon_crtc_handle_vblank(rdev, 1);
4985                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4986                                 DRM_DEBUG("IH: D2 vblank\n");
4987
4988                                 break;
4989                         case 1: /* D2 vline */
4990                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
4991                                         DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
4992
4993                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4994                                 DRM_DEBUG("IH: D2 vline\n");
4995
4996                                 break;
4997                         default:
4998                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4999                                 break;
5000                         }
5001                         break;
5002                 case 3: /* D3 vblank/vline */
5003                         switch (src_data) {
5004                         case 0: /* D3 vblank */
5005                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
5006                                         DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
5007
5008                                 if (rdev->irq.crtc_vblank_int[2]) {
5009                                         drm_handle_vblank(rdev->ddev, 2);
5010                                         rdev->pm.vblank_sync = true;
5011                                         wake_up(&rdev->irq.vblank_queue);
5012                                 }
5013                                 if (atomic_read(&rdev->irq.pflip[2]))
5014                                         radeon_crtc_handle_vblank(rdev, 2);
5015                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5016                                 DRM_DEBUG("IH: D3 vblank\n");
5017
5018                                 break;
5019                         case 1: /* D3 vline */
5020                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5021                                         DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5022
5023                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5024                                 DRM_DEBUG("IH: D3 vline\n");
5025
5026                                 break;
5027                         default:
5028                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5029                                 break;
5030                         }
5031                         break;
5032                 case 4: /* D4 vblank/vline */
5033                         switch (src_data) {
5034                         case 0: /* D4 vblank */
5035                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5036                                         DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5037
5038                                 if (rdev->irq.crtc_vblank_int[3]) {
5039                                         drm_handle_vblank(rdev->ddev, 3);
5040                                         rdev->pm.vblank_sync = true;
5041                                         wake_up(&rdev->irq.vblank_queue);
5042                                 }
5043                                 if (atomic_read(&rdev->irq.pflip[3]))
5044                                         radeon_crtc_handle_vblank(rdev, 3);
5045                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5046                                 DRM_DEBUG("IH: D4 vblank\n");
5047
5048                                 break;
5049                         case 1: /* D4 vline */
5050                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5051                                         DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5052
5053                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5054                                 DRM_DEBUG("IH: D4 vline\n");
5055
5056                                 break;
5057                         default:
5058                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5059                                 break;
5060                         }
5061                         break;
5062                 case 5: /* D5 vblank/vline */
5063                         switch (src_data) {
5064                         case 0: /* D5 vblank */
5065                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5066                                         DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5067
5068                                 if (rdev->irq.crtc_vblank_int[4]) {
5069                                         drm_handle_vblank(rdev->ddev, 4);
5070                                         rdev->pm.vblank_sync = true;
5071                                         wake_up(&rdev->irq.vblank_queue);
5072                                 }
5073                                 if (atomic_read(&rdev->irq.pflip[4]))
5074                                         radeon_crtc_handle_vblank(rdev, 4);
5075                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5076                                 DRM_DEBUG("IH: D5 vblank\n");
5077
5078                                 break;
5079                         case 1: /* D5 vline */
5080                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5081                                         DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5082
5083                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5084                                 DRM_DEBUG("IH: D5 vline\n");
5085
5086                                 break;
5087                         default:
5088                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5089                                 break;
5090                         }
5091                         break;
5092                 case 6: /* D6 vblank/vline */
5093                         switch (src_data) {
5094                         case 0: /* D6 vblank */
5095                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5096                                         DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5097
5098                                 if (rdev->irq.crtc_vblank_int[5]) {
5099                                         drm_handle_vblank(rdev->ddev, 5);
5100                                         rdev->pm.vblank_sync = true;
5101                                         wake_up(&rdev->irq.vblank_queue);
5102                                 }
5103                                 if (atomic_read(&rdev->irq.pflip[5]))
5104                                         radeon_crtc_handle_vblank(rdev, 5);
5105                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5106                                 DRM_DEBUG("IH: D6 vblank\n");
5107
5108                                 break;
5109                         case 1: /* D6 vline */
5110                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5111                                         DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5112
5113                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5114                                 DRM_DEBUG("IH: D6 vline\n");
5115
5116                                 break;
5117                         default:
5118                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5119                                 break;
5120                         }
5121                         break;
5122                 case 8: /* D1 page flip */
5123                 case 10: /* D2 page flip */
5124                 case 12: /* D3 page flip */
5125                 case 14: /* D4 page flip */
5126                 case 16: /* D5 page flip */
5127                 case 18: /* D6 page flip */
5128                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5129                         if (radeon_use_pflipirq > 0)
5130                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5131                         break;
5132                 case 42: /* HPD hotplug */
5133                         switch (src_data) {
5134                         case 0:
5135                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5136                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5137
5138                                 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5139                                 queue_hotplug = true;
5140                                 DRM_DEBUG("IH: HPD1\n");
5141                                 break;
5142                         case 1:
5143                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5144                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5145
5146                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5147                                 queue_hotplug = true;
5148                                 DRM_DEBUG("IH: HPD2\n");
5149                                 break;
5150                         case 2:
5151                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5152                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5153
5154                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5155                                 queue_hotplug = true;
5156                                 DRM_DEBUG("IH: HPD3\n");
5157                                 break;
5158                         case 3:
5159                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5160                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5161
5162                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5163                                 queue_hotplug = true;
5164                                 DRM_DEBUG("IH: HPD4\n");
5165                                 break;
5166                         case 4:
5167                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5168                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5169
5170                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5171                                 queue_hotplug = true;
5172                                 DRM_DEBUG("IH: HPD5\n");
5173                                 break;
5174                         case 5:
5175                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5176                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5177
5178                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5179                                 queue_hotplug = true;
5180                                 DRM_DEBUG("IH: HPD6\n");
5181                                 break;
5182                         case 6:
5183                                 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5184                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5185
5186                                 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5187                                 queue_dp = true;
5188                                 DRM_DEBUG("IH: HPD_RX 1\n");
5189                                 break;
5190                         case 7:
5191                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5192                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5193
5194                                 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5195                                 queue_dp = true;
5196                                 DRM_DEBUG("IH: HPD_RX 2\n");
5197                                 break;
5198                         case 8:
5199                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5200                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5201
5202                                 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5203                                 queue_dp = true;
5204                                 DRM_DEBUG("IH: HPD_RX 3\n");
5205                                 break;
5206                         case 9:
5207                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5208                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5209
5210                                 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5211                                 queue_dp = true;
5212                                 DRM_DEBUG("IH: HPD_RX 4\n");
5213                                 break;
5214                         case 10:
5215                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5216                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5217
5218                                 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5219                                 queue_dp = true;
5220                                 DRM_DEBUG("IH: HPD_RX 5\n");
5221                                 break;
5222                         case 11:
5223                                 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5224                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5225
5226                                 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5227                                 queue_dp = true;
5228                                 DRM_DEBUG("IH: HPD_RX 6\n");
5229                                 break;
5230                         default:
5231                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5232                                 break;
5233                         }
5234                         break;
5235                 case 44: /* hdmi */
5236                         switch (src_data) {
5237                         case 0:
5238                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5239                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5240
5241                                 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5242                                 queue_hdmi = true;
5243                                 DRM_DEBUG("IH: HDMI0\n");
5244                                 break;
5245                         case 1:
5246                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5247                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5248
5249                                 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5250                                 queue_hdmi = true;
5251                                 DRM_DEBUG("IH: HDMI1\n");
5252                                 break;
5253                         case 2:
5254                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5255                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5256
5257                                 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5258                                 queue_hdmi = true;
5259                                 DRM_DEBUG("IH: HDMI2\n");
5260                                 break;
5261                         case 3:
5262                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5263                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5264
5265                                 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5266                                 queue_hdmi = true;
5267                                 DRM_DEBUG("IH: HDMI3\n");
5268                                 break;
5269                         case 4:
5270                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5271                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5272
5273                                 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5274                                 queue_hdmi = true;
5275                                 DRM_DEBUG("IH: HDMI4\n");
5276                                 break;
5277                         case 5:
5278                                 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5279                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5280
5281                                 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5282                                 queue_hdmi = true;
5283                                 DRM_DEBUG("IH: HDMI5\n");
5284                                 break;
5285                         default:
5286                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5287                                 break;
5288                         }
5289                 case 96:
5290                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5291                         WREG32(SRBM_INT_ACK, 0x1);
5292                         break;
5293                 case 124: /* UVD */
5294                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5295                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5296                         break;
5297                 case 146:
5298                 case 147:
5299                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5300                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5301                         /* reset addr and status */
5302                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5303                         if (addr == 0x0 && status == 0x0)
5304                                 break;
5305                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5306                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5307                                 addr);
5308                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5309                                 status);
5310                         cayman_vm_decode_fault(rdev, status, addr);
5311                         break;
5312                 case 176: /* CP_INT in ring buffer */
5313                 case 177: /* CP_INT in IB1 */
5314                 case 178: /* CP_INT in IB2 */
5315                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5316                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5317                         break;
5318                 case 181: /* CP EOP event */
5319                         DRM_DEBUG("IH: CP EOP\n");
5320                         if (rdev->family >= CHIP_CAYMAN) {
5321                                 switch (src_data) {
5322                                 case 0:
5323                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5324                                         break;
5325                                 case 1:
5326                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5327                                         break;
5328                                 case 2:
5329                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5330                                         break;
5331                                 }
5332                         } else
5333                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5334                         break;
5335                 case 224: /* DMA trap event */
5336                         DRM_DEBUG("IH: DMA trap\n");
5337                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5338                         break;
5339                 case 230: /* thermal low to high */
5340                         DRM_DEBUG("IH: thermal low to high\n");
5341                         rdev->pm.dpm.thermal.high_to_low = false;
5342                         queue_thermal = true;
5343                         break;
5344                 case 231: /* thermal high to low */
5345                         DRM_DEBUG("IH: thermal high to low\n");
5346                         rdev->pm.dpm.thermal.high_to_low = true;
5347                         queue_thermal = true;
5348                         break;
5349                 case 233: /* GUI IDLE */
5350                         DRM_DEBUG("IH: GUI idle\n");
5351                         break;
5352                 case 244: /* DMA trap event */
5353                         if (rdev->family >= CHIP_CAYMAN) {
5354                                 DRM_DEBUG("IH: DMA1 trap\n");
5355                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5356                         }
5357                         break;
5358                 default:
5359                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5360                         break;
5361                 }
5362
5363                 /* wptr/rptr are in bytes! */
5364                 rptr += 16;
5365                 rptr &= rdev->ih.ptr_mask;
5366                 WREG32(IH_RB_RPTR, rptr);
5367         }
5368         if (queue_dp)
5369                 schedule_work(&rdev->dp_work);
5370         if (queue_hotplug)
5371                 schedule_work(&rdev->hotplug_work);
5372         if (queue_hdmi)
5373                 schedule_work(&rdev->audio_work);
5374         if (queue_thermal && rdev->pm.dpm_enabled)
5375                 schedule_work(&rdev->pm.dpm.thermal.work);
5376         rdev->ih.rptr = rptr;
5377         atomic_set(&rdev->ih.lock, 0);
5378
5379         /* make sure wptr hasn't changed while processing */
5380         wptr = evergreen_get_ih_wptr(rdev);
5381         if (wptr != rptr)
5382                 goto restart_ih;
5383
5384         return IRQ_HANDLED;
5385 }
5386
5387 static int evergreen_startup(struct radeon_device *rdev)
5388 {
5389         struct radeon_ring *ring;
5390         int r;
5391
5392         /* enable pcie gen2 link */
5393         evergreen_pcie_gen2_enable(rdev);
5394         /* enable aspm */
5395         evergreen_program_aspm(rdev);
5396
5397         /* scratch needs to be initialized before MC */
5398         r = r600_vram_scratch_init(rdev);
5399         if (r)
5400                 return r;
5401
5402         evergreen_mc_program(rdev);
5403
5404         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5405                 r = ni_mc_load_microcode(rdev);
5406                 if (r) {
5407                         DRM_ERROR("Failed to load MC firmware!\n");
5408                         return r;
5409                 }
5410         }
5411
5412         if (rdev->flags & RADEON_IS_AGP) {
5413                 evergreen_agp_enable(rdev);
5414         } else {
5415                 r = evergreen_pcie_gart_enable(rdev);
5416                 if (r)
5417                         return r;
5418         }
5419         evergreen_gpu_init(rdev);
5420
5421         /* allocate rlc buffers */
5422         if (rdev->flags & RADEON_IS_IGP) {
5423                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5424                 rdev->rlc.reg_list_size =
5425                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5426                 rdev->rlc.cs_data = evergreen_cs_data;
5427                 r = sumo_rlc_init(rdev);
5428                 if (r) {
5429                         DRM_ERROR("Failed to init rlc BOs!\n");
5430                         return r;
5431                 }
5432         }
5433
5434         /* allocate wb buffer */
5435         r = radeon_wb_init(rdev);
5436         if (r)
5437                 return r;
5438
5439         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5440         if (r) {
5441                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5442                 return r;
5443         }
5444
5445         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5446         if (r) {
5447                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5448                 return r;
5449         }
5450
5451         r = uvd_v2_2_resume(rdev);
5452         if (!r) {
5453                 r = radeon_fence_driver_start_ring(rdev,
5454                                                    R600_RING_TYPE_UVD_INDEX);
5455                 if (r)
5456                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5457         }
5458
5459         if (r)
5460                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5461
5462         /* Enable IRQ */
5463         if (!rdev->irq.installed) {
5464                 r = radeon_irq_kms_init(rdev);
5465                 if (r)
5466                         return r;
5467         }
5468
5469         r = r600_irq_init(rdev);
5470         if (r) {
5471                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5472                 radeon_irq_kms_fini(rdev);
5473                 return r;
5474         }
5475         evergreen_irq_set(rdev);
5476
5477         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5478         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5479                              RADEON_CP_PACKET2);
5480         if (r)
5481                 return r;
5482
5483         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5484         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5485                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5486         if (r)
5487                 return r;
5488
5489         r = evergreen_cp_load_microcode(rdev);
5490         if (r)
5491                 return r;
5492         r = evergreen_cp_resume(rdev);
5493         if (r)
5494                 return r;
5495         r = r600_dma_resume(rdev);
5496         if (r)
5497                 return r;
5498
5499         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5500         if (ring->ring_size) {
5501                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5502                                      RADEON_CP_PACKET2);
5503                 if (!r)
5504                         r = uvd_v1_0_init(rdev);
5505
5506                 if (r)
5507                         DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5508         }
5509
5510         r = radeon_ib_pool_init(rdev);
5511         if (r) {
5512                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5513                 return r;
5514         }
5515
5516         r = radeon_audio_init(rdev);
5517         if (r) {
5518                 DRM_ERROR("radeon: audio init failed\n");
5519                 return r;
5520         }
5521
5522         return 0;
5523 }
5524
5525 int evergreen_resume(struct radeon_device *rdev)
5526 {
5527         int r;
5528
5529         /* reset the asic, the gfx blocks are often in a bad state
5530          * after the driver is unloaded or after a resume
5531          */
5532         if (radeon_asic_reset(rdev))
5533                 dev_warn(rdev->dev, "GPU reset failed !\n");
5534         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5535          * posting will perform necessary task to bring back GPU into good
5536          * shape.
5537          */
5538         /* post card */
5539         atom_asic_init(rdev->mode_info.atom_context);
5540
5541         /* init golden registers */
5542         evergreen_init_golden_registers(rdev);
5543
5544         if (rdev->pm.pm_method == PM_METHOD_DPM)
5545                 radeon_pm_resume(rdev);
5546
5547         rdev->accel_working = true;
5548         r = evergreen_startup(rdev);
5549         if (r) {
5550                 DRM_ERROR("evergreen startup failed on resume\n");
5551                 rdev->accel_working = false;
5552                 return r;
5553         }
5554
5555         return r;
5556
5557 }
5558
5559 int evergreen_suspend(struct radeon_device *rdev)
5560 {
5561         radeon_pm_suspend(rdev);
5562         radeon_audio_fini(rdev);
5563         uvd_v1_0_fini(rdev);
5564         radeon_uvd_suspend(rdev);
5565         r700_cp_stop(rdev);
5566         r600_dma_stop(rdev);
5567         evergreen_irq_suspend(rdev);
5568         radeon_wb_disable(rdev);
5569         evergreen_pcie_gart_disable(rdev);
5570
5571         return 0;
5572 }
5573
5574 /* Plan is to move initialization in that function and use
5575  * helper function so that radeon_device_init pretty much
5576  * do nothing more than calling asic specific function. This
5577  * should also allow to remove a bunch of callback function
5578  * like vram_info.
5579  */
5580 int evergreen_init(struct radeon_device *rdev)
5581 {
5582         int r;
5583
5584         /* Read BIOS */
5585         if (!radeon_get_bios(rdev)) {
5586                 if (ASIC_IS_AVIVO(rdev))
5587                         return -EINVAL;
5588         }
5589         /* Must be an ATOMBIOS */
5590         if (!rdev->is_atom_bios) {
5591                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5592                 return -EINVAL;
5593         }
5594         r = radeon_atombios_init(rdev);
5595         if (r)
5596                 return r;
5597         /* reset the asic, the gfx blocks are often in a bad state
5598          * after the driver is unloaded or after a resume
5599          */
5600         if (radeon_asic_reset(rdev))
5601                 dev_warn(rdev->dev, "GPU reset failed !\n");
5602         /* Post card if necessary */
5603         if (!radeon_card_posted(rdev)) {
5604                 if (!rdev->bios) {
5605                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5606                         return -EINVAL;
5607                 }
5608                 DRM_INFO("GPU not posted. posting now...\n");
5609                 atom_asic_init(rdev->mode_info.atom_context);
5610         }
5611         /* init golden registers */
5612         evergreen_init_golden_registers(rdev);
5613         /* Initialize scratch registers */
5614         r600_scratch_init(rdev);
5615         /* Initialize surface registers */
5616         radeon_surface_init(rdev);
5617         /* Initialize clocks */
5618         radeon_get_clock_info(rdev->ddev);
5619         /* Fence driver */
5620         r = radeon_fence_driver_init(rdev);
5621         if (r)
5622                 return r;
5623         /* initialize AGP */
5624         if (rdev->flags & RADEON_IS_AGP) {
5625                 r = radeon_agp_init(rdev);
5626                 if (r)
5627                         radeon_agp_disable(rdev);
5628         }
5629         /* initialize memory controller */
5630         r = evergreen_mc_init(rdev);
5631         if (r)
5632                 return r;
5633         /* Memory manager */
5634         r = radeon_bo_init(rdev);
5635         if (r)
5636                 return r;
5637
5638         if (ASIC_IS_DCE5(rdev)) {
5639                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5640                         r = ni_init_microcode(rdev);
5641                         if (r) {
5642                                 DRM_ERROR("Failed to load firmware!\n");
5643                                 return r;
5644                         }
5645                 }
5646         } else {
5647                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5648                         r = r600_init_microcode(rdev);
5649                         if (r) {
5650                                 DRM_ERROR("Failed to load firmware!\n");
5651                                 return r;
5652                         }
5653                 }
5654         }
5655
5656         /* Initialize power management */
5657         radeon_pm_init(rdev);
5658
5659         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5660         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5661
5662         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5663         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5664
5665         r = radeon_uvd_init(rdev);
5666         if (!r) {
5667                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5668                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5669                                4096);
5670         }
5671
5672         rdev->ih.ring_obj = NULL;
5673         r600_ih_ring_init(rdev, 64 * 1024);
5674
5675         r = r600_pcie_gart_init(rdev);
5676         if (r)
5677                 return r;
5678
5679         rdev->accel_working = true;
5680         r = evergreen_startup(rdev);
5681         if (r) {
5682                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5683                 r700_cp_fini(rdev);
5684                 r600_dma_fini(rdev);
5685                 r600_irq_fini(rdev);
5686                 if (rdev->flags & RADEON_IS_IGP)
5687                         sumo_rlc_fini(rdev);
5688                 radeon_wb_fini(rdev);
5689                 radeon_ib_pool_fini(rdev);
5690                 radeon_irq_kms_fini(rdev);
5691                 evergreen_pcie_gart_fini(rdev);
5692                 rdev->accel_working = false;
5693         }
5694
5695         /* Don't start up if the MC ucode is missing on BTC parts.
5696          * The default clocks and voltages before the MC ucode
5697          * is loaded are not suffient for advanced operations.
5698          */
5699         if (ASIC_IS_DCE5(rdev)) {
5700                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5701                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5702                         return -EINVAL;
5703                 }
5704         }
5705
5706         return 0;
5707 }
5708
5709 void evergreen_fini(struct radeon_device *rdev)
5710 {
5711         radeon_pm_fini(rdev);
5712         radeon_audio_fini(rdev);
5713         r700_cp_fini(rdev);
5714         r600_dma_fini(rdev);
5715         r600_irq_fini(rdev);
5716         if (rdev->flags & RADEON_IS_IGP)
5717                 sumo_rlc_fini(rdev);
5718         radeon_wb_fini(rdev);
5719         radeon_ib_pool_fini(rdev);
5720         radeon_irq_kms_fini(rdev);
5721         uvd_v1_0_fini(rdev);
5722         radeon_uvd_fini(rdev);
5723         evergreen_pcie_gart_fini(rdev);
5724         r600_vram_scratch_fini(rdev);
5725         radeon_gem_fini(rdev);
5726         radeon_fence_driver_fini(rdev);
5727         radeon_agp_fini(rdev);
5728         radeon_bo_fini(rdev);
5729         radeon_atombios_fini(rdev);
5730         kfree(rdev->bios);
5731         rdev->bios = NULL;
5732 }
5733
5734 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5735 {
5736         u32 link_width_cntl, speed_cntl;
5737
5738         if (radeon_pcie_gen2 == 0)
5739                 return;
5740
5741         if (rdev->flags & RADEON_IS_IGP)
5742                 return;
5743
5744         if (!(rdev->flags & RADEON_IS_PCIE))
5745                 return;
5746
5747         /* x2 cards have a special sequence */
5748         if (ASIC_IS_X2(rdev))
5749                 return;
5750
5751         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5752                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5753                 return;
5754
5755         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5756         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5757                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5758                 return;
5759         }
5760
5761         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5762
5763         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5764             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5765
5766                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5767                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5768                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5769
5770                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5771                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5772                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5773
5774                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5775                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5776                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5777
5778                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5779                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5780                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5781
5782                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5783                 speed_cntl |= LC_GEN2_EN_STRAP;
5784                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5785
5786         } else {
5787                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5788                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5789                 if (1)
5790                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5791                 else
5792                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5793                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5794         }
5795 }
5796
5797 void evergreen_program_aspm(struct radeon_device *rdev)
5798 {
5799         u32 data, orig;
5800         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5801         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5802         /* fusion_platform = true
5803          * if the system is a fusion system
5804          * (APU or DGPU in a fusion system).
5805          * todo: check if the system is a fusion platform.
5806          */
5807         bool fusion_platform = false;
5808
5809         if (radeon_aspm == 0)
5810                 return;
5811
5812         if (!(rdev->flags & RADEON_IS_PCIE))
5813                 return;
5814
5815         switch (rdev->family) {
5816         case CHIP_CYPRESS:
5817         case CHIP_HEMLOCK:
5818         case CHIP_JUNIPER:
5819         case CHIP_REDWOOD:
5820         case CHIP_CEDAR:
5821         case CHIP_SUMO:
5822         case CHIP_SUMO2:
5823         case CHIP_PALM:
5824         case CHIP_ARUBA:
5825                 disable_l0s = true;
5826                 break;
5827         default:
5828                 disable_l0s = false;
5829                 break;
5830         }
5831
5832         if (rdev->flags & RADEON_IS_IGP)
5833                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5834
5835         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5836         if (fusion_platform)
5837                 data &= ~MULTI_PIF;
5838         else
5839                 data |= MULTI_PIF;
5840         if (data != orig)
5841                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5842
5843         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5844         if (fusion_platform)
5845                 data &= ~MULTI_PIF;
5846         else
5847                 data |= MULTI_PIF;
5848         if (data != orig)
5849                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5850
5851         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5852         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5853         if (!disable_l0s) {
5854                 if (rdev->family >= CHIP_BARTS)
5855                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5856                 else
5857                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5858         }
5859
5860         if (!disable_l1) {
5861                 if (rdev->family >= CHIP_BARTS)
5862                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5863                 else
5864                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5865
5866                 if (!disable_plloff_in_l1) {
5867                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5868                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5869                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5870                         if (data != orig)
5871                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5872
5873                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5874                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5875                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5876                         if (data != orig)
5877                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5878
5879                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5880                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5881                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5882                         if (data != orig)
5883                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5884
5885                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5886                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5887                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5888                         if (data != orig)
5889                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5890
5891                         if (rdev->family >= CHIP_BARTS) {
5892                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5893                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5894                                 data |= PLL_RAMP_UP_TIME_0(4);
5895                                 if (data != orig)
5896                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5897
5898                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5899                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5900                                 data |= PLL_RAMP_UP_TIME_1(4);
5901                                 if (data != orig)
5902                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5903
5904                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5905                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5906                                 data |= PLL_RAMP_UP_TIME_0(4);
5907                                 if (data != orig)
5908                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5909
5910                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5911                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5912                                 data |= PLL_RAMP_UP_TIME_1(4);
5913                                 if (data != orig)
5914                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5915                         }
5916
5917                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5918                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5919                         data |= LC_DYN_LANES_PWR_STATE(3);
5920                         if (data != orig)
5921                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5922
5923                         if (rdev->family >= CHIP_BARTS) {
5924                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5925                                 data &= ~LS2_EXIT_TIME_MASK;
5926                                 data |= LS2_EXIT_TIME(1);
5927                                 if (data != orig)
5928                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5929
5930                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5931                                 data &= ~LS2_EXIT_TIME_MASK;
5932                                 data |= LS2_EXIT_TIME(1);
5933                                 if (data != orig)
5934                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5935                         }
5936                 }
5937         }
5938
5939         /* evergreen parts only */
5940         if (rdev->family < CHIP_BARTS)
5941                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5942
5943         if (pcie_lc_cntl != pcie_lc_cntl_old)
5944                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5945 }