drivers/gpu/drm/radeon/si.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "sid.h"
  32 #include "atom.h"
  33 #include "si_blit_shaders.h"
  34 #include "clearstate_si.h"
  35 #include "radeon_ucode.h"
  36
  37
  38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
  52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
  58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
  62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
  63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
  64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
  65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
  66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
  67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
  68
  69 static void si_pcie_gen3_enable(struct radeon_device *rdev);
  70 static void si_program_aspm(struct radeon_device *rdev);
  71 extern void sumo_rlc_fini(struct radeon_device *rdev);
  72 extern int sumo_rlc_init(struct radeon_device *rdev);
  73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
  76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
  79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
  80 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  81 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
  82                                          bool enable);
  83 static void si_fini_pg(struct radeon_device *rdev);
  84 static void si_fini_cg(struct radeon_device *rdev);
  85 static void si_rlc_stop(struct radeon_device *rdev);
  86
  87 static const u32 verde_rlc_save_restore_register_list[] =
  88 {
  89         (0x8000 << 16) | (0x98f4 >> 2),
  90         0x00000000,
  91         (0x8040 << 16) | (0x98f4 >> 2),
  92         0x00000000,
  93         (0x8000 << 16) | (0xe80 >> 2),
  94         0x00000000,
  95         (0x8040 << 16) | (0xe80 >> 2),
  96         0x00000000,
  97         (0x8000 << 16) | (0x89bc >> 2),
  98         0x00000000,
  99         (0x8040 << 16) | (0x89bc >> 2),
 100         0x00000000,
 101         (0x8000 << 16) | (0x8c1c >> 2),
 102         0x00000000,
 103         (0x8040 << 16) | (0x8c1c >> 2),
 104         0x00000000,
 105         (0x9c00 << 16) | (0x98f0 >> 2),
 106         0x00000000,
 107         (0x9c00 << 16) | (0xe7c >> 2),
 108         0x00000000,
 109         (0x8000 << 16) | (0x9148 >> 2),
 110         0x00000000,
 111         (0x8040 << 16) | (0x9148 >> 2),
 112         0x00000000,
 113         (0x9c00 << 16) | (0x9150 >> 2),
 114         0x00000000,
 115         (0x9c00 << 16) | (0x897c >> 2),
 116         0x00000000,
 117         (0x9c00 << 16) | (0x8d8c >> 2),
 118         0x00000000,
 119         (0x9c00 << 16) | (0xac54 >> 2),
 120         0X00000000,
 121         0x3,
 122         (0x9c00 << 16) | (0x98f8 >> 2),
 123         0x00000000,
 124         (0x9c00 << 16) | (0x9910 >> 2),
 125         0x00000000,
 126         (0x9c00 << 16) | (0x9914 >> 2),
 127         0x00000000,
 128         (0x9c00 << 16) | (0x9918 >> 2),
 129         0x00000000,
 130         (0x9c00 << 16) | (0x991c >> 2),
 131         0x00000000,
 132         (0x9c00 << 16) | (0x9920 >> 2),
 133         0x00000000,
 134         (0x9c00 << 16) | (0x9924 >> 2),
 135         0x00000000,
 136         (0x9c00 << 16) | (0x9928 >> 2),
 137         0x00000000,
 138         (0x9c00 << 16) | (0x992c >> 2),
 139         0x00000000,
 140         (0x9c00 << 16) | (0x9930 >> 2),
 141         0x00000000,
 142         (0x9c00 << 16) | (0x9934 >> 2),
 143         0x00000000,
 144         (0x9c00 << 16) | (0x9938 >> 2),
 145         0x00000000,
 146         (0x9c00 << 16) | (0x993c >> 2),
 147         0x00000000,
 148         (0x9c00 << 16) | (0x9940 >> 2),
 149         0x00000000,
 150         (0x9c00 << 16) | (0x9944 >> 2),
 151         0x00000000,
 152         (0x9c00 << 16) | (0x9948 >> 2),
 153         0x00000000,
 154         (0x9c00 << 16) | (0x994c >> 2),
 155         0x00000000,
 156         (0x9c00 << 16) | (0x9950 >> 2),
 157         0x00000000,
 158         (0x9c00 << 16) | (0x9954 >> 2),
 159         0x00000000,
 160         (0x9c00 << 16) | (0x9958 >> 2),
 161         0x00000000,
 162         (0x9c00 << 16) | (0x995c >> 2),
 163         0x00000000,
 164         (0x9c00 << 16) | (0x9960 >> 2),
 165         0x00000000,
 166         (0x9c00 << 16) | (0x9964 >> 2),
 167         0x00000000,
 168         (0x9c00 << 16) | (0x9968 >> 2),
 169         0x00000000,
 170         (0x9c00 << 16) | (0x996c >> 2),
 171         0x00000000,
 172         (0x9c00 << 16) | (0x9970 >> 2),
 173         0x00000000,
 174         (0x9c00 << 16) | (0x9974 >> 2),
 175         0x00000000,
 176         (0x9c00 << 16) | (0x9978 >> 2),
 177         0x00000000,
 178         (0x9c00 << 16) | (0x997c >> 2),
 179         0x00000000,
 180         (0x9c00 << 16) | (0x9980 >> 2),
 181         0x00000000,
 182         (0x9c00 << 16) | (0x9984 >> 2),
 183         0x00000000,
 184         (0x9c00 << 16) | (0x9988 >> 2),
 185         0x00000000,
 186         (0x9c00 << 16) | (0x998c >> 2),
 187         0x00000000,
 188         (0x9c00 << 16) | (0x8c00 >> 2),
 189         0x00000000,
 190         (0x9c00 << 16) | (0x8c14 >> 2),
 191         0x00000000,
 192         (0x9c00 << 16) | (0x8c04 >> 2),
 193         0x00000000,
 194         (0x9c00 << 16) | (0x8c08 >> 2),
 195         0x00000000,
 196         (0x8000 << 16) | (0x9b7c >> 2),
 197         0x00000000,
 198         (0x8040 << 16) | (0x9b7c >> 2),
 199         0x00000000,
 200         (0x8000 << 16) | (0xe84 >> 2),
 201         0x00000000,
 202         (0x8040 << 16) | (0xe84 >> 2),
 203         0x00000000,
 204         (0x8000 << 16) | (0x89c0 >> 2),
 205         0x00000000,
 206         (0x8040 << 16) | (0x89c0 >> 2),
 207         0x00000000,
 208         (0x8000 << 16) | (0x914c >> 2),
 209         0x00000000,
 210         (0x8040 << 16) | (0x914c >> 2),
 211         0x00000000,
 212         (0x8000 << 16) | (0x8c20 >> 2),
 213         0x00000000,
 214         (0x8040 << 16) | (0x8c20 >> 2),
 215         0x00000000,
 216         (0x8000 << 16) | (0x9354 >> 2),
 217         0x00000000,
 218         (0x8040 << 16) | (0x9354 >> 2),
 219         0x00000000,
 220         (0x9c00 << 16) | (0x9060 >> 2),
 221         0x00000000,
 222         (0x9c00 << 16) | (0x9364 >> 2),
 223         0x00000000,
 224         (0x9c00 << 16) | (0x9100 >> 2),
 225         0x00000000,
 226         (0x9c00 << 16) | (0x913c >> 2),
 227         0x00000000,
 228         (0x8000 << 16) | (0x90e0 >> 2),
 229         0x00000000,
 230         (0x8000 << 16) | (0x90e4 >> 2),
 231         0x00000000,
 232         (0x8000 << 16) | (0x90e8 >> 2),
 233         0x00000000,
 234         (0x8040 << 16) | (0x90e0 >> 2),
 235         0x00000000,
 236         (0x8040 << 16) | (0x90e4 >> 2),
 237         0x00000000,
 238         (0x8040 << 16) | (0x90e8 >> 2),
 239         0x00000000,
 240         (0x9c00 << 16) | (0x8bcc >> 2),
 241         0x00000000,
 242         (0x9c00 << 16) | (0x8b24 >> 2),
 243         0x00000000,
 244         (0x9c00 << 16) | (0x88c4 >> 2),
 245         0x00000000,
 246         (0x9c00 << 16) | (0x8e50 >> 2),
 247         0x00000000,
 248         (0x9c00 << 16) | (0x8c0c >> 2),
 249         0x00000000,
 250         (0x9c00 << 16) | (0x8e58 >> 2),
 251         0x00000000,
 252         (0x9c00 << 16) | (0x8e5c >> 2),
 253         0x00000000,
 254         (0x9c00 << 16) | (0x9508 >> 2),
 255         0x00000000,
 256         (0x9c00 << 16) | (0x950c >> 2),
 257         0x00000000,
 258         (0x9c00 << 16) | (0x9494 >> 2),
 259         0x00000000,
 260         (0x9c00 << 16) | (0xac0c >> 2),
 261         0x00000000,
 262         (0x9c00 << 16) | (0xac10 >> 2),
 263         0x00000000,
 264         (0x9c00 << 16) | (0xac14 >> 2),
 265         0x00000000,
 266         (0x9c00 << 16) | (0xae00 >> 2),
 267         0x00000000,
 268         (0x9c00 << 16) | (0xac08 >> 2),
 269         0x00000000,
 270         (0x9c00 << 16) | (0x88d4 >> 2),
 271         0x00000000,
 272         (0x9c00 << 16) | (0x88c8 >> 2),
 273         0x00000000,
 274         (0x9c00 << 16) | (0x88cc >> 2),
 275         0x00000000,
 276         (0x9c00 << 16) | (0x89b0 >> 2),
 277         0x00000000,
 278         (0x9c00 << 16) | (0x8b10 >> 2),
 279         0x00000000,
 280         (0x9c00 << 16) | (0x8a14 >> 2),
 281         0x00000000,
 282         (0x9c00 << 16) | (0x9830 >> 2),
 283         0x00000000,
 284         (0x9c00 << 16) | (0x9834 >> 2),
 285         0x00000000,
 286         (0x9c00 << 16) | (0x9838 >> 2),
 287         0x00000000,
 288         (0x9c00 << 16) | (0x9a10 >> 2),
 289         0x00000000,
 290         (0x8000 << 16) | (0x9870 >> 2),
 291         0x00000000,
 292         (0x8000 << 16) | (0x9874 >> 2),
 293         0x00000000,
 294         (0x8001 << 16) | (0x9870 >> 2),
 295         0x00000000,
 296         (0x8001 << 16) | (0x9874 >> 2),
 297         0x00000000,
 298         (0x8040 << 16) | (0x9870 >> 2),
 299         0x00000000,
 300         (0x8040 << 16) | (0x9874 >> 2),
 301         0x00000000,
 302         (0x8041 << 16) | (0x9870 >> 2),
 303         0x00000000,
 304         (0x8041 << 16) | (0x9874 >> 2),
 305         0x00000000,
 306         0x00000000
 307 };
 308
 309 static const u32 tahiti_golden_rlc_registers[] =
 310 {
 311         0xc424, 0xffffffff, 0x00601005,
 312         0xc47c, 0xffffffff, 0x10104040,
 313         0xc488, 0xffffffff, 0x0100000a,
 314         0xc314, 0xffffffff, 0x00000800,
 315         0xc30c, 0xffffffff, 0x800000f4,
 316         0xf4a8, 0xffffffff, 0x00000000
 317 };
 318
 319 static const u32 tahiti_golden_registers[] =
 320 {
 321         0x9a10, 0x00010000, 0x00018208,
 322         0x9830, 0xffffffff, 0x00000000,
 323         0x9834, 0xf00fffff, 0x00000400,
 324         0x9838, 0x0002021c, 0x00020200,
 325         0xc78, 0x00000080, 0x00000000,
 326         0xd030, 0x000300c0, 0x00800040,
 327         0xd830, 0x000300c0, 0x00800040,
 328         0x5bb0, 0x000000f0, 0x00000070,
 329         0x5bc0, 0x00200000, 0x50100000,
 330         0x7030, 0x31000311, 0x00000011,
 331         0x277c, 0x00000003, 0x000007ff,
 332         0x240c, 0x000007ff, 0x00000000,
 333         0x8a14, 0xf000001f, 0x00000007,
 334         0x8b24, 0xffffffff, 0x00ffffff,
 335         0x8b10, 0x0000ff0f, 0x00000000,
 336         0x28a4c, 0x07ffffff, 0x4e000000,
 337         0x28350, 0x3f3f3fff, 0x2a00126a,
 338         0x30, 0x000000ff, 0x0040,
 339         0x34, 0x00000040, 0x00004040,
 340         0x9100, 0x07ffffff, 0x03000000,
 341         0x8e88, 0x01ff1f3f, 0x00000000,
 342         0x8e84, 0x01ff1f3f, 0x00000000,
 343         0x9060, 0x0000007f, 0x00000020,
 344         0x9508, 0x00010000, 0x00010000,
 345         0xac14, 0x00000200, 0x000002fb,
 346         0xac10, 0xffffffff, 0x0000543b,
 347         0xac0c, 0xffffffff, 0xa9210876,
 348         0x88d0, 0xffffffff, 0x000fff40,
 349         0x88d4, 0x0000001f, 0x00000010,
 350         0x1410, 0x20000000, 0x20fffed8,
 351         0x15c0, 0x000c0fc0, 0x000c0400
 352 };
 353
 354 static const u32 tahiti_golden_registers2[] =
 355 {
 356         0xc64, 0x00000001, 0x00000001
 357 };
 358
 359 static const u32 pitcairn_golden_rlc_registers[] =
 360 {
 361         0xc424, 0xffffffff, 0x00601004,
 362         0xc47c, 0xffffffff, 0x10102020,
 363         0xc488, 0xffffffff, 0x01000020,
 364         0xc314, 0xffffffff, 0x00000800,
 365         0xc30c, 0xffffffff, 0x800000a4
 366 };
 367
 368 static const u32 pitcairn_golden_registers[] =
 369 {
 370         0x9a10, 0x00010000, 0x00018208,
 371         0x9830, 0xffffffff, 0x00000000,
 372         0x9834, 0xf00fffff, 0x00000400,
 373         0x9838, 0x0002021c, 0x00020200,
 374         0xc78, 0x00000080, 0x00000000,
 375         0xd030, 0x000300c0, 0x00800040,
 376         0xd830, 0x000300c0, 0x00800040,
 377         0x5bb0, 0x000000f0, 0x00000070,
 378         0x5bc0, 0x00200000, 0x50100000,
 379         0x7030, 0x31000311, 0x00000011,
 380         0x2ae4, 0x00073ffe, 0x000022a2,
 381         0x240c, 0x000007ff, 0x00000000,
 382         0x8a14, 0xf000001f, 0x00000007,
 383         0x8b24, 0xffffffff, 0x00ffffff,
 384         0x8b10, 0x0000ff0f, 0x00000000,
 385         0x28a4c, 0x07ffffff, 0x4e000000,
 386         0x28350, 0x3f3f3fff, 0x2a00126a,
 387         0x30, 0x000000ff, 0x0040,
 388         0x34, 0x00000040, 0x00004040,
 389         0x9100, 0x07ffffff, 0x03000000,
 390         0x9060, 0x0000007f, 0x00000020,
 391         0x9508, 0x00010000, 0x00010000,
 392         0xac14, 0x000003ff, 0x000000f7,
 393         0xac10, 0xffffffff, 0x00000000,
 394         0xac0c, 0xffffffff, 0x32761054,
 395         0x88d4, 0x0000001f, 0x00000010,
 396         0x15c0, 0x000c0fc0, 0x000c0400
 397 };
 398
 399 static const u32 verde_golden_rlc_registers[] =
 400 {
 401         0xc424, 0xffffffff, 0x033f1005,
 402         0xc47c, 0xffffffff, 0x10808020,
 403         0xc488, 0xffffffff, 0x00800008,
 404         0xc314, 0xffffffff, 0x00001000,
 405         0xc30c, 0xffffffff, 0x80010014
 406 };
 407
 408 static const u32 verde_golden_registers[] =
 409 {
 410         0x9a10, 0x00010000, 0x00018208,
 411         0x9830, 0xffffffff, 0x00000000,
 412         0x9834, 0xf00fffff, 0x00000400,
 413         0x9838, 0x0002021c, 0x00020200,
 414         0xc78, 0x00000080, 0x00000000,
 415         0xd030, 0x000300c0, 0x00800040,
 416         0xd030, 0x000300c0, 0x00800040,
 417         0xd830, 0x000300c0, 0x00800040,
 418         0xd830, 0x000300c0, 0x00800040,
 419         0x5bb0, 0x000000f0, 0x00000070,
 420         0x5bc0, 0x00200000, 0x50100000,
 421         0x7030, 0x31000311, 0x00000011,
 422         0x2ae4, 0x00073ffe, 0x000022a2,
 423         0x2ae4, 0x00073ffe, 0x000022a2,
 424         0x2ae4, 0x00073ffe, 0x000022a2,
 425         0x240c, 0x000007ff, 0x00000000,
 426         0x240c, 0x000007ff, 0x00000000,
 427         0x240c, 0x000007ff, 0x00000000,
 428         0x8a14, 0xf000001f, 0x00000007,
 429         0x8a14, 0xf000001f, 0x00000007,
 430         0x8a14, 0xf000001f, 0x00000007,
 431         0x8b24, 0xffffffff, 0x00ffffff,
 432         0x8b10, 0x0000ff0f, 0x00000000,
 433         0x28a4c, 0x07ffffff, 0x4e000000,
 434         0x28350, 0x3f3f3fff, 0x0000124a,
 435         0x28350, 0x3f3f3fff, 0x0000124a,
 436         0x28350, 0x3f3f3fff, 0x0000124a,
 437         0x30, 0x000000ff, 0x0040,
 438         0x34, 0x00000040, 0x00004040,
 439         0x9100, 0x07ffffff, 0x03000000,
 440         0x9100, 0x07ffffff, 0x03000000,
 441         0x8e88, 0x01ff1f3f, 0x00000000,
 442         0x8e88, 0x01ff1f3f, 0x00000000,
 443         0x8e88, 0x01ff1f3f, 0x00000000,
 444         0x8e84, 0x01ff1f3f, 0x00000000,
 445         0x8e84, 0x01ff1f3f, 0x00000000,
 446         0x8e84, 0x01ff1f3f, 0x00000000,
 447         0x9060, 0x0000007f, 0x00000020,
 448         0x9508, 0x00010000, 0x00010000,
 449         0xac14, 0x000003ff, 0x00000003,
 450         0xac14, 0x000003ff, 0x00000003,
 451         0xac14, 0x000003ff, 0x00000003,
 452         0xac10, 0xffffffff, 0x00000000,
 453         0xac10, 0xffffffff, 0x00000000,
 454         0xac10, 0xffffffff, 0x00000000,
 455         0xac0c, 0xffffffff, 0x00001032,
 456         0xac0c, 0xffffffff, 0x00001032,
 457         0xac0c, 0xffffffff, 0x00001032,
 458         0x88d4, 0x0000001f, 0x00000010,
 459         0x88d4, 0x0000001f, 0x00000010,
 460         0x88d4, 0x0000001f, 0x00000010,
 461         0x15c0, 0x000c0fc0, 0x000c0400
 462 };
 463
 464 static const u32 oland_golden_rlc_registers[] =
 465 {
 466         0xc424, 0xffffffff, 0x00601005,
 467         0xc47c, 0xffffffff, 0x10104040,
 468         0xc488, 0xffffffff, 0x0100000a,
 469         0xc314, 0xffffffff, 0x00000800,
 470         0xc30c, 0xffffffff, 0x800000f4
 471 };
 472
 473 static const u32 oland_golden_registers[] =
 474 {
 475         0x9a10, 0x00010000, 0x00018208,
 476         0x9830, 0xffffffff, 0x00000000,
 477         0x9834, 0xf00fffff, 0x00000400,
 478         0x9838, 0x0002021c, 0x00020200,
 479         0xc78, 0x00000080, 0x00000000,
 480         0xd030, 0x000300c0, 0x00800040,
 481         0xd830, 0x000300c0, 0x00800040,
 482         0x5bb0, 0x000000f0, 0x00000070,
 483         0x5bc0, 0x00200000, 0x50100000,
 484         0x7030, 0x31000311, 0x00000011,
 485         0x2ae4, 0x00073ffe, 0x000022a2,
 486         0x240c, 0x000007ff, 0x00000000,
 487         0x8a14, 0xf000001f, 0x00000007,
 488         0x8b24, 0xffffffff, 0x00ffffff,
 489         0x8b10, 0x0000ff0f, 0x00000000,
 490         0x28a4c, 0x07ffffff, 0x4e000000,
 491         0x28350, 0x3f3f3fff, 0x00000082,
 492         0x30, 0x000000ff, 0x0040,
 493         0x34, 0x00000040, 0x00004040,
 494         0x9100, 0x07ffffff, 0x03000000,
 495         0x9060, 0x0000007f, 0x00000020,
 496         0x9508, 0x00010000, 0x00010000,
 497         0xac14, 0x000003ff, 0x000000f3,
 498         0xac10, 0xffffffff, 0x00000000,
 499         0xac0c, 0xffffffff, 0x00003210,
 500         0x88d4, 0x0000001f, 0x00000010,
 501         0x15c0, 0x000c0fc0, 0x000c0400
 502 };
 503
 504 static const u32 hainan_golden_registers[] =
 505 {
 506         0x9a10, 0x00010000, 0x00018208,
 507         0x9830, 0xffffffff, 0x00000000,
 508         0x9834, 0xf00fffff, 0x00000400,
 509         0x9838, 0x0002021c, 0x00020200,
 510         0xd0c0, 0xff000fff, 0x00000100,
 511         0xd030, 0x000300c0, 0x00800040,
 512         0xd8c0, 0xff000fff, 0x00000100,
 513         0xd830, 0x000300c0, 0x00800040,
 514         0x2ae4, 0x00073ffe, 0x000022a2,
 515         0x240c, 0x000007ff, 0x00000000,
 516         0x8a14, 0xf000001f, 0x00000007,
 517         0x8b24, 0xffffffff, 0x00ffffff,
 518         0x8b10, 0x0000ff0f, 0x00000000,
 519         0x28a4c, 0x07ffffff, 0x4e000000,
 520         0x28350, 0x3f3f3fff, 0x00000000,
 521         0x30, 0x000000ff, 0x0040,
 522         0x34, 0x00000040, 0x00004040,
 523         0x9100, 0x03e00000, 0x03600000,
 524         0x9060, 0x0000007f, 0x00000020,
 525         0x9508, 0x00010000, 0x00010000,
 526         0xac14, 0x000003ff, 0x000000f1,
 527         0xac10, 0xffffffff, 0x00000000,
 528         0xac0c, 0xffffffff, 0x00003210,
 529         0x88d4, 0x0000001f, 0x00000010,
 530         0x15c0, 0x000c0fc0, 0x000c0400
 531 };
 532
 533 static const u32 hainan_golden_registers2[] =
 534 {
 535         0x98f8, 0xffffffff, 0x02010001
 536 };
 537
 538 static const u32 tahiti_mgcg_cgcg_init[] =
 539 {
 540         0xc400, 0xffffffff, 0xfffffffc,
 541         0x802c, 0xffffffff, 0xe0000000,
 542         0x9a60, 0xffffffff, 0x00000100,
 543         0x92a4, 0xffffffff, 0x00000100,
 544         0xc164, 0xffffffff, 0x00000100,
 545         0x9774, 0xffffffff, 0x00000100,
 546         0x8984, 0xffffffff, 0x06000100,
 547         0x8a18, 0xffffffff, 0x00000100,
 548         0x92a0, 0xffffffff, 0x00000100,
 549         0xc380, 0xffffffff, 0x00000100,
 550         0x8b28, 0xffffffff, 0x00000100,
 551         0x9144, 0xffffffff, 0x00000100,
 552         0x8d88, 0xffffffff, 0x00000100,
 553         0x8d8c, 0xffffffff, 0x00000100,
 554         0x9030, 0xffffffff, 0x00000100,
 555         0x9034, 0xffffffff, 0x00000100,
 556         0x9038, 0xffffffff, 0x00000100,
 557         0x903c, 0xffffffff, 0x00000100,
 558         0xad80, 0xffffffff, 0x00000100,
 559         0xac54, 0xffffffff, 0x00000100,
 560         0x897c, 0xffffffff, 0x06000100,
 561         0x9868, 0xffffffff, 0x00000100,
 562         0x9510, 0xffffffff, 0x00000100,
 563         0xaf04, 0xffffffff, 0x00000100,
 564         0xae04, 0xffffffff, 0x00000100,
 565         0x949c, 0xffffffff, 0x00000100,
 566         0x802c, 0xffffffff, 0xe0000000,
 567         0x9160, 0xffffffff, 0x00010000,
 568         0x9164, 0xffffffff, 0x00030002,
 569         0x9168, 0xffffffff, 0x00040007,
 570         0x916c, 0xffffffff, 0x00060005,
 571         0x9170, 0xffffffff, 0x00090008,
 572         0x9174, 0xffffffff, 0x00020001,
 573         0x9178, 0xffffffff, 0x00040003,
 574         0x917c, 0xffffffff, 0x00000007,
 575         0x9180, 0xffffffff, 0x00060005,
 576         0x9184, 0xffffffff, 0x00090008,
 577         0x9188, 0xffffffff, 0x00030002,
 578         0x918c, 0xffffffff, 0x00050004,
 579         0x9190, 0xffffffff, 0x00000008,
 580         0x9194, 0xffffffff, 0x00070006,
 581         0x9198, 0xffffffff, 0x000a0009,
 582         0x919c, 0xffffffff, 0x00040003,
 583         0x91a0, 0xffffffff, 0x00060005,
 584         0x91a4, 0xffffffff, 0x00000009,
 585         0x91a8, 0xffffffff, 0x00080007,
 586         0x91ac, 0xffffffff, 0x000b000a,
 587         0x91b0, 0xffffffff, 0x00050004,
 588         0x91b4, 0xffffffff, 0x00070006,
 589         0x91b8, 0xffffffff, 0x0008000b,
 590         0x91bc, 0xffffffff, 0x000a0009,
 591         0x91c0, 0xffffffff, 0x000d000c,
 592         0x91c4, 0xffffffff, 0x00060005,
 593         0x91c8, 0xffffffff, 0x00080007,
 594         0x91cc, 0xffffffff, 0x0000000b,
 595         0x91d0, 0xffffffff, 0x000a0009,
 596         0x91d4, 0xffffffff, 0x000d000c,
 597         0x91d8, 0xffffffff, 0x00070006,
 598         0x91dc, 0xffffffff, 0x00090008,
 599         0x91e0, 0xffffffff, 0x0000000c,
 600         0x91e4, 0xffffffff, 0x000b000a,
 601         0x91e8, 0xffffffff, 0x000e000d,
 602         0x91ec, 0xffffffff, 0x00080007,
 603         0x91f0, 0xffffffff, 0x000a0009,
 604         0x91f4, 0xffffffff, 0x0000000d,
 605         0x91f8, 0xffffffff, 0x000c000b,
 606         0x91fc, 0xffffffff, 0x000f000e,
 607         0x9200, 0xffffffff, 0x00090008,
 608         0x9204, 0xffffffff, 0x000b000a,
 609         0x9208, 0xffffffff, 0x000c000f,
 610         0x920c, 0xffffffff, 0x000e000d,
 611         0x9210, 0xffffffff, 0x00110010,
 612         0x9214, 0xffffffff, 0x000a0009,
 613         0x9218, 0xffffffff, 0x000c000b,
 614         0x921c, 0xffffffff, 0x0000000f,
 615         0x9220, 0xffffffff, 0x000e000d,
 616         0x9224, 0xffffffff, 0x00110010,
 617         0x9228, 0xffffffff, 0x000b000a,
 618         0x922c, 0xffffffff, 0x000d000c,
 619         0x9230, 0xffffffff, 0x00000010,
 620         0x9234, 0xffffffff, 0x000f000e,
 621         0x9238, 0xffffffff, 0x00120011,
 622         0x923c, 0xffffffff, 0x000c000b,
 623         0x9240, 0xffffffff, 0x000e000d,
 624         0x9244, 0xffffffff, 0x00000011,
 625         0x9248, 0xffffffff, 0x0010000f,
 626         0x924c, 0xffffffff, 0x00130012,
 627         0x9250, 0xffffffff, 0x000d000c,
 628         0x9254, 0xffffffff, 0x000f000e,
 629         0x9258, 0xffffffff, 0x00100013,
 630         0x925c, 0xffffffff, 0x00120011,
 631         0x9260, 0xffffffff, 0x00150014,
 632         0x9264, 0xffffffff, 0x000e000d,
 633         0x9268, 0xffffffff, 0x0010000f,
 634         0x926c, 0xffffffff, 0x00000013,
 635         0x9270, 0xffffffff, 0x00120011,
 636         0x9274, 0xffffffff, 0x00150014,
 637         0x9278, 0xffffffff, 0x000f000e,
 638         0x927c, 0xffffffff, 0x00110010,
 639         0x9280, 0xffffffff, 0x00000014,
 640         0x9284, 0xffffffff, 0x00130012,
 641         0x9288, 0xffffffff, 0x00160015,
 642         0x928c, 0xffffffff, 0x0010000f,
 643         0x9290, 0xffffffff, 0x00120011,
 644         0x9294, 0xffffffff, 0x00000015,
 645         0x9298, 0xffffffff, 0x00140013,
 646         0x929c, 0xffffffff, 0x00170016,
 647         0x9150, 0xffffffff, 0x96940200,
 648         0x8708, 0xffffffff, 0x00900100,
 649         0xc478, 0xffffffff, 0x00000080,
 650         0xc404, 0xffffffff, 0x0020003f,
 651         0x30, 0xffffffff, 0x0000001c,
 652         0x34, 0x000f0000, 0x000f0000,
 653         0x160c, 0xffffffff, 0x00000100,
 654         0x1024, 0xffffffff, 0x00000100,
 655         0x102c, 0x00000101, 0x00000000,
 656         0x20a8, 0xffffffff, 0x00000104,
 657         0x264c, 0x000c0000, 0x000c0000,
 658         0x2648, 0x000c0000, 0x000c0000,
 659         0x55e4, 0xff000fff, 0x00000100,
 660         0x55e8, 0x00000001, 0x00000001,
 661         0x2f50, 0x00000001, 0x00000001,
 662         0x30cc, 0xc0000fff, 0x00000104,
 663         0xc1e4, 0x00000001, 0x00000001,
 664         0xd0c0, 0xfffffff0, 0x00000100,
 665         0xd8c0, 0xfffffff0, 0x00000100
 666 };
 667
 668 static const u32 pitcairn_mgcg_cgcg_init[] =
 669 {
 670         0xc400, 0xffffffff, 0xfffffffc,
 671         0x802c, 0xffffffff, 0xe0000000,
 672         0x9a60, 0xffffffff, 0x00000100,
 673         0x92a4, 0xffffffff, 0x00000100,
 674         0xc164, 0xffffffff, 0x00000100,
 675         0x9774, 0xffffffff, 0x00000100,
 676         0x8984, 0xffffffff, 0x06000100,
 677         0x8a18, 0xffffffff, 0x00000100,
 678         0x92a0, 0xffffffff, 0x00000100,
 679         0xc380, 0xffffffff, 0x00000100,
 680         0x8b28, 0xffffffff, 0x00000100,
 681         0x9144, 0xffffffff, 0x00000100,
 682         0x8d88, 0xffffffff, 0x00000100,
 683         0x8d8c, 0xffffffff, 0x00000100,
 684         0x9030, 0xffffffff, 0x00000100,
 685         0x9034, 0xffffffff, 0x00000100,
 686         0x9038, 0xffffffff, 0x00000100,
 687         0x903c, 0xffffffff, 0x00000100,
 688         0xad80, 0xffffffff, 0x00000100,
 689         0xac54, 0xffffffff, 0x00000100,
 690         0x897c, 0xffffffff, 0x06000100,
 691         0x9868, 0xffffffff, 0x00000100,
 692         0x9510, 0xffffffff, 0x00000100,
 693         0xaf04, 0xffffffff, 0x00000100,
 694         0xae04, 0xffffffff, 0x00000100,
 695         0x949c, 0xffffffff, 0x00000100,
 696         0x802c, 0xffffffff, 0xe0000000,
 697         0x9160, 0xffffffff, 0x00010000,
 698         0x9164, 0xffffffff, 0x00030002,
 699         0x9168, 0xffffffff, 0x00040007,
 700         0x916c, 0xffffffff, 0x00060005,
 701         0x9170, 0xffffffff, 0x00090008,
 702         0x9174, 0xffffffff, 0x00020001,
 703         0x9178, 0xffffffff, 0x00040003,
 704         0x917c, 0xffffffff, 0x00000007,
 705         0x9180, 0xffffffff, 0x00060005,
 706         0x9184, 0xffffffff, 0x00090008,
 707         0x9188, 0xffffffff, 0x00030002,
 708         0x918c, 0xffffffff, 0x00050004,
 709         0x9190, 0xffffffff, 0x00000008,
 710         0x9194, 0xffffffff, 0x00070006,
 711         0x9198, 0xffffffff, 0x000a0009,
 712         0x919c, 0xffffffff, 0x00040003,
 713         0x91a0, 0xffffffff, 0x00060005,
 714         0x91a4, 0xffffffff, 0x00000009,
 715         0x91a8, 0xffffffff, 0x00080007,
 716         0x91ac, 0xffffffff, 0x000b000a,
 717         0x91b0, 0xffffffff, 0x00050004,
 718         0x91b4, 0xffffffff, 0x00070006,
 719         0x91b8, 0xffffffff, 0x0008000b,
 720         0x91bc, 0xffffffff, 0x000a0009,
 721         0x91c0, 0xffffffff, 0x000d000c,
 722         0x9200, 0xffffffff, 0x00090008,
 723         0x9204, 0xffffffff, 0x000b000a,
 724         0x9208, 0xffffffff, 0x000c000f,
 725         0x920c, 0xffffffff, 0x000e000d,
 726         0x9210, 0xffffffff, 0x00110010,
 727         0x9214, 0xffffffff, 0x000a0009,
 728         0x9218, 0xffffffff, 0x000c000b,
 729         0x921c, 0xffffffff, 0x0000000f,
 730         0x9220, 0xffffffff, 0x000e000d,
 731         0x9224, 0xffffffff, 0x00110010,
 732         0x9228, 0xffffffff, 0x000b000a,
 733         0x922c, 0xffffffff, 0x000d000c,
 734         0x9230, 0xffffffff, 0x00000010,
 735         0x9234, 0xffffffff, 0x000f000e,
 736         0x9238, 0xffffffff, 0x00120011,
 737         0x923c, 0xffffffff, 0x000c000b,
 738         0x9240, 0xffffffff, 0x000e000d,
 739         0x9244, 0xffffffff, 0x00000011,
 740         0x9248, 0xffffffff, 0x0010000f,
 741         0x924c, 0xffffffff, 0x00130012,
 742         0x9250, 0xffffffff, 0x000d000c,
 743         0x9254, 0xffffffff, 0x000f000e,
 744         0x9258, 0xffffffff, 0x00100013,
 745         0x925c, 0xffffffff, 0x00120011,
 746         0x9260, 0xffffffff, 0x00150014,
 747         0x9150, 0xffffffff, 0x96940200,
 748         0x8708, 0xffffffff, 0x00900100,
 749         0xc478, 0xffffffff, 0x00000080,
 750         0xc404, 0xffffffff, 0x0020003f,
 751         0x30, 0xffffffff, 0x0000001c,
 752         0x34, 0x000f0000, 0x000f0000,
 753         0x160c, 0xffffffff, 0x00000100,
 754         0x1024, 0xffffffff, 0x00000100,
 755         0x102c, 0x00000101, 0x00000000,
 756         0x20a8, 0xffffffff, 0x00000104,
 757         0x55e4, 0xff000fff, 0x00000100,
 758         0x55e8, 0x00000001, 0x00000001,
 759         0x2f50, 0x00000001, 0x00000001,
 760         0x30cc, 0xc0000fff, 0x00000104,
 761         0xc1e4, 0x00000001, 0x00000001,
 762         0xd0c0, 0xfffffff0, 0x00000100,
 763         0xd8c0, 0xfffffff0, 0x00000100
 764 };
 765
 766 static const u32 verde_mgcg_cgcg_init[] =
 767 {
 768         0xc400, 0xffffffff, 0xfffffffc,
 769         0x802c, 0xffffffff, 0xe0000000,
 770         0x9a60, 0xffffffff, 0x00000100,
 771         0x92a4, 0xffffffff, 0x00000100,
 772         0xc164, 0xffffffff, 0x00000100,
 773         0x9774, 0xffffffff, 0x00000100,
 774         0x8984, 0xffffffff, 0x06000100,
 775         0x8a18, 0xffffffff, 0x00000100,
 776         0x92a0, 0xffffffff, 0x00000100,
 777         0xc380, 0xffffffff, 0x00000100,
 778         0x8b28, 0xffffffff, 0x00000100,
 779         0x9144, 0xffffffff, 0x00000100,
 780         0x8d88, 0xffffffff, 0x00000100,
 781         0x8d8c, 0xffffffff, 0x00000100,
 782         0x9030, 0xffffffff, 0x00000100,
 783         0x9034, 0xffffffff, 0x00000100,
 784         0x9038, 0xffffffff, 0x00000100,
 785         0x903c, 0xffffffff, 0x00000100,
 786         0xad80, 0xffffffff, 0x00000100,
 787         0xac54, 0xffffffff, 0x00000100,
 788         0x897c, 0xffffffff, 0x06000100,
 789         0x9868, 0xffffffff, 0x00000100,
 790         0x9510, 0xffffffff, 0x00000100,
 791         0xaf04, 0xffffffff, 0x00000100,
 792         0xae04, 0xffffffff, 0x00000100,
 793         0x949c, 0xffffffff, 0x00000100,
 794         0x802c, 0xffffffff, 0xe0000000,
 795         0x9160, 0xffffffff, 0x00010000,
 796         0x9164, 0xffffffff, 0x00030002,
 797         0x9168, 0xffffffff, 0x00040007,
 798         0x916c, 0xffffffff, 0x00060005,
 799         0x9170, 0xffffffff, 0x00090008,
 800         0x9174, 0xffffffff, 0x00020001,
 801         0x9178, 0xffffffff, 0x00040003,
 802         0x917c, 0xffffffff, 0x00000007,
 803         0x9180, 0xffffffff, 0x00060005,
 804         0x9184, 0xffffffff, 0x00090008,
 805         0x9188, 0xffffffff, 0x00030002,
 806         0x918c, 0xffffffff, 0x00050004,
 807         0x9190, 0xffffffff, 0x00000008,
 808         0x9194, 0xffffffff, 0x00070006,
 809         0x9198, 0xffffffff, 0x000a0009,
 810         0x919c, 0xffffffff, 0x00040003,
 811         0x91a0, 0xffffffff, 0x00060005,
 812         0x91a4, 0xffffffff, 0x00000009,
 813         0x91a8, 0xffffffff, 0x00080007,
 814         0x91ac, 0xffffffff, 0x000b000a,
 815         0x91b0, 0xffffffff, 0x00050004,
 816         0x91b4, 0xffffffff, 0x00070006,
 817         0x91b8, 0xffffffff, 0x0008000b,
 818         0x91bc, 0xffffffff, 0x000a0009,
 819         0x91c0, 0xffffffff, 0x000d000c,
 820         0x9200, 0xffffffff, 0x00090008,
 821         0x9204, 0xffffffff, 0x000b000a,
 822         0x9208, 0xffffffff, 0x000c000f,
 823         0x920c, 0xffffffff, 0x000e000d,
 824         0x9210, 0xffffffff, 0x00110010,
 825         0x9214, 0xffffffff, 0x000a0009,
 826         0x9218, 0xffffffff, 0x000c000b,
 827         0x921c, 0xffffffff, 0x0000000f,
 828         0x9220, 0xffffffff, 0x000e000d,
 829         0x9224, 0xffffffff, 0x00110010,
 830         0x9228, 0xffffffff, 0x000b000a,
 831         0x922c, 0xffffffff, 0x000d000c,
 832         0x9230, 0xffffffff, 0x00000010,
 833         0x9234, 0xffffffff, 0x000f000e,
 834         0x9238, 0xffffffff, 0x00120011,
 835         0x923c, 0xffffffff, 0x000c000b,
 836         0x9240, 0xffffffff, 0x000e000d,
 837         0x9244, 0xffffffff, 0x00000011,
 838         0x9248, 0xffffffff, 0x0010000f,
 839         0x924c, 0xffffffff, 0x00130012,
 840         0x9250, 0xffffffff, 0x000d000c,
 841         0x9254, 0xffffffff, 0x000f000e,
 842         0x9258, 0xffffffff, 0x00100013,
 843         0x925c, 0xffffffff, 0x00120011,
 844         0x9260, 0xffffffff, 0x00150014,
 845         0x9150, 0xffffffff, 0x96940200,
 846         0x8708, 0xffffffff, 0x00900100,
 847         0xc478, 0xffffffff, 0x00000080,
 848         0xc404, 0xffffffff, 0x0020003f,
 849         0x30, 0xffffffff, 0x0000001c,
 850         0x34, 0x000f0000, 0x000f0000,
 851         0x160c, 0xffffffff, 0x00000100,
 852         0x1024, 0xffffffff, 0x00000100,
 853         0x102c, 0x00000101, 0x00000000,
 854         0x20a8, 0xffffffff, 0x00000104,
 855         0x264c, 0x000c0000, 0x000c0000,
 856         0x2648, 0x000c0000, 0x000c0000,
 857         0x55e4, 0xff000fff, 0x00000100,
 858         0x55e8, 0x00000001, 0x00000001,
 859         0x2f50, 0x00000001, 0x00000001,
 860         0x30cc, 0xc0000fff, 0x00000104,
 861         0xc1e4, 0x00000001, 0x00000001,
 862         0xd0c0, 0xfffffff0, 0x00000100,
 863         0xd8c0, 0xfffffff0, 0x00000100
 864 };
 865
 866 static const u32 oland_mgcg_cgcg_init[] =
 867 {
 868         0xc400, 0xffffffff, 0xfffffffc,
 869         0x802c, 0xffffffff, 0xe0000000,
 870         0x9a60, 0xffffffff, 0x00000100,
 871         0x92a4, 0xffffffff, 0x00000100,
 872         0xc164, 0xffffffff, 0x00000100,
 873         0x9774, 0xffffffff, 0x00000100,
 874         0x8984, 0xffffffff, 0x06000100,
 875         0x8a18, 0xffffffff, 0x00000100,
 876         0x92a0, 0xffffffff, 0x00000100,
 877         0xc380, 0xffffffff, 0x00000100,
 878         0x8b28, 0xffffffff, 0x00000100,
 879         0x9144, 0xffffffff, 0x00000100,
 880         0x8d88, 0xffffffff, 0x00000100,
 881         0x8d8c, 0xffffffff, 0x00000100,
 882         0x9030, 0xffffffff, 0x00000100,
 883         0x9034, 0xffffffff, 0x00000100,
 884         0x9038, 0xffffffff, 0x00000100,
 885         0x903c, 0xffffffff, 0x00000100,
 886         0xad80, 0xffffffff, 0x00000100,
 887         0xac54, 0xffffffff, 0x00000100,
 888         0x897c, 0xffffffff, 0x06000100,
 889         0x9868, 0xffffffff, 0x00000100,
 890         0x9510, 0xffffffff, 0x00000100,
 891         0xaf04, 0xffffffff, 0x00000100,
 892         0xae04, 0xffffffff, 0x00000100,
 893         0x949c, 0xffffffff, 0x00000100,
 894         0x802c, 0xffffffff, 0xe0000000,
 895         0x9160, 0xffffffff, 0x00010000,
 896         0x9164, 0xffffffff, 0x00030002,
 897         0x9168, 0xffffffff, 0x00040007,
 898         0x916c, 0xffffffff, 0x00060005,
 899         0x9170, 0xffffffff, 0x00090008,
 900         0x9174, 0xffffffff, 0x00020001,
 901         0x9178, 0xffffffff, 0x00040003,
 902         0x917c, 0xffffffff, 0x00000007,
 903         0x9180, 0xffffffff, 0x00060005,
 904         0x9184, 0xffffffff, 0x00090008,
 905         0x9188, 0xffffffff, 0x00030002,
 906         0x918c, 0xffffffff, 0x00050004,
 907         0x9190, 0xffffffff, 0x00000008,
 908         0x9194, 0xffffffff, 0x00070006,
 909         0x9198, 0xffffffff, 0x000a0009,
 910         0x919c, 0xffffffff, 0x00040003,
 911         0x91a0, 0xffffffff, 0x00060005,
 912         0x91a4, 0xffffffff, 0x00000009,
 913         0x91a8, 0xffffffff, 0x00080007,
 914         0x91ac, 0xffffffff, 0x000b000a,
 915         0x91b0, 0xffffffff, 0x00050004,
 916         0x91b4, 0xffffffff, 0x00070006,
 917         0x91b8, 0xffffffff, 0x0008000b,
 918         0x91bc, 0xffffffff, 0x000a0009,
 919         0x91c0, 0xffffffff, 0x000d000c,
 920         0x91c4, 0xffffffff, 0x00060005,
 921         0x91c8, 0xffffffff, 0x00080007,
 922         0x91cc, 0xffffffff, 0x0000000b,
 923         0x91d0, 0xffffffff, 0x000a0009,
 924         0x91d4, 0xffffffff, 0x000d000c,
 925         0x9150, 0xffffffff, 0x96940200,
 926         0x8708, 0xffffffff, 0x00900100,
 927         0xc478, 0xffffffff, 0x00000080,
 928         0xc404, 0xffffffff, 0x0020003f,
 929         0x30, 0xffffffff, 0x0000001c,
 930         0x34, 0x000f0000, 0x000f0000,
 931         0x160c, 0xffffffff, 0x00000100,
 932         0x1024, 0xffffffff, 0x00000100,
 933         0x102c, 0x00000101, 0x00000000,
 934         0x20a8, 0xffffffff, 0x00000104,
 935         0x264c, 0x000c0000, 0x000c0000,
 936         0x2648, 0x000c0000, 0x000c0000,
 937         0x55e4, 0xff000fff, 0x00000100,
 938         0x55e8, 0x00000001, 0x00000001,
 939         0x2f50, 0x00000001, 0x00000001,
 940         0x30cc, 0xc0000fff, 0x00000104,
 941         0xc1e4, 0x00000001, 0x00000001,
 942         0xd0c0, 0xfffffff0, 0x00000100,
 943         0xd8c0, 0xfffffff0, 0x00000100
 944 };
 945
 946 static const u32 hainan_mgcg_cgcg_init[] =
 947 {
 948         0xc400, 0xffffffff, 0xfffffffc,
 949         0x802c, 0xffffffff, 0xe0000000,
 950         0x9a60, 0xffffffff, 0x00000100,
 951         0x92a4, 0xffffffff, 0x00000100,
 952         0xc164, 0xffffffff, 0x00000100,
 953         0x9774, 0xffffffff, 0x00000100,
 954         0x8984, 0xffffffff, 0x06000100,
 955         0x8a18, 0xffffffff, 0x00000100,
 956         0x92a0, 0xffffffff, 0x00000100,
 957         0xc380, 0xffffffff, 0x00000100,
 958         0x8b28, 0xffffffff, 0x00000100,
 959         0x9144, 0xffffffff, 0x00000100,
 960         0x8d88, 0xffffffff, 0x00000100,
 961         0x8d8c, 0xffffffff, 0x00000100,
 962         0x9030, 0xffffffff, 0x00000100,
 963         0x9034, 0xffffffff, 0x00000100,
 964         0x9038, 0xffffffff, 0x00000100,
 965         0x903c, 0xffffffff, 0x00000100,
 966         0xad80, 0xffffffff, 0x00000100,
 967         0xac54, 0xffffffff, 0x00000100,
 968         0x897c, 0xffffffff, 0x06000100,
 969         0x9868, 0xffffffff, 0x00000100,
 970         0x9510, 0xffffffff, 0x00000100,
 971         0xaf04, 0xffffffff, 0x00000100,
 972         0xae04, 0xffffffff, 0x00000100,
 973         0x949c, 0xffffffff, 0x00000100,
 974         0x802c, 0xffffffff, 0xe0000000,
 975         0x9160, 0xffffffff, 0x00010000,
 976         0x9164, 0xffffffff, 0x00030002,
 977         0x9168, 0xffffffff, 0x00040007,
 978         0x916c, 0xffffffff, 0x00060005,
 979         0x9170, 0xffffffff, 0x00090008,
 980         0x9174, 0xffffffff, 0x00020001,
 981         0x9178, 0xffffffff, 0x00040003,
 982         0x917c, 0xffffffff, 0x00000007,
 983         0x9180, 0xffffffff, 0x00060005,
 984         0x9184, 0xffffffff, 0x00090008,
 985         0x9188, 0xffffffff, 0x00030002,
 986         0x918c, 0xffffffff, 0x00050004,
 987         0x9190, 0xffffffff, 0x00000008,
 988         0x9194, 0xffffffff, 0x00070006,
 989         0x9198, 0xffffffff, 0x000a0009,
 990         0x919c, 0xffffffff, 0x00040003,
 991         0x91a0, 0xffffffff, 0x00060005,
 992         0x91a4, 0xffffffff, 0x00000009,
 993         0x91a8, 0xffffffff, 0x00080007,
 994         0x91ac, 0xffffffff, 0x000b000a,
 995         0x91b0, 0xffffffff, 0x00050004,
 996         0x91b4, 0xffffffff, 0x00070006,
 997         0x91b8, 0xffffffff, 0x0008000b,
 998         0x91bc, 0xffffffff, 0x000a0009,
 999         0x91c0, 0xffffffff, 0x000d000c,
1000         0x91c4, 0xffffffff, 0x00060005,
1001         0x91c8, 0xffffffff, 0x00080007,
1002         0x91cc, 0xffffffff, 0x0000000b,
1003         0x91d0, 0xffffffff, 0x000a0009,
1004         0x91d4, 0xffffffff, 0x000d000c,
1005         0x9150, 0xffffffff, 0x96940200,
1006         0x8708, 0xffffffff, 0x00900100,
1007         0xc478, 0xffffffff, 0x00000080,
1008         0xc404, 0xffffffff, 0x0020003f,
1009         0x30, 0xffffffff, 0x0000001c,
1010         0x34, 0x000f0000, 0x000f0000,
1011         0x160c, 0xffffffff, 0x00000100,
1012         0x1024, 0xffffffff, 0x00000100,
1013         0x20a8, 0xffffffff, 0x00000104,
1014         0x264c, 0x000c0000, 0x000c0000,
1015         0x2648, 0x000c0000, 0x000c0000,
1016         0x2f50, 0x00000001, 0x00000001,
1017         0x30cc, 0xc0000fff, 0x00000104,
1018         0xc1e4, 0x00000001, 0x00000001,
1019         0xd0c0, 0xfffffff0, 0x00000100,
1020         0xd8c0, 0xfffffff0, 0x00000100
1021 };
1022
1023 static u32 verde_pg_init[] =
1024 {
1025         0x353c, 0xffffffff, 0x40000,
1026         0x3538, 0xffffffff, 0x200010ff,
1027         0x353c, 0xffffffff, 0x0,
1028         0x353c, 0xffffffff, 0x0,
1029         0x353c, 0xffffffff, 0x0,
1030         0x353c, 0xffffffff, 0x0,
1031         0x353c, 0xffffffff, 0x0,
1032         0x353c, 0xffffffff, 0x7007,
1033         0x3538, 0xffffffff, 0x300010ff,
1034         0x353c, 0xffffffff, 0x0,
1035         0x353c, 0xffffffff, 0x0,
1036         0x353c, 0xffffffff, 0x0,
1037         0x353c, 0xffffffff, 0x0,
1038         0x353c, 0xffffffff, 0x0,
1039         0x353c, 0xffffffff, 0x400000,
1040         0x3538, 0xffffffff, 0x100010ff,
1041         0x353c, 0xffffffff, 0x0,
1042         0x353c, 0xffffffff, 0x0,
1043         0x353c, 0xffffffff, 0x0,
1044         0x353c, 0xffffffff, 0x0,
1045         0x353c, 0xffffffff, 0x0,
1046         0x353c, 0xffffffff, 0x120200,
1047         0x3538, 0xffffffff, 0x500010ff,
1048         0x353c, 0xffffffff, 0x0,
1049         0x353c, 0xffffffff, 0x0,
1050         0x353c, 0xffffffff, 0x0,
1051         0x353c, 0xffffffff, 0x0,
1052         0x353c, 0xffffffff, 0x0,
1053         0x353c, 0xffffffff, 0x1e1e16,
1054         0x3538, 0xffffffff, 0x600010ff,
1055         0x353c, 0xffffffff, 0x0,
1056         0x353c, 0xffffffff, 0x0,
1057         0x353c, 0xffffffff, 0x0,
1058         0x353c, 0xffffffff, 0x0,
1059         0x353c, 0xffffffff, 0x0,
1060         0x353c, 0xffffffff, 0x171f1e,
1061         0x3538, 0xffffffff, 0x700010ff,
1062         0x353c, 0xffffffff, 0x0,
1063         0x353c, 0xffffffff, 0x0,
1064         0x353c, 0xffffffff, 0x0,
1065         0x353c, 0xffffffff, 0x0,
1066         0x353c, 0xffffffff, 0x0,
1067         0x353c, 0xffffffff, 0x0,
1068         0x3538, 0xffffffff, 0x9ff,
1069         0x3500, 0xffffffff, 0x0,
1070         0x3504, 0xffffffff, 0x10000800,
1071         0x3504, 0xffffffff, 0xf,
1072         0x3504, 0xffffffff, 0xf,
1073         0x3500, 0xffffffff, 0x4,
1074         0x3504, 0xffffffff, 0x1000051e,
1075         0x3504, 0xffffffff, 0xffff,
1076         0x3504, 0xffffffff, 0xffff,
1077         0x3500, 0xffffffff, 0x8,
1078         0x3504, 0xffffffff, 0x80500,
1079         0x3500, 0xffffffff, 0x12,
1080         0x3504, 0xffffffff, 0x9050c,
1081         0x3500, 0xffffffff, 0x1d,
1082         0x3504, 0xffffffff, 0xb052c,
1083         0x3500, 0xffffffff, 0x2a,
1084         0x3504, 0xffffffff, 0x1053e,
1085         0x3500, 0xffffffff, 0x2d,
1086         0x3504, 0xffffffff, 0x10546,
1087         0x3500, 0xffffffff, 0x30,
1088         0x3504, 0xffffffff, 0xa054e,
1089         0x3500, 0xffffffff, 0x3c,
1090         0x3504, 0xffffffff, 0x1055f,
1091         0x3500, 0xffffffff, 0x3f,
1092         0x3504, 0xffffffff, 0x10567,
1093         0x3500, 0xffffffff, 0x42,
1094         0x3504, 0xffffffff, 0x1056f,
1095         0x3500, 0xffffffff, 0x45,
1096         0x3504, 0xffffffff, 0x10572,
1097         0x3500, 0xffffffff, 0x48,
1098         0x3504, 0xffffffff, 0x20575,
1099         0x3500, 0xffffffff, 0x4c,
1100         0x3504, 0xffffffff, 0x190801,
1101         0x3500, 0xffffffff, 0x67,
1102         0x3504, 0xffffffff, 0x1082a,
1103         0x3500, 0xffffffff, 0x6a,
1104         0x3504, 0xffffffff, 0x1b082d,
1105         0x3500, 0xffffffff, 0x87,
1106         0x3504, 0xffffffff, 0x310851,
1107         0x3500, 0xffffffff, 0xba,
1108         0x3504, 0xffffffff, 0x891,
1109         0x3500, 0xffffffff, 0xbc,
1110         0x3504, 0xffffffff, 0x893,
1111         0x3500, 0xffffffff, 0xbe,
1112         0x3504, 0xffffffff, 0x20895,
1113         0x3500, 0xffffffff, 0xc2,
1114         0x3504, 0xffffffff, 0x20899,
1115         0x3500, 0xffffffff, 0xc6,
1116         0x3504, 0xffffffff, 0x2089d,
1117         0x3500, 0xffffffff, 0xca,
1118         0x3504, 0xffffffff, 0x8a1,
1119         0x3500, 0xffffffff, 0xcc,
1120         0x3504, 0xffffffff, 0x8a3,
1121         0x3500, 0xffffffff, 0xce,
1122         0x3504, 0xffffffff, 0x308a5,
1123         0x3500, 0xffffffff, 0xd3,
1124         0x3504, 0xffffffff, 0x6d08cd,
1125         0x3500, 0xffffffff, 0x142,
1126         0x3504, 0xffffffff, 0x2000095a,
1127         0x3504, 0xffffffff, 0x1,
1128         0x3500, 0xffffffff, 0x144,
1129         0x3504, 0xffffffff, 0x301f095b,
1130         0x3500, 0xffffffff, 0x165,
1131         0x3504, 0xffffffff, 0xc094d,
1132         0x3500, 0xffffffff, 0x173,
1133         0x3504, 0xffffffff, 0xf096d,
1134         0x3500, 0xffffffff, 0x184,
1135         0x3504, 0xffffffff, 0x15097f,
1136         0x3500, 0xffffffff, 0x19b,
1137         0x3504, 0xffffffff, 0xc0998,
1138         0x3500, 0xffffffff, 0x1a9,
1139         0x3504, 0xffffffff, 0x409a7,
1140         0x3500, 0xffffffff, 0x1af,
1141         0x3504, 0xffffffff, 0xcdc,
1142         0x3500, 0xffffffff, 0x1b1,
1143         0x3504, 0xffffffff, 0x800,
1144         0x3508, 0xffffffff, 0x6c9b2000,
1145         0x3510, 0xfc00, 0x2000,
1146         0x3544, 0xffffffff, 0xfc0,
1147         0x28d4, 0x00000100, 0x100
1148 };
1149
1150 static void si_init_golden_registers(struct radeon_device *rdev)
1151 {
1152         switch (rdev->family) {
1153         case CHIP_TAHITI:
1154                 radeon_program_register_sequence(rdev,
1155                                                  tahiti_golden_registers,
1156                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
1157                 radeon_program_register_sequence(rdev,
1158                                                  tahiti_golden_rlc_registers,
1159                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1160                 radeon_program_register_sequence(rdev,
1161                                                  tahiti_mgcg_cgcg_init,
1162                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1163                 radeon_program_register_sequence(rdev,
1164                                                  tahiti_golden_registers2,
1165                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1166                 break;
1167         case CHIP_PITCAIRN:
1168                 radeon_program_register_sequence(rdev,
1169                                                  pitcairn_golden_registers,
1170                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1171                 radeon_program_register_sequence(rdev,
1172                                                  pitcairn_golden_rlc_registers,
1173                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1174                 radeon_program_register_sequence(rdev,
1175                                                  pitcairn_mgcg_cgcg_init,
1176                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1177                 break;
1178         case CHIP_VERDE:
1179                 radeon_program_register_sequence(rdev,
1180                                                  verde_golden_registers,
1181                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
1182                 radeon_program_register_sequence(rdev,
1183                                                  verde_golden_rlc_registers,
1184                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1185                 radeon_program_register_sequence(rdev,
1186                                                  verde_mgcg_cgcg_init,
1187                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1188                 radeon_program_register_sequence(rdev,
1189                                                  verde_pg_init,
1190                                                  (const u32)ARRAY_SIZE(verde_pg_init));
1191                 break;
1192         case CHIP_OLAND:
1193                 radeon_program_register_sequence(rdev,
1194                                                  oland_golden_registers,
1195                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
1196                 radeon_program_register_sequence(rdev,
1197                                                  oland_golden_rlc_registers,
1198                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1199                 radeon_program_register_sequence(rdev,
1200                                                  oland_mgcg_cgcg_init,
1201                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1202                 break;
1203         case CHIP_HAINAN:
1204                 radeon_program_register_sequence(rdev,
1205                                                  hainan_golden_registers,
1206                                                  (const u32)ARRAY_SIZE(hainan_golden_registers));
1207                 radeon_program_register_sequence(rdev,
1208                                                  hainan_golden_registers2,
1209                                                  (const u32)ARRAY_SIZE(hainan_golden_registers2));
1210                 radeon_program_register_sequence(rdev,
1211                                                  hainan_mgcg_cgcg_init,
1212                                                  (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1213                 break;
1214         default:
1215                 break;
1216         }
1217 }
1218
1219 #define PCIE_BUS_CLK                10000
1220 #define TCLK                        (PCIE_BUS_CLK / 10)
1221
1222 /**
1223  * si_get_xclk - get the xclk
1224  *
1225  * @rdev: radeon_device pointer
1226  *
1227  * Returns the reference clock used by the gfx engine
1228  * (SI).
1229  */
1230 u32 si_get_xclk(struct radeon_device *rdev)
1231 {
1232         u32 reference_clock = rdev->clock.spll.reference_freq;
1233         u32 tmp;
1234
1235         tmp = RREG32(CG_CLKPIN_CNTL_2);
1236         if (tmp & MUX_TCLK_TO_XCLK)
1237                 return TCLK;
1238
1239         tmp = RREG32(CG_CLKPIN_CNTL);
1240         if (tmp & XTALIN_DIVIDE)
1241                 return reference_clock / 4;
1242
1243         return reference_clock;
1244 }
1245
1246 /* get temperature in millidegrees */
1247 int si_get_temp(struct radeon_device *rdev)
1248 {
1249         u32 temp;
1250         int actual_temp = 0;
1251
1252         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1253                 CTF_TEMP_SHIFT;
1254
1255         if (temp & 0x200)
1256                 actual_temp = 255;
1257         else
1258                 actual_temp = temp & 0x1ff;
1259
1260         actual_temp = (actual_temp * 1000);
1261
1262         return actual_temp;
1263 }
1264
1265 #define TAHITI_IO_MC_REGS_SIZE 36
1266
1267 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1268         {0x0000006f, 0x03044000},
1269         {0x00000070, 0x0480c018},
1270         {0x00000071, 0x00000040},
1271         {0x00000072, 0x01000000},
1272         {0x00000074, 0x000000ff},
1273         {0x00000075, 0x00143400},
1274         {0x00000076, 0x08ec0800},
1275         {0x00000077, 0x040000cc},
1276         {0x00000079, 0x00000000},
1277         {0x0000007a, 0x21000409},
1278         {0x0000007c, 0x00000000},
1279         {0x0000007d, 0xe8000000},
1280         {0x0000007e, 0x044408a8},
1281         {0x0000007f, 0x00000003},
1282         {0x00000080, 0x00000000},
1283         {0x00000081, 0x01000000},
1284         {0x00000082, 0x02000000},
1285         {0x00000083, 0x00000000},
1286         {0x00000084, 0xe3f3e4f4},
1287         {0x00000085, 0x00052024},
1288         {0x00000087, 0x00000000},
1289         {0x00000088, 0x66036603},
1290         {0x00000089, 0x01000000},
1291         {0x0000008b, 0x1c0a0000},
1292         {0x0000008c, 0xff010000},
1293         {0x0000008e, 0xffffefff},
1294         {0x0000008f, 0xfff3efff},
1295         {0x00000090, 0xfff3efbf},
1296         {0x00000094, 0x00101101},
1297         {0x00000095, 0x00000fff},
1298         {0x00000096, 0x00116fff},
1299         {0x00000097, 0x60010000},
1300         {0x00000098, 0x10010000},
1301         {0x00000099, 0x00006000},
1302         {0x0000009a, 0x00001000},
1303         {0x0000009f, 0x00a77400}
1304 };
1305
1306 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1307         {0x0000006f, 0x03044000},
1308         {0x00000070, 0x0480c018},
1309         {0x00000071, 0x00000040},
1310         {0x00000072, 0x01000000},
1311         {0x00000074, 0x000000ff},
1312         {0x00000075, 0x00143400},
1313         {0x00000076, 0x08ec0800},
1314         {0x00000077, 0x040000cc},
1315         {0x00000079, 0x00000000},
1316         {0x0000007a, 0x21000409},
1317         {0x0000007c, 0x00000000},
1318         {0x0000007d, 0xe8000000},
1319         {0x0000007e, 0x044408a8},
1320         {0x0000007f, 0x00000003},
1321         {0x00000080, 0x00000000},
1322         {0x00000081, 0x01000000},
1323         {0x00000082, 0x02000000},
1324         {0x00000083, 0x00000000},
1325         {0x00000084, 0xe3f3e4f4},
1326         {0x00000085, 0x00052024},
1327         {0x00000087, 0x00000000},
1328         {0x00000088, 0x66036603},
1329         {0x00000089, 0x01000000},
1330         {0x0000008b, 0x1c0a0000},
1331         {0x0000008c, 0xff010000},
1332         {0x0000008e, 0xffffefff},
1333         {0x0000008f, 0xfff3efff},
1334         {0x00000090, 0xfff3efbf},
1335         {0x00000094, 0x00101101},
1336         {0x00000095, 0x00000fff},
1337         {0x00000096, 0x00116fff},
1338         {0x00000097, 0x60010000},
1339         {0x00000098, 0x10010000},
1340         {0x00000099, 0x00006000},
1341         {0x0000009a, 0x00001000},
1342         {0x0000009f, 0x00a47400}
1343 };
1344
1345 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346         {0x0000006f, 0x03044000},
1347         {0x00000070, 0x0480c018},
1348         {0x00000071, 0x00000040},
1349         {0x00000072, 0x01000000},
1350         {0x00000074, 0x000000ff},
1351         {0x00000075, 0x00143400},
1352         {0x00000076, 0x08ec0800},
1353         {0x00000077, 0x040000cc},
1354         {0x00000079, 0x00000000},
1355         {0x0000007a, 0x21000409},
1356         {0x0000007c, 0x00000000},
1357         {0x0000007d, 0xe8000000},
1358         {0x0000007e, 0x044408a8},
1359         {0x0000007f, 0x00000003},
1360         {0x00000080, 0x00000000},
1361         {0x00000081, 0x01000000},
1362         {0x00000082, 0x02000000},
1363         {0x00000083, 0x00000000},
1364         {0x00000084, 0xe3f3e4f4},
1365         {0x00000085, 0x00052024},
1366         {0x00000087, 0x00000000},
1367         {0x00000088, 0x66036603},
1368         {0x00000089, 0x01000000},
1369         {0x0000008b, 0x1c0a0000},
1370         {0x0000008c, 0xff010000},
1371         {0x0000008e, 0xffffefff},
1372         {0x0000008f, 0xfff3efff},
1373         {0x00000090, 0xfff3efbf},
1374         {0x00000094, 0x00101101},
1375         {0x00000095, 0x00000fff},
1376         {0x00000096, 0x00116fff},
1377         {0x00000097, 0x60010000},
1378         {0x00000098, 0x10010000},
1379         {0x00000099, 0x00006000},
1380         {0x0000009a, 0x00001000},
1381         {0x0000009f, 0x00a37400}
1382 };
1383
1384 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385         {0x0000006f, 0x03044000},
1386         {0x00000070, 0x0480c018},
1387         {0x00000071, 0x00000040},
1388         {0x00000072, 0x01000000},
1389         {0x00000074, 0x000000ff},
1390         {0x00000075, 0x00143400},
1391         {0x00000076, 0x08ec0800},
1392         {0x00000077, 0x040000cc},
1393         {0x00000079, 0x00000000},
1394         {0x0000007a, 0x21000409},
1395         {0x0000007c, 0x00000000},
1396         {0x0000007d, 0xe8000000},
1397         {0x0000007e, 0x044408a8},
1398         {0x0000007f, 0x00000003},
1399         {0x00000080, 0x00000000},
1400         {0x00000081, 0x01000000},
1401         {0x00000082, 0x02000000},
1402         {0x00000083, 0x00000000},
1403         {0x00000084, 0xe3f3e4f4},
1404         {0x00000085, 0x00052024},
1405         {0x00000087, 0x00000000},
1406         {0x00000088, 0x66036603},
1407         {0x00000089, 0x01000000},
1408         {0x0000008b, 0x1c0a0000},
1409         {0x0000008c, 0xff010000},
1410         {0x0000008e, 0xffffefff},
1411         {0x0000008f, 0xfff3efff},
1412         {0x00000090, 0xfff3efbf},
1413         {0x00000094, 0x00101101},
1414         {0x00000095, 0x00000fff},
1415         {0x00000096, 0x00116fff},
1416         {0x00000097, 0x60010000},
1417         {0x00000098, 0x10010000},
1418         {0x00000099, 0x00006000},
1419         {0x0000009a, 0x00001000},
1420         {0x0000009f, 0x00a17730}
1421 };
1422
1423 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424         {0x0000006f, 0x03044000},
1425         {0x00000070, 0x0480c018},
1426         {0x00000071, 0x00000040},
1427         {0x00000072, 0x01000000},
1428         {0x00000074, 0x000000ff},
1429         {0x00000075, 0x00143400},
1430         {0x00000076, 0x08ec0800},
1431         {0x00000077, 0x040000cc},
1432         {0x00000079, 0x00000000},
1433         {0x0000007a, 0x21000409},
1434         {0x0000007c, 0x00000000},
1435         {0x0000007d, 0xe8000000},
1436         {0x0000007e, 0x044408a8},
1437         {0x0000007f, 0x00000003},
1438         {0x00000080, 0x00000000},
1439         {0x00000081, 0x01000000},
1440         {0x00000082, 0x02000000},
1441         {0x00000083, 0x00000000},
1442         {0x00000084, 0xe3f3e4f4},
1443         {0x00000085, 0x00052024},
1444         {0x00000087, 0x00000000},
1445         {0x00000088, 0x66036603},
1446         {0x00000089, 0x01000000},
1447         {0x0000008b, 0x1c0a0000},
1448         {0x0000008c, 0xff010000},
1449         {0x0000008e, 0xffffefff},
1450         {0x0000008f, 0xfff3efff},
1451         {0x00000090, 0xfff3efbf},
1452         {0x00000094, 0x00101101},
1453         {0x00000095, 0x00000fff},
1454         {0x00000096, 0x00116fff},
1455         {0x00000097, 0x60010000},
1456         {0x00000098, 0x10010000},
1457         {0x00000099, 0x00006000},
1458         {0x0000009a, 0x00001000},
1459         {0x0000009f, 0x00a07730}
1460 };
1461
1462 /* ucode loading */
1463 static int si_mc_load_microcode(struct radeon_device *rdev)
1464 {
1465         const __be32 *fw_data;
1466         u32 running, blackout = 0;
1467         u32 *io_mc_regs;
1468         int i, ucode_size, regs_size;
1469
1470         if (!rdev->mc_fw)
1471                 return -EINVAL;
1472
1473         switch (rdev->family) {
1474         case CHIP_TAHITI:
1475                 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1476                 ucode_size = SI_MC_UCODE_SIZE;
1477                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478                 break;
1479         case CHIP_PITCAIRN:
1480                 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1481                 ucode_size = SI_MC_UCODE_SIZE;
1482                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483                 break;
1484         case CHIP_VERDE:
1485         default:
1486                 io_mc_regs = (u32 *)&verde_io_mc_regs;
1487                 ucode_size = SI_MC_UCODE_SIZE;
1488                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1489                 break;
1490         case CHIP_OLAND:
1491                 io_mc_regs = (u32 *)&oland_io_mc_regs;
1492                 ucode_size = OLAND_MC_UCODE_SIZE;
1493                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1494                 break;
1495         case CHIP_HAINAN:
1496                 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1497                 ucode_size = OLAND_MC_UCODE_SIZE;
1498                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1499                 break;
1500         }
1501
1502         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1503
1504         if (running == 0) {
1505                 if (running) {
1506                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1507                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1508                 }
1509
1510                 /* reset the engine and set to writable */
1511                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1512                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1513
1514                 /* load mc io regs */
1515                 for (i = 0; i < regs_size; i++) {
1516                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1517                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1518                 }
1519                 /* load the MC ucode */
1520                 fw_data = (const __be32 *)rdev->mc_fw->data;
1521                 for (i = 0; i < ucode_size; i++)
1522                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1523
1524                 /* put the engine back into the active state */
1525                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1526                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1527                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1528
1529                 /* wait for training to complete */
1530                 for (i = 0; i < rdev->usec_timeout; i++) {
1531                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1532                                 break;
1533                         udelay(1);
1534                 }
1535                 for (i = 0; i < rdev->usec_timeout; i++) {
1536                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1537                                 break;
1538                         udelay(1);
1539                 }
1540
1541                 if (running)
1542                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1543         }
1544
1545         return 0;
1546 }
1547
1548 static int si_init_microcode(struct radeon_device *rdev)
1549 {
1550         const char *chip_name;
1551         const char *rlc_chip_name;
1552         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1553         size_t smc_req_size;
1554         char fw_name[30];
1555         int err;
1556
1557         DRM_DEBUG("\n");
1558
1559         switch (rdev->family) {
1560         case CHIP_TAHITI:
1561                 chip_name = "TAHITI";
1562                 rlc_chip_name = "TAHITI";
1563                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1564                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1565                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1566                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1567                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1568                 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1569                 break;
1570         case CHIP_PITCAIRN:
1571                 chip_name = "PITCAIRN";
1572                 rlc_chip_name = "PITCAIRN";
1573                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1574                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1575                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1576                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1577                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1578                 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1579                 break;
1580         case CHIP_VERDE:
1581                 chip_name = "VERDE";
1582                 rlc_chip_name = "VERDE";
1583                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1584                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1585                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1586                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1587                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1588                 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1589                 break;
1590         case CHIP_OLAND:
1591                 chip_name = "OLAND";
1592                 rlc_chip_name = "OLAND";
1593                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1594                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1595                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1596                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1597                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1598                 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1599                 break;
1600         case CHIP_HAINAN:
1601                 chip_name = "HAINAN";
1602                 rlc_chip_name = "HAINAN";
1603                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1604                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1605                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1606                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1607                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1608                 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1609                 break;
1610         default: BUG();
1611         }
1612
1613         DRM_INFO("Loading %s Microcode\n", chip_name);
1614
1615         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1616         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1617         if (err)
1618                 goto out;
1619         if (rdev->pfp_fw->size != pfp_req_size) {
1620                 printk(KERN_ERR
1621                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1622                        rdev->pfp_fw->size, fw_name);
1623                 err = -EINVAL;
1624                 goto out;
1625         }
1626
1627         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1628         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1629         if (err)
1630                 goto out;
1631         if (rdev->me_fw->size != me_req_size) {
1632                 printk(KERN_ERR
1633                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1634                        rdev->me_fw->size, fw_name);
1635                 err = -EINVAL;
1636         }
1637
1638         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1639         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1640         if (err)
1641                 goto out;
1642         if (rdev->ce_fw->size != ce_req_size) {
1643                 printk(KERN_ERR
1644                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1645                        rdev->ce_fw->size, fw_name);
1646                 err = -EINVAL;
1647         }
1648
1649         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1650         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1651         if (err)
1652                 goto out;
1653         if (rdev->rlc_fw->size != rlc_req_size) {
1654                 printk(KERN_ERR
1655                        "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1656                        rdev->rlc_fw->size, fw_name);
1657                 err = -EINVAL;
1658         }
1659
1660         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1661         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1662         if (err)
1663                 goto out;
1664         if (rdev->mc_fw->size != mc_req_size) {
1665                 printk(KERN_ERR
1666                        "si_mc: Bogus length %zu in firmware \"%s\"\n",
1667                        rdev->mc_fw->size, fw_name);
1668                 err = -EINVAL;
1669         }
1670
1671         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1672         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1673         if (err) {
1674                 printk(KERN_ERR
1675                        "smc: error loading firmware \"%s\"\n",
1676                        fw_name);
1677                 release_firmware(rdev->smc_fw);
1678                 rdev->smc_fw = NULL;
1679                 err = 0;
1680         } else if (rdev->smc_fw->size != smc_req_size) {
1681                 printk(KERN_ERR
1682                        "si_smc: Bogus length %zu in firmware \"%s\"\n",
1683                        rdev->smc_fw->size, fw_name);
1684                 err = -EINVAL;
1685         }
1686
1687 out:
1688         if (err) {
1689                 if (err != -EINVAL)
1690                         printk(KERN_ERR
1691                                "si_cp: Failed to load firmware \"%s\"\n",
1692                                fw_name);
1693                 release_firmware(rdev->pfp_fw);
1694                 rdev->pfp_fw = NULL;
1695                 release_firmware(rdev->me_fw);
1696                 rdev->me_fw = NULL;
1697                 release_firmware(rdev->ce_fw);
1698                 rdev->ce_fw = NULL;
1699                 release_firmware(rdev->rlc_fw);
1700                 rdev->rlc_fw = NULL;
1701                 release_firmware(rdev->mc_fw);
1702                 rdev->mc_fw = NULL;
1703                 release_firmware(rdev->smc_fw);
1704                 rdev->smc_fw = NULL;
1705         }
1706         return err;
1707 }
1708
1709 /* watermark setup */
1710 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1711                                    struct radeon_crtc *radeon_crtc,
1712                                    struct drm_display_mode *mode,
1713                                    struct drm_display_mode *other_mode)
1714 {
1715         u32 tmp, buffer_alloc, i;
1716         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1717         /*
1718          * Line Buffer Setup
1719          * There are 3 line buffers, each one shared by 2 display controllers.
1720          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1721          * the display controllers.  The paritioning is done via one of four
1722          * preset allocations specified in bits 21:20:
1723          *  0 - half lb
1724          *  2 - whole lb, other crtc must be disabled
1725          */
1726         /* this can get tricky if we have two large displays on a paired group
1727          * of crtcs.  Ideally for multiple large displays we'd assign them to
1728          * non-linked crtcs for maximum line buffer allocation.
1729          */
1730         if (radeon_crtc->base.enabled && mode) {
1731                 if (other_mode) {
1732                         tmp = 0; /* 1/2 */
1733                         buffer_alloc = 1;
1734                 } else {
1735                         tmp = 2; /* whole */
1736                         buffer_alloc = 2;
1737                 }
1738         } else {
1739                 tmp = 0;
1740                 buffer_alloc = 0;
1741         }
1742
1743         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1744                DC_LB_MEMORY_CONFIG(tmp));
1745
1746         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1747                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1748         for (i = 0; i < rdev->usec_timeout; i++) {
1749                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1750                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
1751                         break;
1752                 udelay(1);
1753         }
1754
1755         if (radeon_crtc->base.enabled && mode) {
1756                 switch (tmp) {
1757                 case 0:
1758                 default:
1759                         return 4096 * 2;
1760                 case 2:
1761                         return 8192 * 2;
1762                 }
1763         }
1764
1765         /* controller not enabled, so no lb used */
1766         return 0;
1767 }
1768
1769 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1770 {
1771         u32 tmp = RREG32(MC_SHARED_CHMAP);
1772
1773         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1774         case 0:
1775         default:
1776                 return 1;
1777         case 1:
1778                 return 2;
1779         case 2:
1780                 return 4;
1781         case 3:
1782                 return 8;
1783         case 4:
1784                 return 3;
1785         case 5:
1786                 return 6;
1787         case 6:
1788                 return 10;
1789         case 7:
1790                 return 12;
1791         case 8:
1792                 return 16;
1793         }
1794 }
1795
1796 struct dce6_wm_params {
1797         u32 dram_channels; /* number of dram channels */
1798         u32 yclk;          /* bandwidth per dram data pin in kHz */
1799         u32 sclk;          /* engine clock in kHz */
1800         u32 disp_clk;      /* display clock in kHz */
1801         u32 src_width;     /* viewport width */
1802         u32 active_time;   /* active display time in ns */
1803         u32 blank_time;    /* blank time in ns */
1804         bool interlaced;    /* mode is interlaced */
1805         fixed20_12 vsc;    /* vertical scale ratio */
1806         u32 num_heads;     /* number of active crtcs */
1807         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1808         u32 lb_size;       /* line buffer allocated to pipe */
1809         u32 vtaps;         /* vertical scaler taps */
1810 };
1811
1812 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1813 {
1814         /* Calculate raw DRAM Bandwidth */
1815         fixed20_12 dram_efficiency; /* 0.7 */
1816         fixed20_12 yclk, dram_channels, bandwidth;
1817         fixed20_12 a;
1818
1819         a.full = dfixed_const(1000);
1820         yclk.full = dfixed_const(wm->yclk);
1821         yclk.full = dfixed_div(yclk, a);
1822         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1823         a.full = dfixed_const(10);
1824         dram_efficiency.full = dfixed_const(7);
1825         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1826         bandwidth.full = dfixed_mul(dram_channels, yclk);
1827         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1828
1829         return dfixed_trunc(bandwidth);
1830 }
1831
1832 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1833 {
1834         /* Calculate DRAM Bandwidth and the part allocated to display. */
1835         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1836         fixed20_12 yclk, dram_channels, bandwidth;
1837         fixed20_12 a;
1838
1839         a.full = dfixed_const(1000);
1840         yclk.full = dfixed_const(wm->yclk);
1841         yclk.full = dfixed_div(yclk, a);
1842         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1843         a.full = dfixed_const(10);
1844         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1845         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1846         bandwidth.full = dfixed_mul(dram_channels, yclk);
1847         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1848
1849         return dfixed_trunc(bandwidth);
1850 }
1851
1852 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1853 {
1854         /* Calculate the display Data return Bandwidth */
1855         fixed20_12 return_efficiency; /* 0.8 */
1856         fixed20_12 sclk, bandwidth;
1857         fixed20_12 a;
1858
1859         a.full = dfixed_const(1000);
1860         sclk.full = dfixed_const(wm->sclk);
1861         sclk.full = dfixed_div(sclk, a);
1862         a.full = dfixed_const(10);
1863         return_efficiency.full = dfixed_const(8);
1864         return_efficiency.full = dfixed_div(return_efficiency, a);
1865         a.full = dfixed_const(32);
1866         bandwidth.full = dfixed_mul(a, sclk);
1867         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1868
1869         return dfixed_trunc(bandwidth);
1870 }
1871
1872 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1873 {
1874         return 32;
1875 }
1876
1877 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1878 {
1879         /* Calculate the DMIF Request Bandwidth */
1880         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1881         fixed20_12 disp_clk, sclk, bandwidth;
1882         fixed20_12 a, b1, b2;
1883         u32 min_bandwidth;
1884
1885         a.full = dfixed_const(1000);
1886         disp_clk.full = dfixed_const(wm->disp_clk);
1887         disp_clk.full = dfixed_div(disp_clk, a);
1888         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1889         b1.full = dfixed_mul(a, disp_clk);
1890
1891         a.full = dfixed_const(1000);
1892         sclk.full = dfixed_const(wm->sclk);
1893         sclk.full = dfixed_div(sclk, a);
1894         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1895         b2.full = dfixed_mul(a, sclk);
1896
1897         a.full = dfixed_const(10);
1898         disp_clk_request_efficiency.full = dfixed_const(8);
1899         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1900
1901         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1902
1903         a.full = dfixed_const(min_bandwidth);
1904         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1905
1906         return dfixed_trunc(bandwidth);
1907 }
1908
1909 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1910 {
1911         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1912         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1913         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1914         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1915
1916         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1917 }
1918
1919 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1920 {
1921         /* Calculate the display mode Average Bandwidth
1922          * DisplayMode should contain the source and destination dimensions,
1923          * timing, etc.
1924          */
1925         fixed20_12 bpp;
1926         fixed20_12 line_time;
1927         fixed20_12 src_width;
1928         fixed20_12 bandwidth;
1929         fixed20_12 a;
1930
1931         a.full = dfixed_const(1000);
1932         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1933         line_time.full = dfixed_div(line_time, a);
1934         bpp.full = dfixed_const(wm->bytes_per_pixel);
1935         src_width.full = dfixed_const(wm->src_width);
1936         bandwidth.full = dfixed_mul(src_width, bpp);
1937         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1938         bandwidth.full = dfixed_div(bandwidth, line_time);
1939
1940         return dfixed_trunc(bandwidth);
1941 }
1942
1943 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1944 {
1945         /* First calcualte the latency in ns */
1946         u32 mc_latency = 2000; /* 2000 ns. */
1947         u32 available_bandwidth = dce6_available_bandwidth(wm);
1948         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1949         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1950         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1951         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1952                 (wm->num_heads * cursor_line_pair_return_time);
1953         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1954         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1955         u32 tmp, dmif_size = 12288;
1956         fixed20_12 a, b, c;
1957
1958         if (wm->num_heads == 0)
1959                 return 0;
1960
1961         a.full = dfixed_const(2);
1962         b.full = dfixed_const(1);
1963         if ((wm->vsc.full > a.full) ||
1964             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1965             (wm->vtaps >= 5) ||
1966             ((wm->vsc.full >= a.full) && wm->interlaced))
1967                 max_src_lines_per_dst_line = 4;
1968         else
1969                 max_src_lines_per_dst_line = 2;
1970
1971         a.full = dfixed_const(available_bandwidth);
1972         b.full = dfixed_const(wm->num_heads);
1973         a.full = dfixed_div(a, b);
1974
1975         b.full = dfixed_const(mc_latency + 512);
1976         c.full = dfixed_const(wm->disp_clk);
1977         b.full = dfixed_div(b, c);
1978
1979         c.full = dfixed_const(dmif_size);
1980         b.full = dfixed_div(c, b);
1981
1982         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1983
1984         b.full = dfixed_const(1000);
1985         c.full = dfixed_const(wm->disp_clk);
1986         b.full = dfixed_div(c, b);
1987         c.full = dfixed_const(wm->bytes_per_pixel);
1988         b.full = dfixed_mul(b, c);
1989
1990         lb_fill_bw = min(tmp, dfixed_trunc(b));
1991
1992         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1993         b.full = dfixed_const(1000);
1994         c.full = dfixed_const(lb_fill_bw);
1995         b.full = dfixed_div(c, b);
1996         a.full = dfixed_div(a, b);
1997         line_fill_time = dfixed_trunc(a);
1998
1999         if (line_fill_time < wm->active_time)
2000                 return latency;
2001         else
2002                 return latency + (line_fill_time - wm->active_time);
2003
2004 }
2005
2006 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2007 {
2008         if (dce6_average_bandwidth(wm) <=
2009             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2010                 return true;
2011         else
2012                 return false;
2013 };
2014
2015 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2016 {
2017         if (dce6_average_bandwidth(wm) <=
2018             (dce6_available_bandwidth(wm) / wm->num_heads))
2019                 return true;
2020         else
2021                 return false;
2022 };
2023
2024 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2025 {
2026         u32 lb_partitions = wm->lb_size / wm->src_width;
2027         u32 line_time = wm->active_time + wm->blank_time;
2028         u32 latency_tolerant_lines;
2029         u32 latency_hiding;
2030         fixed20_12 a;
2031
2032         a.full = dfixed_const(1);
2033         if (wm->vsc.full > a.full)
2034                 latency_tolerant_lines = 1;
2035         else {
2036                 if (lb_partitions <= (wm->vtaps + 1))
2037                         latency_tolerant_lines = 1;
2038                 else
2039                         latency_tolerant_lines = 2;
2040         }
2041
2042         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2043
2044         if (dce6_latency_watermark(wm) <= latency_hiding)
2045                 return true;
2046         else
2047                 return false;
2048 }
2049
2050 static void dce6_program_watermarks(struct radeon_device *rdev,
2051                                          struct radeon_crtc *radeon_crtc,
2052                                          u32 lb_size, u32 num_heads)
2053 {
2054         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2055         struct dce6_wm_params wm_low, wm_high;
2056         u32 dram_channels;
2057         u32 pixel_period;
2058         u32 line_time = 0;
2059         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2060         u32 priority_a_mark = 0, priority_b_mark = 0;
2061         u32 priority_a_cnt = PRIORITY_OFF;
2062         u32 priority_b_cnt = PRIORITY_OFF;
2063         u32 tmp, arb_control3;
2064         fixed20_12 a, b, c;
2065
2066         if (radeon_crtc->base.enabled && num_heads && mode) {
2067                 pixel_period = 1000000 / (u32)mode->clock;
2068                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2069                 priority_a_cnt = 0;
2070                 priority_b_cnt = 0;
2071
2072                 if (rdev->family == CHIP_ARUBA)
2073                         dram_channels = evergreen_get_number_of_dram_channels(rdev);
2074                 else
2075                         dram_channels = si_get_number_of_dram_channels(rdev);
2076
2077                 /* watermark for high clocks */
2078                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2079                         wm_high.yclk =
2080                                 radeon_dpm_get_mclk(rdev, false) * 10;
2081                         wm_high.sclk =
2082                                 radeon_dpm_get_sclk(rdev, false) * 10;
2083                 } else {
2084                         wm_high.yclk = rdev->pm.current_mclk * 10;
2085                         wm_high.sclk = rdev->pm.current_sclk * 10;
2086                 }
2087
2088                 wm_high.disp_clk = mode->clock;
2089                 wm_high.src_width = mode->crtc_hdisplay;
2090                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2091                 wm_high.blank_time = line_time - wm_high.active_time;
2092                 wm_high.interlaced = false;
2093                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2094                         wm_high.interlaced = true;
2095                 wm_high.vsc = radeon_crtc->vsc;
2096                 wm_high.vtaps = 1;
2097                 if (radeon_crtc->rmx_type != RMX_OFF)
2098                         wm_high.vtaps = 2;
2099                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2100                 wm_high.lb_size = lb_size;
2101                 wm_high.dram_channels = dram_channels;
2102                 wm_high.num_heads = num_heads;
2103
2104                 /* watermark for low clocks */
2105                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2106                         wm_low.yclk =
2107                                 radeon_dpm_get_mclk(rdev, true) * 10;
2108                         wm_low.sclk =
2109                                 radeon_dpm_get_sclk(rdev, true) * 10;
2110                 } else {
2111                         wm_low.yclk = rdev->pm.current_mclk * 10;
2112                         wm_low.sclk = rdev->pm.current_sclk * 10;
2113                 }
2114
2115                 wm_low.disp_clk = mode->clock;
2116                 wm_low.src_width = mode->crtc_hdisplay;
2117                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2118                 wm_low.blank_time = line_time - wm_low.active_time;
2119                 wm_low.interlaced = false;
2120                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2121                         wm_low.interlaced = true;
2122                 wm_low.vsc = radeon_crtc->vsc;
2123                 wm_low.vtaps = 1;
2124                 if (radeon_crtc->rmx_type != RMX_OFF)
2125                         wm_low.vtaps = 2;
2126                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2127                 wm_low.lb_size = lb_size;
2128                 wm_low.dram_channels = dram_channels;
2129                 wm_low.num_heads = num_heads;
2130
2131                 /* set for high clocks */
2132                 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2133                 /* set for low clocks */
2134                 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2135
2136                 /* possibly force display priority to high */
2137                 /* should really do this at mode validation time... */
2138                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2139                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2140                     !dce6_check_latency_hiding(&wm_high) ||
2141                     (rdev->disp_priority == 2)) {
2142                         DRM_DEBUG_KMS("force priority to high\n");
2143                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2144                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2145                 }
2146                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2147                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2148                     !dce6_check_latency_hiding(&wm_low) ||
2149                     (rdev->disp_priority == 2)) {
2150                         DRM_DEBUG_KMS("force priority to high\n");
2151                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2152                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2153                 }
2154
2155                 a.full = dfixed_const(1000);
2156                 b.full = dfixed_const(mode->clock);
2157                 b.full = dfixed_div(b, a);
2158                 c.full = dfixed_const(latency_watermark_a);
2159                 c.full = dfixed_mul(c, b);
2160                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2161                 c.full = dfixed_div(c, a);
2162                 a.full = dfixed_const(16);
2163                 c.full = dfixed_div(c, a);
2164                 priority_a_mark = dfixed_trunc(c);
2165                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2166
2167                 a.full = dfixed_const(1000);
2168                 b.full = dfixed_const(mode->clock);
2169                 b.full = dfixed_div(b, a);
2170                 c.full = dfixed_const(latency_watermark_b);
2171                 c.full = dfixed_mul(c, b);
2172                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2173                 c.full = dfixed_div(c, a);
2174                 a.full = dfixed_const(16);
2175                 c.full = dfixed_div(c, a);
2176                 priority_b_mark = dfixed_trunc(c);
2177                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2178         }
2179
2180         /* select wm A */
2181         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2182         tmp = arb_control3;
2183         tmp &= ~LATENCY_WATERMARK_MASK(3);
2184         tmp |= LATENCY_WATERMARK_MASK(1);
2185         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2186         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2187                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2188                 LATENCY_HIGH_WATERMARK(line_time)));
2189         /* select wm B */
2190         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2191         tmp &= ~LATENCY_WATERMARK_MASK(3);
2192         tmp |= LATENCY_WATERMARK_MASK(2);
2193         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2194         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2195                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2196                 LATENCY_HIGH_WATERMARK(line_time)));
2197         /* restore original selection */
2198         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2199
2200         /* write the priority marks */
2201         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2202         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2203
2204         /* save values for DPM */
2205         radeon_crtc->line_time = line_time;
2206         radeon_crtc->wm_high = latency_watermark_a;
2207         radeon_crtc->wm_low = latency_watermark_b;
2208 }
2209
2210 void dce6_bandwidth_update(struct radeon_device *rdev)
2211 {
2212         struct drm_display_mode *mode0 = NULL;
2213         struct drm_display_mode *mode1 = NULL;
2214         u32 num_heads = 0, lb_size;
2215         int i;
2216
2217         radeon_update_display_priority(rdev);
2218
2219         for (i = 0; i < rdev->num_crtc; i++) {
2220                 if (rdev->mode_info.crtcs[i]->base.enabled)
2221                         num_heads++;
2222         }
2223         for (i = 0; i < rdev->num_crtc; i += 2) {
2224                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2225                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2226                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2227                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2228                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2229                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2230         }
2231 }
2232
2233 /*
2234  * Core functions
2235  */
2236 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2237 {
2238         const u32 num_tile_mode_states = 32;
2239         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2240
2241         switch (rdev->config.si.mem_row_size_in_kb) {
2242         case 1:
2243                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2244                 break;
2245         case 2:
2246         default:
2247                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2248                 break;
2249         case 4:
2250                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2251                 break;
2252         }
2253
2254         if ((rdev->family == CHIP_TAHITI) ||
2255             (rdev->family == CHIP_PITCAIRN)) {
2256                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2257                         switch (reg_offset) {
2258                         case 0:  /* non-AA compressed depth or any compressed stencil */
2259                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2260                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2261                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2262                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2263                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2264                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2265                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2266                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2267                                 break;
2268                         case 1:  /* 2xAA/4xAA compressed depth only */
2269                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2270                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2271                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2272                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2273                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2274                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2275                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2276                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2277                                 break;
2278                         case 2:  /* 8xAA compressed depth only */
2279                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2280                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2281                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2282                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2283                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2284                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2285                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2286                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2287                                 break;
2288                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2289                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2291                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2292                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2293                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2294                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2296                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2297                                 break;
2298                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2299                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2300                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2301                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2302                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2303                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2304                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2305                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2306                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2307                                 break;
2308                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2309                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2310                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2311                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2312                                                  TILE_SPLIT(split_equal_to_row_size) |
2313                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2314                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2315                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2316                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2317                                 break;
2318                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2319                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2320                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2321                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2322                                                  TILE_SPLIT(split_equal_to_row_size) |
2323                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2324                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2325                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2326                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2327                                 break;
2328                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2329                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2330                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2331                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2332                                                  TILE_SPLIT(split_equal_to_row_size) |
2333                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2334                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2335                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2336                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2337                                 break;
2338                         case 8:  /* 1D and 1D Array Surfaces */
2339                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2340                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2341                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2342                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2343                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2344                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2345                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2346                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2347                                 break;
2348                         case 9:  /* Displayable maps. */
2349                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2350                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2351                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2352                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2353                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2354                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2355                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2356                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2357                                 break;
2358                         case 10:  /* Display 8bpp. */
2359                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2360                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2361                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2362                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2363                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2364                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2365                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2366                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2367                                 break;
2368                         case 11:  /* Display 16bpp. */
2369                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2370                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2371                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2372                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2373                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2374                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2376                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2377                                 break;
2378                         case 12:  /* Display 32bpp. */
2379                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2380                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2381                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2382                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2383                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2384                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2385                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2386                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2387                                 break;
2388                         case 13:  /* Thin. */
2389                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2390                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2391                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2392                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2393                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2394                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2395                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2396                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2397                                 break;
2398                         case 14:  /* Thin 8 bpp. */
2399                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2400                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2401                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2402                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2403                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2404                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2406                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2407                                 break;
2408                         case 15:  /* Thin 16 bpp. */
2409                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2410                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2411                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2412                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2413                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2414                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2416                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2417                                 break;
2418                         case 16:  /* Thin 32 bpp. */
2419                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2420                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2421                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2422                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2423                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2424                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2425                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2426                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2427                                 break;
2428                         case 17:  /* Thin 64 bpp. */
2429                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2431                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2432                                                  TILE_SPLIT(split_equal_to_row_size) |
2433                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2434                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2437                                 break;
2438                         case 21:  /* 8 bpp PRT. */
2439                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2441                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2442                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2443                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2444                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2445                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2446                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2447                                 break;
2448                         case 22:  /* 16 bpp PRT */
2449                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2451                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2452                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2453                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2454                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2455                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2456                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2457                                 break;
2458                         case 23:  /* 32 bpp PRT */
2459                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2461                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2462                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2463                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2464                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2466                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2467                                 break;
2468                         case 24:  /* 64 bpp PRT */
2469                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2470                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2471                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2472                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2473                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2474                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2475                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2476                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2477                                 break;
2478                         case 25:  /* 128 bpp PRT */
2479                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2481                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2482                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2483                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2484                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2487                                 break;
2488                         default:
2489                                 gb_tile_moden = 0;
2490                                 break;
2491                         }
2492                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2493                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2494                 }
2495         } else if ((rdev->family == CHIP_VERDE) ||
2496                    (rdev->family == CHIP_OLAND) ||
2497                    (rdev->family == CHIP_HAINAN)) {
2498                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2499                         switch (reg_offset) {
2500                         case 0:  /* non-AA compressed depth or any compressed stencil */
2501                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2502                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2503                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2504                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2505                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2506                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2507                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2508                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2509                                 break;
2510                         case 1:  /* 2xAA/4xAA compressed depth only */
2511                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2512                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2513                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2514                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2515                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2516                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2517                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2518                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2519                                 break;
2520                         case 2:  /* 8xAA compressed depth only */
2521                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2523                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2524                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2525                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2526                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2527                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2528                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2529                                 break;
2530                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2531                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2533                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2534                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2535                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2536                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2537                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2538                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2539                                 break;
2540                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2541                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2542                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2543                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2545                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2546                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2548                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2549                                 break;
2550                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2551                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2553                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2554                                                  TILE_SPLIT(split_equal_to_row_size) |
2555                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2556                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559                                 break;
2560                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2561                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2562                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2563                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2564                                                  TILE_SPLIT(split_equal_to_row_size) |
2565                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2566                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2567                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2568                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2569                                 break;
2570                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2571                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2572                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2573                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2574                                                  TILE_SPLIT(split_equal_to_row_size) |
2575                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2576                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2577                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2578                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2579                                 break;
2580                         case 8:  /* 1D and 1D Array Surfaces */
2581                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2582                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2585                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2586                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2587                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2588                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2589                                 break;
2590                         case 9:  /* Displayable maps. */
2591                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2592                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2593                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2594                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2595                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2596                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2597                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2598                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2599                                 break;
2600                         case 10:  /* Display 8bpp. */
2601                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2602                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2603                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2604                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2605                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2606                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2607                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2608                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2609                                 break;
2610                         case 11:  /* Display 16bpp. */
2611                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2612                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2613                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2615                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2616                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2617                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2618                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2619                                 break;
2620                         case 12:  /* Display 32bpp. */
2621                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2622                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2623                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2624                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2625                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2626                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2627                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2628                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2629                                 break;
2630                         case 13:  /* Thin. */
2631                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2632                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2634                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2635                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2636                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2638                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2639                                 break;
2640                         case 14:  /* Thin 8 bpp. */
2641                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2643                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2644                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2646                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2648                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2649                                 break;
2650                         case 15:  /* Thin 16 bpp. */
2651                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2652                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2653                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2655                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2656                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2657                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2658                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2659                                 break;
2660                         case 16:  /* Thin 32 bpp. */
2661                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2663                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2665                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2666                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2669                                 break;
2670                         case 17:  /* Thin 64 bpp. */
2671                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2673                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2674                                                  TILE_SPLIT(split_equal_to_row_size) |
2675                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2676                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2677                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2678                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2679                                 break;
2680                         case 21:  /* 8 bpp PRT. */
2681                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2682                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2683                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2684                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2685                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2686                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2687                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2688                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2689                                 break;
2690                         case 22:  /* 16 bpp PRT */
2691                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2692                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2693                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2694                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2695                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2696                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2698                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2699                                 break;
2700                         case 23:  /* 32 bpp PRT */
2701                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2702                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2703                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2704                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2705                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2706                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2707                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2708                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2709                                 break;
2710                         case 24:  /* 64 bpp PRT */
2711                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2713                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2714                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2715                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2716                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2718                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2719                                 break;
2720                         case 25:  /* 128 bpp PRT */
2721                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2722                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2723                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2724                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2725                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2726                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2729                                 break;
2730                         default:
2731                                 gb_tile_moden = 0;
2732                                 break;
2733                         }
2734                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2735                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2736                 }
2737         } else
2738                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2739 }
2740
2741 static void si_select_se_sh(struct radeon_device *rdev,
2742                             u32 se_num, u32 sh_num)
2743 {
2744         u32 data = INSTANCE_BROADCAST_WRITES;
2745
2746         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2747                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2748         else if (se_num == 0xffffffff)
2749                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2750         else if (sh_num == 0xffffffff)
2751                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2752         else
2753                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2754         WREG32(GRBM_GFX_INDEX, data);
2755 }
2756
2757 static u32 si_create_bitmask(u32 bit_width)
2758 {
2759         u32 i, mask = 0;
2760
2761         for (i = 0; i < bit_width; i++) {
2762                 mask <<= 1;
2763                 mask |= 1;
2764         }
2765         return mask;
2766 }
2767
2768 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2769 {
2770         u32 data, mask;
2771
2772         data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2773         if (data & 1)
2774                 data &= INACTIVE_CUS_MASK;
2775         else
2776                 data = 0;
2777         data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2778
2779         data >>= INACTIVE_CUS_SHIFT;
2780
2781         mask = si_create_bitmask(cu_per_sh);
2782
2783         return ~data & mask;
2784 }
2785
2786 static void si_setup_spi(struct radeon_device *rdev,
2787                          u32 se_num, u32 sh_per_se,
2788                          u32 cu_per_sh)
2789 {
2790         int i, j, k;
2791         u32 data, mask, active_cu;
2792
2793         for (i = 0; i < se_num; i++) {
2794                 for (j = 0; j < sh_per_se; j++) {
2795                         si_select_se_sh(rdev, i, j);
2796                         data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2797                         active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2798
2799                         mask = 1;
2800                         for (k = 0; k < 16; k++) {
2801                                 mask <<= k;
2802                                 if (active_cu & mask) {
2803                                         data &= ~mask;
2804                                         WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2805                                         break;
2806                                 }
2807                         }
2808                 }
2809         }
2810         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2811 }
2812
2813 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2814                               u32 max_rb_num_per_se,
2815                               u32 sh_per_se)
2816 {
2817         u32 data, mask;
2818
2819         data = RREG32(CC_RB_BACKEND_DISABLE);
2820         if (data & 1)
2821                 data &= BACKEND_DISABLE_MASK;
2822         else
2823                 data = 0;
2824         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2825
2826         data >>= BACKEND_DISABLE_SHIFT;
2827
2828         mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2829
2830         return data & mask;
2831 }
2832
2833 static void si_setup_rb(struct radeon_device *rdev,
2834                         u32 se_num, u32 sh_per_se,
2835                         u32 max_rb_num_per_se)
2836 {
2837         int i, j;
2838         u32 data, mask;
2839         u32 disabled_rbs = 0;
2840         u32 enabled_rbs = 0;
2841
2842         for (i = 0; i < se_num; i++) {
2843                 for (j = 0; j < sh_per_se; j++) {
2844                         si_select_se_sh(rdev, i, j);
2845                         data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2846                         disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2847                 }
2848         }
2849         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2850
2851         mask = 1;
2852         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2853                 if (!(disabled_rbs & mask))
2854                         enabled_rbs |= mask;
2855                 mask <<= 1;
2856         }
2857
2858         rdev->config.si.backend_enable_mask = enabled_rbs;
2859
2860         for (i = 0; i < se_num; i++) {
2861                 si_select_se_sh(rdev, i, 0xffffffff);
2862                 data = 0;
2863                 for (j = 0; j < sh_per_se; j++) {
2864                         switch (enabled_rbs & 3) {
2865                         case 1:
2866                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2867                                 break;
2868                         case 2:
2869                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2870                                 break;
2871                         case 3:
2872                         default:
2873                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2874                                 break;
2875                         }
2876                         enabled_rbs >>= 2;
2877                 }
2878                 WREG32(PA_SC_RASTER_CONFIG, data);
2879         }
2880         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2881 }
2882
2883 static void si_gpu_init(struct radeon_device *rdev)
2884 {
2885         u32 gb_addr_config = 0;
2886         u32 mc_shared_chmap, mc_arb_ramcfg;
2887         u32 sx_debug_1;
2888         u32 hdp_host_path_cntl;
2889         u32 tmp;
2890         int i, j;
2891
2892         switch (rdev->family) {
2893         case CHIP_TAHITI:
2894                 rdev->config.si.max_shader_engines = 2;
2895                 rdev->config.si.max_tile_pipes = 12;
2896                 rdev->config.si.max_cu_per_sh = 8;
2897                 rdev->config.si.max_sh_per_se = 2;
2898                 rdev->config.si.max_backends_per_se = 4;
2899                 rdev->config.si.max_texture_channel_caches = 12;
2900                 rdev->config.si.max_gprs = 256;
2901                 rdev->config.si.max_gs_threads = 32;
2902                 rdev->config.si.max_hw_contexts = 8;
2903
2904                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2905                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2906                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2907                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2908                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2909                 break;
2910         case CHIP_PITCAIRN:
2911                 rdev->config.si.max_shader_engines = 2;
2912                 rdev->config.si.max_tile_pipes = 8;
2913                 rdev->config.si.max_cu_per_sh = 5;
2914                 rdev->config.si.max_sh_per_se = 2;
2915                 rdev->config.si.max_backends_per_se = 4;
2916                 rdev->config.si.max_texture_channel_caches = 8;
2917                 rdev->config.si.max_gprs = 256;
2918                 rdev->config.si.max_gs_threads = 32;
2919                 rdev->config.si.max_hw_contexts = 8;
2920
2921                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2922                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2923                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2924                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2925                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2926                 break;
2927         case CHIP_VERDE:
2928         default:
2929                 rdev->config.si.max_shader_engines = 1;
2930                 rdev->config.si.max_tile_pipes = 4;
2931                 rdev->config.si.max_cu_per_sh = 5;
2932                 rdev->config.si.max_sh_per_se = 2;
2933                 rdev->config.si.max_backends_per_se = 4;
2934                 rdev->config.si.max_texture_channel_caches = 4;
2935                 rdev->config.si.max_gprs = 256;
2936                 rdev->config.si.max_gs_threads = 32;
2937                 rdev->config.si.max_hw_contexts = 8;
2938
2939                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2940                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2941                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2942                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2943                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2944                 break;
2945         case CHIP_OLAND:
2946                 rdev->config.si.max_shader_engines = 1;
2947                 rdev->config.si.max_tile_pipes = 4;
2948                 rdev->config.si.max_cu_per_sh = 6;
2949                 rdev->config.si.max_sh_per_se = 1;
2950                 rdev->config.si.max_backends_per_se = 2;
2951                 rdev->config.si.max_texture_channel_caches = 4;
2952                 rdev->config.si.max_gprs = 256;
2953                 rdev->config.si.max_gs_threads = 16;
2954                 rdev->config.si.max_hw_contexts = 8;
2955
2956                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2957                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2958                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2959                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2960                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2961                 break;
2962         case CHIP_HAINAN:
2963                 rdev->config.si.max_shader_engines = 1;
2964                 rdev->config.si.max_tile_pipes = 4;
2965                 rdev->config.si.max_cu_per_sh = 5;
2966                 rdev->config.si.max_sh_per_se = 1;
2967                 rdev->config.si.max_backends_per_se = 1;
2968                 rdev->config.si.max_texture_channel_caches = 2;
2969                 rdev->config.si.max_gprs = 256;
2970                 rdev->config.si.max_gs_threads = 16;
2971                 rdev->config.si.max_hw_contexts = 8;
2972
2973                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2974                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2975                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2976                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2977                 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2978                 break;
2979         }
2980
2981         /* Initialize HDP */
2982         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2983                 WREG32((0x2c14 + j), 0x00000000);
2984                 WREG32((0x2c18 + j), 0x00000000);
2985                 WREG32((0x2c1c + j), 0x00000000);
2986                 WREG32((0x2c20 + j), 0x00000000);
2987                 WREG32((0x2c24 + j), 0x00000000);
2988         }
2989
2990         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2991
2992         evergreen_fix_pci_max_read_req_size(rdev);
2993
2994         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2995
2996         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2997         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2998
2999         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3000         rdev->config.si.mem_max_burst_length_bytes = 256;
3001         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3002         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3003         if (rdev->config.si.mem_row_size_in_kb > 4)
3004                 rdev->config.si.mem_row_size_in_kb = 4;
3005         /* XXX use MC settings? */
3006         rdev->config.si.shader_engine_tile_size = 32;
3007         rdev->config.si.num_gpus = 1;
3008         rdev->config.si.multi_gpu_tile_size = 64;
3009
3010         /* fix up row size */
3011         gb_addr_config &= ~ROW_SIZE_MASK;
3012         switch (rdev->config.si.mem_row_size_in_kb) {
3013         case 1:
3014         default:
3015                 gb_addr_config |= ROW_SIZE(0);
3016                 break;
3017         case 2:
3018                 gb_addr_config |= ROW_SIZE(1);
3019                 break;
3020         case 4:
3021                 gb_addr_config |= ROW_SIZE(2);
3022                 break;
3023         }
3024
3025         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3026          * not have bank info, so create a custom tiling dword.
3027          * bits 3:0   num_pipes
3028          * bits 7:4   num_banks
3029          * bits 11:8  group_size
3030          * bits 15:12 row_size
3031          */
3032         rdev->config.si.tile_config = 0;
3033         switch (rdev->config.si.num_tile_pipes) {
3034         case 1:
3035                 rdev->config.si.tile_config |= (0 << 0);
3036                 break;
3037         case 2:
3038                 rdev->config.si.tile_config |= (1 << 0);
3039                 break;
3040         case 4:
3041                 rdev->config.si.tile_config |= (2 << 0);
3042                 break;
3043         case 8:
3044         default:
3045                 /* XXX what about 12? */
3046                 rdev->config.si.tile_config |= (3 << 0);
3047                 break;
3048         }
3049         switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3050         case 0: /* four banks */
3051                 rdev->config.si.tile_config |= 0 << 4;
3052                 break;
3053         case 1: /* eight banks */
3054                 rdev->config.si.tile_config |= 1 << 4;
3055                 break;
3056         case 2: /* sixteen banks */
3057         default:
3058                 rdev->config.si.tile_config |= 2 << 4;
3059                 break;
3060         }
3061         rdev->config.si.tile_config |=
3062                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3063         rdev->config.si.tile_config |=
3064                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3065
3066         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3067         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3068         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3069         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3070         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3071         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3072         if (rdev->has_uvd) {
3073                 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3074                 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3075                 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3076         }
3077
3078         si_tiling_mode_table_init(rdev);
3079
3080         si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3081                     rdev->config.si.max_sh_per_se,
3082                     rdev->config.si.max_backends_per_se);
3083
3084         si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3085                      rdev->config.si.max_sh_per_se,
3086                      rdev->config.si.max_cu_per_sh);
3087
3088
3089         /* set HW defaults for 3D engine */
3090         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3091                                      ROQ_IB2_START(0x2b)));
3092         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3093
3094         sx_debug_1 = RREG32(SX_DEBUG_1);
3095         WREG32(SX_DEBUG_1, sx_debug_1);
3096
3097         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3098
3099         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3100                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3101                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3102                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3103
3104         WREG32(VGT_NUM_INSTANCES, 1);
3105
3106         WREG32(CP_PERFMON_CNTL, 0);
3107
3108         WREG32(SQ_CONFIG, 0);
3109
3110         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3111                                           FORCE_EOV_MAX_REZ_CNT(255)));
3112
3113         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3114                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3115
3116         WREG32(VGT_GS_VERTEX_REUSE, 16);
3117         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3118
3119         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3120         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3121         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3122         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3123         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3124         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3125         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3126         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3127
3128         tmp = RREG32(HDP_MISC_CNTL);
3129         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3130         WREG32(HDP_MISC_CNTL, tmp);
3131
3132         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3133         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3134
3135         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3136
3137         udelay(50);
3138 }
3139
3140 /*
3141  * GPU scratch registers helpers function.
3142  */
3143 static void si_scratch_init(struct radeon_device *rdev)
3144 {
3145         int i;
3146
3147         rdev->scratch.num_reg = 7;
3148         rdev->scratch.reg_base = SCRATCH_REG0;
3149         for (i = 0; i < rdev->scratch.num_reg; i++) {
3150                 rdev->scratch.free[i] = true;
3151                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3152         }
3153 }
3154
3155 void si_fence_ring_emit(struct radeon_device *rdev,
3156                         struct radeon_fence *fence)
3157 {
3158         struct radeon_ring *ring = &rdev->ring[fence->ring];
3159         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3160
3161         /* flush read cache over gart */
3162         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3163         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3164         radeon_ring_write(ring, 0);
3165         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3166         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3167                           PACKET3_TC_ACTION_ENA |
3168                           PACKET3_SH_KCACHE_ACTION_ENA |
3169                           PACKET3_SH_ICACHE_ACTION_ENA);
3170         radeon_ring_write(ring, 0xFFFFFFFF);
3171         radeon_ring_write(ring, 0);
3172         radeon_ring_write(ring, 10); /* poll interval */
3173         /* EVENT_WRITE_EOP - flush caches, send int */
3174         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3175         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3176         radeon_ring_write(ring, addr & 0xffffffff);
3177         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3178         radeon_ring_write(ring, fence->seq);
3179         radeon_ring_write(ring, 0);
3180 }
3181
3182 /*
3183  * IB stuff
3184  */
3185 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3186 {
3187         struct radeon_ring *ring = &rdev->ring[ib->ring];
3188         u32 header;
3189
3190         if (ib->is_const_ib) {
3191                 /* set switch buffer packet before const IB */
3192                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3193                 radeon_ring_write(ring, 0);
3194
3195                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3196         } else {
3197                 u32 next_rptr;
3198                 if (ring->rptr_save_reg) {
3199                         next_rptr = ring->wptr + 3 + 4 + 8;
3200                         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3201                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3202                                                   PACKET3_SET_CONFIG_REG_START) >> 2));
3203                         radeon_ring_write(ring, next_rptr);
3204                 } else if (rdev->wb.enabled) {
3205                         next_rptr = ring->wptr + 5 + 4 + 8;
3206                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3207                         radeon_ring_write(ring, (1 << 8));
3208                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3209                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3210                         radeon_ring_write(ring, next_rptr);
3211                 }
3212
3213                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3214         }
3215
3216         radeon_ring_write(ring, header);
3217         radeon_ring_write(ring,
3218 #ifdef __BIG_ENDIAN
3219                           (2 << 0) |
3220 #endif
3221                           (ib->gpu_addr & 0xFFFFFFFC));
3222         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3223         radeon_ring_write(ring, ib->length_dw |
3224                           (ib->vm ? (ib->vm->id << 24) : 0));
3225
3226         if (!ib->is_const_ib) {
3227                 /* flush read cache over gart for this vmid */
3228                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3229                 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3230                 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3231                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3232                 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3233                                   PACKET3_TC_ACTION_ENA |
3234                                   PACKET3_SH_KCACHE_ACTION_ENA |
3235                                   PACKET3_SH_ICACHE_ACTION_ENA);
3236                 radeon_ring_write(ring, 0xFFFFFFFF);
3237                 radeon_ring_write(ring, 0);
3238                 radeon_ring_write(ring, 10); /* poll interval */
3239         }
3240 }
3241
3242 /*
3243  * CP.
3244  */
3245 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3246 {
3247         if (enable)
3248                 WREG32(CP_ME_CNTL, 0);
3249         else {
3250                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3251                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3252                 WREG32(SCRATCH_UMSK, 0);
3253                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3254                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3255                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3256         }
3257         udelay(50);
3258 }
3259
3260 static int si_cp_load_microcode(struct radeon_device *rdev)
3261 {
3262         const __be32 *fw_data;
3263         int i;
3264
3265         if (!rdev->me_fw || !rdev->pfp_fw)
3266                 return -EINVAL;
3267
3268         si_cp_enable(rdev, false);
3269
3270         /* PFP */
3271         fw_data = (const __be32 *)rdev->pfp_fw->data;
3272         WREG32(CP_PFP_UCODE_ADDR, 0);
3273         for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3274                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3275         WREG32(CP_PFP_UCODE_ADDR, 0);
3276
3277         /* CE */
3278         fw_data = (const __be32 *)rdev->ce_fw->data;
3279         WREG32(CP_CE_UCODE_ADDR, 0);
3280         for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3281                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3282         WREG32(CP_CE_UCODE_ADDR, 0);
3283
3284         /* ME */
3285         fw_data = (const __be32 *)rdev->me_fw->data;
3286         WREG32(CP_ME_RAM_WADDR, 0);
3287         for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3288                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3289         WREG32(CP_ME_RAM_WADDR, 0);
3290
3291         WREG32(CP_PFP_UCODE_ADDR, 0);
3292         WREG32(CP_CE_UCODE_ADDR, 0);
3293         WREG32(CP_ME_RAM_WADDR, 0);
3294         WREG32(CP_ME_RAM_RADDR, 0);
3295         return 0;
3296 }
3297
3298 static int si_cp_start(struct radeon_device *rdev)
3299 {
3300         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3301         int r, i;
3302
3303         r = radeon_ring_lock(rdev, ring, 7 + 4);
3304         if (r) {
3305                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3306                 return r;
3307         }
3308         /* init the CP */
3309         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3310         radeon_ring_write(ring, 0x1);
3311         radeon_ring_write(ring, 0x0);
3312         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3313         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3314         radeon_ring_write(ring, 0);
3315         radeon_ring_write(ring, 0);
3316
3317         /* init the CE partitions */
3318         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3319         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3320         radeon_ring_write(ring, 0xc000);
3321         radeon_ring_write(ring, 0xe000);
3322         radeon_ring_unlock_commit(rdev, ring);
3323
3324         si_cp_enable(rdev, true);
3325
3326         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3327         if (r) {
3328                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3329                 return r;
3330         }
3331
3332         /* setup clear context state */
3333         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3334         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3335
3336         for (i = 0; i < si_default_size; i++)
3337                 radeon_ring_write(ring, si_default_state[i]);
3338
3339         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3340         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3341
3342         /* set clear context state */
3343         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3344         radeon_ring_write(ring, 0);
3345
3346         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3347         radeon_ring_write(ring, 0x00000316);
3348         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3349         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3350
3351         radeon_ring_unlock_commit(rdev, ring);
3352
3353         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3354                 ring = &rdev->ring[i];
3355                 r = radeon_ring_lock(rdev, ring, 2);
3356
3357                 /* clear the compute context state */
3358                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3359                 radeon_ring_write(ring, 0);
3360
3361                 radeon_ring_unlock_commit(rdev, ring);
3362         }
3363
3364         return 0;
3365 }
3366
3367 static void si_cp_fini(struct radeon_device *rdev)
3368 {
3369         struct radeon_ring *ring;
3370         si_cp_enable(rdev, false);
3371
3372         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3373         radeon_ring_fini(rdev, ring);
3374         radeon_scratch_free(rdev, ring->rptr_save_reg);
3375
3376         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3377         radeon_ring_fini(rdev, ring);
3378         radeon_scratch_free(rdev, ring->rptr_save_reg);
3379
3380         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3381         radeon_ring_fini(rdev, ring);
3382         radeon_scratch_free(rdev, ring->rptr_save_reg);
3383 }
3384
3385 static int si_cp_resume(struct radeon_device *rdev)
3386 {
3387         struct radeon_ring *ring;
3388         u32 tmp;
3389         u32 rb_bufsz;
3390         int r;
3391
3392         si_enable_gui_idle_interrupt(rdev, false);
3393
3394         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3395         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3396
3397         /* Set the write pointer delay */
3398         WREG32(CP_RB_WPTR_DELAY, 0);
3399
3400         WREG32(CP_DEBUG, 0);
3401         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3402
3403         /* ring 0 - compute and gfx */
3404         /* Set ring buffer size */
3405         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3406         rb_bufsz = order_base_2(ring->ring_size / 8);
3407         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3408 #ifdef __BIG_ENDIAN
3409         tmp |= BUF_SWAP_32BIT;
3410 #endif
3411         WREG32(CP_RB0_CNTL, tmp);
3412
3413         /* Initialize the ring buffer's read and write pointers */
3414         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3415         ring->wptr = 0;
3416         WREG32(CP_RB0_WPTR, ring->wptr);
3417
3418         /* set the wb address whether it's enabled or not */
3419         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3420         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3421
3422         if (rdev->wb.enabled)
3423                 WREG32(SCRATCH_UMSK, 0xff);
3424         else {
3425                 tmp |= RB_NO_UPDATE;
3426                 WREG32(SCRATCH_UMSK, 0);
3427         }
3428
3429         mdelay(1);
3430         WREG32(CP_RB0_CNTL, tmp);
3431
3432         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3433
3434         ring->rptr = RREG32(CP_RB0_RPTR);
3435
3436         /* ring1  - compute only */
3437         /* Set ring buffer size */
3438         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3439         rb_bufsz = order_base_2(ring->ring_size / 8);
3440         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3441 #ifdef __BIG_ENDIAN
3442         tmp |= BUF_SWAP_32BIT;
3443 #endif
3444         WREG32(CP_RB1_CNTL, tmp);
3445
3446         /* Initialize the ring buffer's read and write pointers */
3447         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3448         ring->wptr = 0;
3449         WREG32(CP_RB1_WPTR, ring->wptr);
3450
3451         /* set the wb address whether it's enabled or not */
3452         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3453         WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3454
3455         mdelay(1);
3456         WREG32(CP_RB1_CNTL, tmp);
3457
3458         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3459
3460         ring->rptr = RREG32(CP_RB1_RPTR);
3461
3462         /* ring2 - compute only */
3463         /* Set ring buffer size */
3464         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3465         rb_bufsz = order_base_2(ring->ring_size / 8);
3466         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3467 #ifdef __BIG_ENDIAN
3468         tmp |= BUF_SWAP_32BIT;
3469 #endif
3470         WREG32(CP_RB2_CNTL, tmp);
3471
3472         /* Initialize the ring buffer's read and write pointers */
3473         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3474         ring->wptr = 0;
3475         WREG32(CP_RB2_WPTR, ring->wptr);
3476
3477         /* set the wb address whether it's enabled or not */
3478         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3479         WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3480
3481         mdelay(1);
3482         WREG32(CP_RB2_CNTL, tmp);
3483
3484         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3485
3486         ring->rptr = RREG32(CP_RB2_RPTR);
3487
3488         /* start the rings */
3489         si_cp_start(rdev);
3490         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3491         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3492         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3493         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3494         if (r) {
3495                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3496                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3497                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3498                 return r;
3499         }
3500         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3501         if (r) {
3502                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3503         }
3504         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3505         if (r) {
3506                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3507         }
3508
3509         si_enable_gui_idle_interrupt(rdev, true);
3510
3511         return 0;
3512 }
3513
3514 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3515 {
3516         u32 reset_mask = 0;
3517         u32 tmp;
3518
3519         /* GRBM_STATUS */
3520         tmp = RREG32(GRBM_STATUS);
3521         if (tmp & (PA_BUSY | SC_BUSY |
3522                    BCI_BUSY | SX_BUSY |
3523                    TA_BUSY | VGT_BUSY |
3524                    DB_BUSY | CB_BUSY |
3525                    GDS_BUSY | SPI_BUSY |
3526                    IA_BUSY | IA_BUSY_NO_DMA))
3527                 reset_mask |= RADEON_RESET_GFX;
3528
3529         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3530                    CP_BUSY | CP_COHERENCY_BUSY))
3531                 reset_mask |= RADEON_RESET_CP;
3532
3533         if (tmp & GRBM_EE_BUSY)
3534                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3535
3536         /* GRBM_STATUS2 */
3537         tmp = RREG32(GRBM_STATUS2);
3538         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3539                 reset_mask |= RADEON_RESET_RLC;
3540
3541         /* DMA_STATUS_REG 0 */
3542         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3543         if (!(tmp & DMA_IDLE))
3544                 reset_mask |= RADEON_RESET_DMA;
3545
3546         /* DMA_STATUS_REG 1 */
3547         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3548         if (!(tmp & DMA_IDLE))
3549                 reset_mask |= RADEON_RESET_DMA1;
3550
3551         /* SRBM_STATUS2 */
3552         tmp = RREG32(SRBM_STATUS2);
3553         if (tmp & DMA_BUSY)
3554                 reset_mask |= RADEON_RESET_DMA;
3555
3556         if (tmp & DMA1_BUSY)
3557                 reset_mask |= RADEON_RESET_DMA1;
3558
3559         /* SRBM_STATUS */
3560         tmp = RREG32(SRBM_STATUS);
3561
3562         if (tmp & IH_BUSY)
3563                 reset_mask |= RADEON_RESET_IH;
3564
3565         if (tmp & SEM_BUSY)
3566                 reset_mask |= RADEON_RESET_SEM;
3567
3568         if (tmp & GRBM_RQ_PENDING)
3569                 reset_mask |= RADEON_RESET_GRBM;
3570
3571         if (tmp & VMC_BUSY)
3572                 reset_mask |= RADEON_RESET_VMC;
3573
3574         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3575                    MCC_BUSY | MCD_BUSY))
3576                 reset_mask |= RADEON_RESET_MC;
3577
3578         if (evergreen_is_display_hung(rdev))
3579                 reset_mask |= RADEON_RESET_DISPLAY;
3580
3581         /* VM_L2_STATUS */
3582         tmp = RREG32(VM_L2_STATUS);
3583         if (tmp & L2_BUSY)
3584                 reset_mask |= RADEON_RESET_VMC;
3585
3586         /* Skip MC reset as it's mostly likely not hung, just busy */
3587         if (reset_mask & RADEON_RESET_MC) {
3588                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3589                 reset_mask &= ~RADEON_RESET_MC;
3590         }
3591
3592         return reset_mask;
3593 }
3594
3595 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3596 {
3597         struct evergreen_mc_save save;
3598         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3599         u32 tmp;
3600
3601         if (reset_mask == 0)
3602                 return;
3603
3604         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3605
3606         evergreen_print_gpu_status_regs(rdev);
3607         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3608                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3609         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3610                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3611
3612         /* disable PG/CG */
3613         si_fini_pg(rdev);
3614         si_fini_cg(rdev);
3615
3616         /* stop the rlc */
3617         si_rlc_stop(rdev);
3618
3619         /* Disable CP parsing/prefetching */
3620         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3621
3622         if (reset_mask & RADEON_RESET_DMA) {
3623                 /* dma0 */
3624                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3625                 tmp &= ~DMA_RB_ENABLE;
3626                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3627         }
3628         if (reset_mask & RADEON_RESET_DMA1) {
3629                 /* dma1 */
3630                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3631                 tmp &= ~DMA_RB_ENABLE;
3632                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3633         }
3634
3635         udelay(50);
3636
3637         evergreen_mc_stop(rdev, &save);
3638         if (evergreen_mc_wait_for_idle(rdev)) {
3639                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3640         }
3641
3642         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3643                 grbm_soft_reset = SOFT_RESET_CB |
3644                         SOFT_RESET_DB |
3645                         SOFT_RESET_GDS |
3646                         SOFT_RESET_PA |
3647                         SOFT_RESET_SC |
3648                         SOFT_RESET_BCI |
3649                         SOFT_RESET_SPI |
3650                         SOFT_RESET_SX |
3651                         SOFT_RESET_TC |
3652                         SOFT_RESET_TA |
3653                         SOFT_RESET_VGT |
3654                         SOFT_RESET_IA;
3655         }
3656
3657         if (reset_mask & RADEON_RESET_CP) {
3658                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3659
3660                 srbm_soft_reset |= SOFT_RESET_GRBM;
3661         }
3662
3663         if (reset_mask & RADEON_RESET_DMA)
3664                 srbm_soft_reset |= SOFT_RESET_DMA;
3665
3666         if (reset_mask & RADEON_RESET_DMA1)
3667                 srbm_soft_reset |= SOFT_RESET_DMA1;
3668
3669         if (reset_mask & RADEON_RESET_DISPLAY)
3670                 srbm_soft_reset |= SOFT_RESET_DC;
3671
3672         if (reset_mask & RADEON_RESET_RLC)
3673                 grbm_soft_reset |= SOFT_RESET_RLC;
3674
3675         if (reset_mask & RADEON_RESET_SEM)
3676                 srbm_soft_reset |= SOFT_RESET_SEM;
3677
3678         if (reset_mask & RADEON_RESET_IH)
3679                 srbm_soft_reset |= SOFT_RESET_IH;
3680
3681         if (reset_mask & RADEON_RESET_GRBM)
3682                 srbm_soft_reset |= SOFT_RESET_GRBM;
3683
3684         if (reset_mask & RADEON_RESET_VMC)
3685                 srbm_soft_reset |= SOFT_RESET_VMC;
3686
3687         if (reset_mask & RADEON_RESET_MC)
3688                 srbm_soft_reset |= SOFT_RESET_MC;
3689
3690         if (grbm_soft_reset) {
3691                 tmp = RREG32(GRBM_SOFT_RESET);
3692                 tmp |= grbm_soft_reset;
3693                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3694                 WREG32(GRBM_SOFT_RESET, tmp);
3695                 tmp = RREG32(GRBM_SOFT_RESET);
3696
3697                 udelay(50);
3698
3699                 tmp &= ~grbm_soft_reset;
3700                 WREG32(GRBM_SOFT_RESET, tmp);
3701                 tmp = RREG32(GRBM_SOFT_RESET);
3702         }
3703
3704         if (srbm_soft_reset) {
3705                 tmp = RREG32(SRBM_SOFT_RESET);
3706                 tmp |= srbm_soft_reset;
3707                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3708                 WREG32(SRBM_SOFT_RESET, tmp);
3709                 tmp = RREG32(SRBM_SOFT_RESET);
3710
3711                 udelay(50);
3712
3713                 tmp &= ~srbm_soft_reset;
3714                 WREG32(SRBM_SOFT_RESET, tmp);
3715                 tmp = RREG32(SRBM_SOFT_RESET);
3716         }
3717
3718         /* Wait a little for things to settle down */
3719         udelay(50);
3720
3721         evergreen_mc_resume(rdev, &save);
3722         udelay(50);
3723
3724         evergreen_print_gpu_status_regs(rdev);
3725 }
3726
3727 int si_asic_reset(struct radeon_device *rdev)
3728 {
3729         u32 reset_mask;
3730
3731         reset_mask = si_gpu_check_soft_reset(rdev);
3732
3733         if (reset_mask)
3734                 r600_set_bios_scratch_engine_hung(rdev, true);
3735
3736         si_gpu_soft_reset(rdev, reset_mask);
3737
3738         reset_mask = si_gpu_check_soft_reset(rdev);
3739
3740         if (!reset_mask)
3741                 r600_set_bios_scratch_engine_hung(rdev, false);
3742
3743         return 0;
3744 }
3745
3746 /**
3747  * si_gfx_is_lockup - Check if the GFX engine is locked up
3748  *
3749  * @rdev: radeon_device pointer
3750  * @ring: radeon_ring structure holding ring information
3751  *
3752  * Check if the GFX engine is locked up.
3753  * Returns true if the engine appears to be locked up, false if not.
3754  */
3755 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3756 {
3757         u32 reset_mask = si_gpu_check_soft_reset(rdev);
3758
3759         if (!(reset_mask & (RADEON_RESET_GFX |
3760                             RADEON_RESET_COMPUTE |
3761                             RADEON_RESET_CP))) {
3762                 radeon_ring_lockup_update(ring);
3763                 return false;
3764         }
3765         /* force CP activities */
3766         radeon_ring_force_activity(rdev, ring);
3767         return radeon_ring_test_lockup(rdev, ring);
3768 }
3769
3770 /* MC */
3771 static void si_mc_program(struct radeon_device *rdev)
3772 {
3773         struct evergreen_mc_save save;
3774         u32 tmp;
3775         int i, j;
3776
3777         /* Initialize HDP */
3778         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3779                 WREG32((0x2c14 + j), 0x00000000);
3780                 WREG32((0x2c18 + j), 0x00000000);
3781                 WREG32((0x2c1c + j), 0x00000000);
3782                 WREG32((0x2c20 + j), 0x00000000);
3783                 WREG32((0x2c24 + j), 0x00000000);
3784         }
3785         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3786
3787         evergreen_mc_stop(rdev, &save);
3788         if (radeon_mc_wait_for_idle(rdev)) {
3789                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3790         }
3791         if (!ASIC_IS_NODCE(rdev))
3792                 /* Lockout access through VGA aperture*/
3793                 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3794         /* Update configuration */
3795         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3796                rdev->mc.vram_start >> 12);
3797         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3798                rdev->mc.vram_end >> 12);
3799         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3800                rdev->vram_scratch.gpu_addr >> 12);
3801         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3802         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3803         WREG32(MC_VM_FB_LOCATION, tmp);
3804         /* XXX double check these! */
3805         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3806         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3807         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3808         WREG32(MC_VM_AGP_BASE, 0);
3809         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3810         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3811         if (radeon_mc_wait_for_idle(rdev)) {
3812                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3813         }
3814         evergreen_mc_resume(rdev, &save);
3815         if (!ASIC_IS_NODCE(rdev)) {
3816                 /* we need to own VRAM, so turn off the VGA renderer here
3817                  * to stop it overwriting our objects */
3818                 rv515_vga_render_disable(rdev);
3819         }
3820 }
3821
3822 void si_vram_gtt_location(struct radeon_device *rdev,
3823                           struct radeon_mc *mc)
3824 {
3825         if (mc->mc_vram_size > 0xFFC0000000ULL) {
3826                 /* leave room for at least 1024M GTT */
3827                 dev_warn(rdev->dev, "limiting VRAM\n");
3828                 mc->real_vram_size = 0xFFC0000000ULL;
3829                 mc->mc_vram_size = 0xFFC0000000ULL;
3830         }
3831         radeon_vram_location(rdev, &rdev->mc, 0);
3832         rdev->mc.gtt_base_align = 0;
3833         radeon_gtt_location(rdev, mc);
3834 }
3835
3836 static int si_mc_init(struct radeon_device *rdev)
3837 {
3838         u32 tmp;
3839         int chansize, numchan;
3840
3841         /* Get VRAM informations */
3842         rdev->mc.vram_is_ddr = true;
3843         tmp = RREG32(MC_ARB_RAMCFG);
3844         if (tmp & CHANSIZE_OVERRIDE) {
3845                 chansize = 16;
3846         } else if (tmp & CHANSIZE_MASK) {
3847                 chansize = 64;
3848         } else {
3849                 chansize = 32;
3850         }
3851         tmp = RREG32(MC_SHARED_CHMAP);
3852         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3853         case 0:
3854         default:
3855                 numchan = 1;
3856                 break;
3857         case 1:
3858                 numchan = 2;
3859                 break;
3860         case 2:
3861                 numchan = 4;
3862                 break;
3863         case 3:
3864                 numchan = 8;
3865                 break;
3866         case 4:
3867                 numchan = 3;
3868                 break;
3869         case 5:
3870                 numchan = 6;
3871                 break;
3872         case 6:
3873                 numchan = 10;
3874                 break;
3875         case 7:
3876                 numchan = 12;
3877                 break;
3878         case 8:
3879                 numchan = 16;
3880                 break;
3881         }
3882         rdev->mc.vram_width = numchan * chansize;
3883         /* Could aper size report 0 ? */
3884         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3885         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3886         /* size in MB on si */
3887         tmp = RREG32(CONFIG_MEMSIZE);
3888         /* some boards may have garbage in the upper 16 bits */
3889         if (tmp & 0xffff0000) {
3890                 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
3891                 if (tmp & 0xffff)
3892                         tmp &= 0xffff;
3893         }
3894         rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
3895         rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
3896         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3897         si_vram_gtt_location(rdev, &rdev->mc);
3898         radeon_update_bandwidth_info(rdev);
3899
3900         return 0;
3901 }
3902
3903 /*
3904  * GART
3905  */
3906 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3907 {
3908         /* flush hdp cache */
3909         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3910
3911         /* bits 0-15 are the VM contexts0-15 */
3912         WREG32(VM_INVALIDATE_REQUEST, 1);
3913 }
3914
3915 static int si_pcie_gart_enable(struct radeon_device *rdev)
3916 {
3917         int r, i;
3918
3919         if (rdev->gart.robj == NULL) {
3920                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3921                 return -EINVAL;
3922         }
3923         r = radeon_gart_table_vram_pin(rdev);
3924         if (r)
3925                 return r;
3926         radeon_gart_restore(rdev);
3927         /* Setup TLB control */
3928         WREG32(MC_VM_MX_L1_TLB_CNTL,
3929                (0xA << 7) |
3930                ENABLE_L1_TLB |
3931                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3932                ENABLE_ADVANCED_DRIVER_MODEL |
3933                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3934         /* Setup L2 cache */
3935         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3936                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3937                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3938                EFFECTIVE_L2_QUEUE_SIZE(7) |
3939                CONTEXT1_IDENTITY_ACCESS_MODE(1));
3940         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3941         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3942                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3943         /* setup context0 */
3944         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3945         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3946         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3947         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3948                         (u32)(rdev->dummy_page.addr >> 12));
3949         WREG32(VM_CONTEXT0_CNTL2, 0);
3950         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3951                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3952
3953         WREG32(0x15D4, 0);
3954         WREG32(0x15D8, 0);
3955         WREG32(0x15DC, 0);
3956
3957         /* empty context1-15 */
3958         /* set vm size, must be a multiple of 4 */
3959         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3960         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3961         /* Assign the pt base to something valid for now; the pts used for
3962          * the VMs are determined by the application and setup and assigned
3963          * on the fly in the vm part of radeon_gart.c
3964          */
3965         for (i = 1; i < 16; i++) {
3966                 if (i < 8)
3967                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3968                                rdev->gart.table_addr >> 12);
3969                 else
3970                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3971                                rdev->gart.table_addr >> 12);
3972         }
3973
3974         /* enable context1-15 */
3975         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3976                (u32)(rdev->dummy_page.addr >> 12));
3977         WREG32(VM_CONTEXT1_CNTL2, 4);
3978         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3979                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3980                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3981                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3982                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3983                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3984                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3985                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3986                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3987                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3988                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3989                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3990                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3991
3992         si_pcie_gart_tlb_flush(rdev);
3993         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3994                  (unsigned)(rdev->mc.gtt_size >> 20),
3995                  (unsigned long long)rdev->gart.table_addr);
3996         rdev->gart.ready = true;
3997         return 0;
3998 }
3999
4000 static void si_pcie_gart_disable(struct radeon_device *rdev)
4001 {
4002         /* Disable all tables */
4003         WREG32(VM_CONTEXT0_CNTL, 0);
4004         WREG32(VM_CONTEXT1_CNTL, 0);
4005         /* Setup TLB control */
4006         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4007                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4008         /* Setup L2 cache */
4009         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4010                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4011                EFFECTIVE_L2_QUEUE_SIZE(7) |
4012                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4013         WREG32(VM_L2_CNTL2, 0);
4014         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4015                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4016         radeon_gart_table_vram_unpin(rdev);
4017 }
4018
4019 static void si_pcie_gart_fini(struct radeon_device *rdev)
4020 {
4021         si_pcie_gart_disable(rdev);
4022         radeon_gart_table_vram_free(rdev);
4023         radeon_gart_fini(rdev);
4024 }
4025
4026 /* vm parser */
4027 static bool si_vm_reg_valid(u32 reg)
4028 {
4029         /* context regs are fine */
4030         if (reg >= 0x28000)
4031                 return true;
4032
4033         /* check config regs */
4034         switch (reg) {
4035         case GRBM_GFX_INDEX:
4036         case CP_STRMOUT_CNTL:
4037         case VGT_VTX_VECT_EJECT_REG:
4038         case VGT_CACHE_INVALIDATION:
4039         case VGT_ESGS_RING_SIZE:
4040         case VGT_GSVS_RING_SIZE:
4041         case VGT_GS_VERTEX_REUSE:
4042         case VGT_PRIMITIVE_TYPE:
4043         case VGT_INDEX_TYPE:
4044         case VGT_NUM_INDICES:
4045         case VGT_NUM_INSTANCES:
4046         case VGT_TF_RING_SIZE:
4047         case VGT_HS_OFFCHIP_PARAM:
4048         case VGT_TF_MEMORY_BASE:
4049         case PA_CL_ENHANCE:
4050         case PA_SU_LINE_STIPPLE_VALUE:
4051         case PA_SC_LINE_STIPPLE_STATE:
4052         case PA_SC_ENHANCE:
4053         case SQC_CACHES:
4054         case SPI_STATIC_THREAD_MGMT_1:
4055         case SPI_STATIC_THREAD_MGMT_2:
4056         case SPI_STATIC_THREAD_MGMT_3:
4057         case SPI_PS_MAX_WAVE_ID:
4058         case SPI_CONFIG_CNTL:
4059         case SPI_CONFIG_CNTL_1:
4060         case TA_CNTL_AUX:
4061                 return true;
4062         default:
4063                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4064                 return false;
4065         }
4066 }
4067
4068 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4069                                   u32 *ib, struct radeon_cs_packet *pkt)
4070 {
4071         switch (pkt->opcode) {
4072         case PACKET3_NOP:
4073         case PACKET3_SET_BASE:
4074         case PACKET3_SET_CE_DE_COUNTERS:
4075         case PACKET3_LOAD_CONST_RAM:
4076         case PACKET3_WRITE_CONST_RAM:
4077         case PACKET3_WRITE_CONST_RAM_OFFSET:
4078         case PACKET3_DUMP_CONST_RAM:
4079         case PACKET3_INCREMENT_CE_COUNTER:
4080         case PACKET3_WAIT_ON_DE_COUNTER:
4081         case PACKET3_CE_WRITE:
4082                 break;
4083         default:
4084                 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4085                 return -EINVAL;
4086         }
4087         return 0;
4088 }
4089
4090 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4091 {
4092         u32 start_reg, reg, i;
4093         u32 command = ib[idx + 4];
4094         u32 info = ib[idx + 1];
4095         u32 idx_value = ib[idx];
4096         if (command & PACKET3_CP_DMA_CMD_SAS) {
4097                 /* src address space is register */
4098                 if (((info & 0x60000000) >> 29) == 0) {
4099                         start_reg = idx_value << 2;
4100                         if (command & PACKET3_CP_DMA_CMD_SAIC) {
4101                                 reg = start_reg;
4102                                 if (!si_vm_reg_valid(reg)) {
4103                                         DRM_ERROR("CP DMA Bad SRC register\n");
4104                                         return -EINVAL;
4105                                 }
4106                         } else {
4107                                 for (i = 0; i < (command & 0x1fffff); i++) {
4108                                         reg = start_reg + (4 * i);
4109                                         if (!si_vm_reg_valid(reg)) {
4110                                                 DRM_ERROR("CP DMA Bad SRC register\n");
4111                                                 return -EINVAL;
4112                                         }
4113                                 }
4114                         }
4115                 }
4116         }
4117         if (command & PACKET3_CP_DMA_CMD_DAS) {
4118                 /* dst address space is register */
4119                 if (((info & 0x00300000) >> 20) == 0) {
4120                         start_reg = ib[idx + 2];
4121                         if (command & PACKET3_CP_DMA_CMD_DAIC) {
4122                                 reg = start_reg;
4123                                 if (!si_vm_reg_valid(reg)) {
4124                                         DRM_ERROR("CP DMA Bad DST register\n");
4125                                         return -EINVAL;
4126                                 }
4127                         } else {
4128                                 for (i = 0; i < (command & 0x1fffff); i++) {
4129                                         reg = start_reg + (4 * i);
4130                                 if (!si_vm_reg_valid(reg)) {
4131                                                 DRM_ERROR("CP DMA Bad DST register\n");
4132                                                 return -EINVAL;
4133                                         }
4134                                 }
4135                         }
4136                 }
4137         }
4138         return 0;
4139 }
4140
4141 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4142                                    u32 *ib, struct radeon_cs_packet *pkt)
4143 {
4144         int r;
4145         u32 idx = pkt->idx + 1;
4146         u32 idx_value = ib[idx];
4147         u32 start_reg, end_reg, reg, i;
4148
4149         switch (pkt->opcode) {
4150         case PACKET3_NOP:
4151         case PACKET3_SET_BASE:
4152         case PACKET3_CLEAR_STATE:
4153         case PACKET3_INDEX_BUFFER_SIZE:
4154         case PACKET3_DISPATCH_DIRECT:
4155         case PACKET3_DISPATCH_INDIRECT:
4156         case PACKET3_ALLOC_GDS:
4157         case PACKET3_WRITE_GDS_RAM:
4158         case PACKET3_ATOMIC_GDS:
4159         case PACKET3_ATOMIC:
4160         case PACKET3_OCCLUSION_QUERY:
4161         case PACKET3_SET_PREDICATION:
4162         case PACKET3_COND_EXEC:
4163         case PACKET3_PRED_EXEC:
4164         case PACKET3_DRAW_INDIRECT:
4165         case PACKET3_DRAW_INDEX_INDIRECT:
4166         case PACKET3_INDEX_BASE:
4167         case PACKET3_DRAW_INDEX_2:
4168         case PACKET3_CONTEXT_CONTROL:
4169         case PACKET3_INDEX_TYPE:
4170         case PACKET3_DRAW_INDIRECT_MULTI:
4171         case PACKET3_DRAW_INDEX_AUTO:
4172         case PACKET3_DRAW_INDEX_IMMD:
4173         case PACKET3_NUM_INSTANCES:
4174         case PACKET3_DRAW_INDEX_MULTI_AUTO:
4175         case PACKET3_STRMOUT_BUFFER_UPDATE:
4176         case PACKET3_DRAW_INDEX_OFFSET_2:
4177         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4178         case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4179         case PACKET3_MPEG_INDEX:
4180         case PACKET3_WAIT_REG_MEM:
4181         case PACKET3_MEM_WRITE:
4182         case PACKET3_PFP_SYNC_ME:
4183         case PACKET3_SURFACE_SYNC:
4184         case PACKET3_EVENT_WRITE:
4185         case PACKET3_EVENT_WRITE_EOP:
4186         case PACKET3_EVENT_WRITE_EOS:
4187         case PACKET3_SET_CONTEXT_REG:
4188         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4189         case PACKET3_SET_SH_REG:
4190         case PACKET3_SET_SH_REG_OFFSET:
4191         case PACKET3_INCREMENT_DE_COUNTER:
4192         case PACKET3_WAIT_ON_CE_COUNTER:
4193         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4194         case PACKET3_ME_WRITE:
4195                 break;
4196         case PACKET3_COPY_DATA:
4197                 if ((idx_value & 0xf00) == 0) {
4198                         reg = ib[idx + 3] * 4;
4199                         if (!si_vm_reg_valid(reg))
4200                                 return -EINVAL;
4201                 }
4202                 break;
4203         case PACKET3_WRITE_DATA:
4204                 if ((idx_value & 0xf00) == 0) {
4205                         start_reg = ib[idx + 1] * 4;
4206                         if (idx_value & 0x10000) {
4207                                 if (!si_vm_reg_valid(start_reg))
4208                                         return -EINVAL;
4209                         } else {
4210                                 for (i = 0; i < (pkt->count - 2); i++) {
4211                                         reg = start_reg + (4 * i);
4212                                         if (!si_vm_reg_valid(reg))
4213                                                 return -EINVAL;
4214                                 }
4215                         }
4216                 }
4217                 break;
4218         case PACKET3_COND_WRITE:
4219                 if (idx_value & 0x100) {
4220                         reg = ib[idx + 5] * 4;
4221                         if (!si_vm_reg_valid(reg))
4222                                 return -EINVAL;
4223                 }
4224                 break;
4225         case PACKET3_COPY_DW:
4226                 if (idx_value & 0x2) {
4227                         reg = ib[idx + 3] * 4;
4228                         if (!si_vm_reg_valid(reg))
4229                                 return -EINVAL;
4230                 }
4231                 break;
4232         case PACKET3_SET_CONFIG_REG:
4233                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4234                 end_reg = 4 * pkt->count + start_reg - 4;
4235                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4236                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4237                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4238                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4239                         return -EINVAL;
4240                 }
4241                 for (i = 0; i < pkt->count; i++) {
4242                         reg = start_reg + (4 * i);
4243                         if (!si_vm_reg_valid(reg))
4244                                 return -EINVAL;
4245                 }
4246                 break;
4247         case PACKET3_CP_DMA:
4248                 r = si_vm_packet3_cp_dma_check(ib, idx);
4249                 if (r)
4250                         return r;
4251                 break;
4252         default:
4253                 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4254                 return -EINVAL;
4255         }
4256         return 0;
4257 }
4258
4259 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4260                                        u32 *ib, struct radeon_cs_packet *pkt)
4261 {
4262         int r;
4263         u32 idx = pkt->idx + 1;
4264         u32 idx_value = ib[idx];
4265         u32 start_reg, reg, i;
4266
4267         switch (pkt->opcode) {
4268         case PACKET3_NOP:
4269         case PACKET3_SET_BASE:
4270         case PACKET3_CLEAR_STATE:
4271         case PACKET3_DISPATCH_DIRECT:
4272         case PACKET3_DISPATCH_INDIRECT:
4273         case PACKET3_ALLOC_GDS:
4274         case PACKET3_WRITE_GDS_RAM:
4275         case PACKET3_ATOMIC_GDS:
4276         case PACKET3_ATOMIC:
4277         case PACKET3_OCCLUSION_QUERY:
4278         case PACKET3_SET_PREDICATION:
4279         case PACKET3_COND_EXEC:
4280         case PACKET3_PRED_EXEC:
4281         case PACKET3_CONTEXT_CONTROL:
4282         case PACKET3_STRMOUT_BUFFER_UPDATE:
4283         case PACKET3_WAIT_REG_MEM:
4284         case PACKET3_MEM_WRITE:
4285         case PACKET3_PFP_SYNC_ME:
4286         case PACKET3_SURFACE_SYNC:
4287         case PACKET3_EVENT_WRITE:
4288         case PACKET3_EVENT_WRITE_EOP:
4289         case PACKET3_EVENT_WRITE_EOS:
4290         case PACKET3_SET_CONTEXT_REG:
4291         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4292         case PACKET3_SET_SH_REG:
4293         case PACKET3_SET_SH_REG_OFFSET:
4294         case PACKET3_INCREMENT_DE_COUNTER:
4295         case PACKET3_WAIT_ON_CE_COUNTER:
4296         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4297         case PACKET3_ME_WRITE:
4298                 break;
4299         case PACKET3_COPY_DATA:
4300                 if ((idx_value & 0xf00) == 0) {
4301                         reg = ib[idx + 3] * 4;
4302                         if (!si_vm_reg_valid(reg))
4303                                 return -EINVAL;
4304                 }
4305                 break;
4306         case PACKET3_WRITE_DATA:
4307                 if ((idx_value & 0xf00) == 0) {
4308                         start_reg = ib[idx + 1] * 4;
4309                         if (idx_value & 0x10000) {
4310                                 if (!si_vm_reg_valid(start_reg))
4311                                         return -EINVAL;
4312                         } else {
4313                                 for (i = 0; i < (pkt->count - 2); i++) {
4314                                         reg = start_reg + (4 * i);
4315                                         if (!si_vm_reg_valid(reg))
4316                                                 return -EINVAL;
4317                                 }
4318                         }
4319                 }
4320                 break;
4321         case PACKET3_COND_WRITE:
4322                 if (idx_value & 0x100) {
4323                         reg = ib[idx + 5] * 4;
4324                         if (!si_vm_reg_valid(reg))
4325                                 return -EINVAL;
4326                 }
4327                 break;
4328         case PACKET3_COPY_DW:
4329                 if (idx_value & 0x2) {
4330                         reg = ib[idx + 3] * 4;
4331                         if (!si_vm_reg_valid(reg))
4332                                 return -EINVAL;
4333                 }
4334                 break;
4335         case PACKET3_CP_DMA:
4336                 r = si_vm_packet3_cp_dma_check(ib, idx);
4337                 if (r)
4338                         return r;
4339                 break;
4340         default:
4341                 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4342                 return -EINVAL;
4343         }
4344         return 0;
4345 }
4346
4347 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4348 {
4349         int ret = 0;
4350         u32 idx = 0;
4351         struct radeon_cs_packet pkt;
4352
4353         do {
4354                 pkt.idx = idx;
4355                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4356                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4357                 pkt.one_reg_wr = 0;
4358                 switch (pkt.type) {
4359                 case RADEON_PACKET_TYPE0:
4360                         dev_err(rdev->dev, "Packet0 not allowed!\n");
4361                         ret = -EINVAL;
4362                         break;
4363                 case RADEON_PACKET_TYPE2:
4364                         idx += 1;
4365                         break;
4366                 case RADEON_PACKET_TYPE3:
4367                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4368                         if (ib->is_const_ib)
4369                                 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4370                         else {
4371                                 switch (ib->ring) {
4372                                 case RADEON_RING_TYPE_GFX_INDEX:
4373                                         ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4374                                         break;
4375                                 case CAYMAN_RING_TYPE_CP1_INDEX:
4376                                 case CAYMAN_RING_TYPE_CP2_INDEX:
4377                                         ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4378                                         break;
4379                                 default:
4380                                         dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4381                                         ret = -EINVAL;
4382                                         break;
4383                                 }
4384                         }
4385                         idx += pkt.count + 2;
4386                         break;
4387                 default:
4388                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4389                         ret = -EINVAL;
4390                         break;
4391                 }
4392                 if (ret)
4393                         break;
4394         } while (idx < ib->length_dw);
4395
4396         return ret;
4397 }
4398
4399 /*
4400  * vm
4401  */
4402 int si_vm_init(struct radeon_device *rdev)
4403 {
4404         /* number of VMs */
4405         rdev->vm_manager.nvm = 16;
4406         /* base offset of vram pages */
4407         rdev->vm_manager.vram_base_offset = 0;
4408
4409         return 0;
4410 }
4411
4412 void si_vm_fini(struct radeon_device *rdev)
4413 {
4414 }
4415
4416 /**
4417  * si_vm_decode_fault - print human readable fault info
4418  *
4419  * @rdev: radeon_device pointer
4420  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4421  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4422  *
4423  * Print human readable fault information (SI).
4424  */
4425 static void si_vm_decode_fault(struct radeon_device *rdev,
4426                                u32 status, u32 addr)
4427 {
4428         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4429         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4430         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4431         char *block;
4432
4433         if (rdev->family == CHIP_TAHITI) {
4434                 switch (mc_id) {
4435                 case 160:
4436                 case 144:
4437                 case 96:
4438                 case 80:
4439                 case 224:
4440                 case 208:
4441                 case 32:
4442                 case 16:
4443                         block = "CB";
4444                         break;
4445                 case 161:
4446                 case 145:
4447                 case 97:
4448                 case 81:
4449                 case 225:
4450                 case 209:
4451                 case 33:
4452                 case 17:
4453                         block = "CB_FMASK";
4454                         break;
4455                 case 162:
4456                 case 146:
4457                 case 98:
4458                 case 82:
4459                 case 226:
4460                 case 210:
4461                 case 34:
4462                 case 18:
4463                         block = "CB_CMASK";
4464                         break;
4465                 case 163:
4466                 case 147:
4467                 case 99:
4468                 case 83:
4469                 case 227:
4470                 case 211:
4471                 case 35:
4472                 case 19:
4473                         block = "CB_IMMED";
4474                         break;
4475                 case 164:
4476                 case 148:
4477                 case 100:
4478                 case 84:
4479                 case 228:
4480                 case 212:
4481                 case 36:
4482                 case 20:
4483                         block = "DB";
4484                         break;
4485                 case 165:
4486                 case 149:
4487                 case 101:
4488                 case 85:
4489                 case 229:
4490                 case 213:
4491                 case 37:
4492                 case 21:
4493                         block = "DB_HTILE";
4494                         break;
4495                 case 167:
4496                 case 151:
4497                 case 103:
4498                 case 87:
4499                 case 231:
4500                 case 215:
4501                 case 39:
4502                 case 23:
4503                         block = "DB_STEN";
4504                         break;
4505                 case 72:
4506                 case 68:
4507                 case 64:
4508                 case 8:
4509                 case 4:
4510                 case 0:
4511                 case 136:
4512                 case 132:
4513                 case 128:
4514                 case 200:
4515                 case 196:
4516                 case 192:
4517                         block = "TC";
4518                         break;
4519                 case 112:
4520                 case 48:
4521                         block = "CP";
4522                         break;
4523                 case 49:
4524                 case 177:
4525                 case 50:
4526                 case 178:
4527                         block = "SH";
4528                         break;
4529                 case 53:
4530                 case 190:
4531                         block = "VGT";
4532                         break;
4533                 case 117:
4534                         block = "IH";
4535                         break;
4536                 case 51:
4537                 case 115:
4538                         block = "RLC";
4539                         break;
4540                 case 119:
4541                 case 183:
4542                         block = "DMA0";
4543                         break;
4544                 case 61:
4545                         block = "DMA1";
4546                         break;
4547                 case 248:
4548                 case 120:
4549                         block = "HDP";
4550                         break;
4551                 default:
4552                         block = "unknown";
4553                         break;
4554                 }
4555         } else {
4556                 switch (mc_id) {
4557                 case 32:
4558                 case 16:
4559                 case 96:
4560                 case 80:
4561                 case 160:
4562                 case 144:
4563                 case 224:
4564                 case 208:
4565                         block = "CB";
4566                         break;
4567                 case 33:
4568                 case 17:
4569                 case 97:
4570                 case 81:
4571                 case 161:
4572                 case 145:
4573                 case 225:
4574                 case 209:
4575                         block = "CB_FMASK";
4576                         break;
4577                 case 34:
4578                 case 18:
4579                 case 98:
4580                 case 82:
4581                 case 162:
4582                 case 146:
4583                 case 226:
4584                 case 210:
4585                         block = "CB_CMASK";
4586                         break;
4587                 case 35:
4588                 case 19:
4589                 case 99:
4590                 case 83:
4591                 case 163:
4592                 case 147:
4593                 case 227:
4594                 case 211:
4595                         block = "CB_IMMED";
4596                         break;
4597                 case 36:
4598                 case 20:
4599                 case 100:
4600                 case 84:
4601                 case 164:
4602                 case 148:
4603                 case 228:
4604                 case 212:
4605                         block = "DB";
4606                         break;
4607                 case 37:
4608                 case 21:
4609                 case 101:
4610                 case 85:
4611                 case 165:
4612                 case 149:
4613                 case 229:
4614                 case 213:
4615                         block = "DB_HTILE";
4616                         break;
4617                 case 39:
4618                 case 23:
4619                 case 103:
4620                 case 87:
4621                 case 167:
4622                 case 151:
4623                 case 231:
4624                 case 215:
4625                         block = "DB_STEN";
4626                         break;
4627                 case 72:
4628                 case 68:
4629                 case 8:
4630                 case 4:
4631                 case 136:
4632                 case 132:
4633                 case 200:
4634                 case 196:
4635                         block = "TC";
4636                         break;
4637                 case 112:
4638                 case 48:
4639                         block = "CP";
4640                         break;
4641                 case 49:
4642                 case 177:
4643                 case 50:
4644                 case 178:
4645                         block = "SH";
4646                         break;
4647                 case 53:
4648                         block = "VGT";
4649                         break;
4650                 case 117:
4651                         block = "IH";
4652                         break;
4653                 case 51:
4654                 case 115:
4655                         block = "RLC";
4656                         break;
4657                 case 119:
4658                 case 183:
4659                         block = "DMA0";
4660                         break;
4661                 case 61:
4662                         block = "DMA1";
4663                         break;
4664                 case 248:
4665                 case 120:
4666                         block = "HDP";
4667                         break;
4668                 default:
4669                         block = "unknown";
4670                         break;
4671                 }
4672         }
4673
4674         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4675                protections, vmid, addr,
4676                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4677                block, mc_id);
4678 }
4679
4680 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4681 {
4682         struct radeon_ring *ring = &rdev->ring[ridx];
4683
4684         if (vm == NULL)
4685                 return;
4686
4687         /* write new base address */
4688         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4689         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4690                                  WRITE_DATA_DST_SEL(0)));
4691
4692         if (vm->id < 8) {
4693                 radeon_ring_write(ring,
4694                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4695         } else {
4696                 radeon_ring_write(ring,
4697                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4698         }
4699         radeon_ring_write(ring, 0);
4700         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4701
4702         /* flush hdp cache */
4703         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4704         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4705                                  WRITE_DATA_DST_SEL(0)));
4706         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4707         radeon_ring_write(ring, 0);
4708         radeon_ring_write(ring, 0x1);
4709
4710         /* bits 0-15 are the VM contexts0-15 */
4711         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4712         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4713                                  WRITE_DATA_DST_SEL(0)));
4714         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4715         radeon_ring_write(ring, 0);
4716         radeon_ring_write(ring, 1 << vm->id);
4717
4718         /* sync PFP to ME, otherwise we might get invalid PFP reads */
4719         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4720         radeon_ring_write(ring, 0x0);
4721 }
4722
4723 /*
4724  *  Power and clock gating
4725  */
4726 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4727 {
4728         int i;
4729
4730         for (i = 0; i < rdev->usec_timeout; i++) {
4731                 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4732                         break;
4733                 udelay(1);
4734         }
4735
4736         for (i = 0; i < rdev->usec_timeout; i++) {
4737                 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4738                         break;
4739                 udelay(1);
4740         }
4741 }
4742
4743 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4744                                          bool enable)
4745 {
4746         u32 tmp = RREG32(CP_INT_CNTL_RING0);
4747         u32 mask;
4748         int i;
4749
4750         if (enable)
4751                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4752         else
4753                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4754         WREG32(CP_INT_CNTL_RING0, tmp);
4755
4756         if (!enable) {
4757                 /* read a gfx register */
4758                 tmp = RREG32(DB_DEPTH_INFO);
4759
4760                 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4761                 for (i = 0; i < rdev->usec_timeout; i++) {
4762                         if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4763                                 break;
4764                         udelay(1);
4765                 }
4766         }
4767 }
4768
4769 static void si_set_uvd_dcm(struct radeon_device *rdev,
4770                            bool sw_mode)
4771 {
4772         u32 tmp, tmp2;
4773
4774         tmp = RREG32(UVD_CGC_CTRL);
4775         tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4776         tmp |= DCM | CG_DT(1) | CLK_OD(4);
4777
4778         if (sw_mode) {
4779                 tmp &= ~0x7ffff800;
4780                 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4781         } else {
4782                 tmp |= 0x7ffff800;
4783                 tmp2 = 0;
4784         }
4785
4786         WREG32(UVD_CGC_CTRL, tmp);
4787         WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4788 }
4789
4790 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4791 {
4792         bool hw_mode = true;
4793
4794         if (hw_mode) {
4795                 si_set_uvd_dcm(rdev, false);
4796         } else {
4797                 u32 tmp = RREG32(UVD_CGC_CTRL);
4798                 tmp &= ~DCM;
4799                 WREG32(UVD_CGC_CTRL, tmp);
4800         }
4801 }
4802
4803 static u32 si_halt_rlc(struct radeon_device *rdev)
4804 {
4805         u32 data, orig;
4806
4807         orig = data = RREG32(RLC_CNTL);
4808
4809         if (data & RLC_ENABLE) {
4810                 data &= ~RLC_ENABLE;
4811                 WREG32(RLC_CNTL, data);
4812
4813                 si_wait_for_rlc_serdes(rdev);
4814         }
4815
4816         return orig;
4817 }
4818
4819 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4820 {
4821         u32 tmp;
4822
4823         tmp = RREG32(RLC_CNTL);
4824         if (tmp != rlc)
4825                 WREG32(RLC_CNTL, rlc);
4826 }
4827
4828 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4829 {
4830         u32 data, orig;
4831
4832         orig = data = RREG32(DMA_PG);
4833         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4834                 data |= PG_CNTL_ENABLE;
4835         else
4836                 data &= ~PG_CNTL_ENABLE;
4837         if (orig != data)
4838                 WREG32(DMA_PG, data);
4839 }
4840
4841 static void si_init_dma_pg(struct radeon_device *rdev)
4842 {
4843         u32 tmp;
4844
4845         WREG32(DMA_PGFSM_WRITE,  0x00002000);
4846         WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4847
4848         for (tmp = 0; tmp < 5; tmp++)
4849                 WREG32(DMA_PGFSM_WRITE, 0);
4850 }
4851
4852 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4853                                bool enable)
4854 {
4855         u32 tmp;
4856
4857         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4858                 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4859                 WREG32(RLC_TTOP_D, tmp);
4860
4861                 tmp = RREG32(RLC_PG_CNTL);
4862                 tmp |= GFX_PG_ENABLE;
4863                 WREG32(RLC_PG_CNTL, tmp);
4864
4865                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4866                 tmp |= AUTO_PG_EN;
4867                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4868         } else {
4869                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4870                 tmp &= ~AUTO_PG_EN;
4871                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4872
4873                 tmp = RREG32(DB_RENDER_CONTROL);
4874         }
4875 }
4876
4877 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4878 {
4879         u32 tmp;
4880
4881         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4882
4883         tmp = RREG32(RLC_PG_CNTL);
4884         tmp |= GFX_PG_SRC;
4885         WREG32(RLC_PG_CNTL, tmp);
4886
4887         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4888
4889         tmp = RREG32(RLC_AUTO_PG_CTRL);
4890
4891         tmp &= ~GRBM_REG_SGIT_MASK;
4892         tmp |= GRBM_REG_SGIT(0x700);
4893         tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4894         WREG32(RLC_AUTO_PG_CTRL, tmp);
4895 }
4896
4897 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4898 {
4899         u32 mask = 0, tmp, tmp1;
4900         int i;
4901
4902         si_select_se_sh(rdev, se, sh);
4903         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4904         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4905         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4906
4907         tmp &= 0xffff0000;
4908
4909         tmp |= tmp1;
4910         tmp >>= 16;
4911
4912         for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4913                 mask <<= 1;
4914                 mask |= 1;
4915         }
4916
4917         return (~tmp) & mask;
4918 }
4919
4920 static void si_init_ao_cu_mask(struct radeon_device *rdev)
4921 {
4922         u32 i, j, k, active_cu_number = 0;
4923         u32 mask, counter, cu_bitmap;
4924         u32 tmp = 0;
4925
4926         for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4927                 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4928                         mask = 1;
4929                         cu_bitmap = 0;
4930                         counter  = 0;
4931                         for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4932                                 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
4933                                         if (counter < 2)
4934                                                 cu_bitmap |= mask;
4935                                         counter++;
4936                                 }
4937                                 mask <<= 1;
4938                         }
4939
4940                         active_cu_number += counter;
4941                         tmp |= (cu_bitmap << (i * 16 + j * 8));
4942                 }
4943         }
4944
4945         WREG32(RLC_PG_AO_CU_MASK, tmp);
4946
4947         tmp = RREG32(RLC_MAX_PG_CU);
4948         tmp &= ~MAX_PU_CU_MASK;
4949         tmp |= MAX_PU_CU(active_cu_number);
4950         WREG32(RLC_MAX_PG_CU, tmp);
4951 }
4952
4953 static void si_enable_cgcg(struct radeon_device *rdev,
4954                            bool enable)
4955 {
4956         u32 data, orig, tmp;
4957
4958         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4959
4960         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
4961                 si_enable_gui_idle_interrupt(rdev, true);
4962
4963                 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4964
4965                 tmp = si_halt_rlc(rdev);
4966
4967                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4968                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4969                 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
4970
4971                 si_wait_for_rlc_serdes(rdev);
4972
4973                 si_update_rlc(rdev, tmp);
4974
4975                 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
4976
4977                 data |= CGCG_EN | CGLS_EN;
4978         } else {
4979                 si_enable_gui_idle_interrupt(rdev, false);
4980
4981                 RREG32(CB_CGTT_SCLK_CTRL);
4982                 RREG32(CB_CGTT_SCLK_CTRL);
4983                 RREG32(CB_CGTT_SCLK_CTRL);
4984                 RREG32(CB_CGTT_SCLK_CTRL);
4985
4986                 data &= ~(CGCG_EN | CGLS_EN);
4987         }
4988
4989         if (orig != data)
4990                 WREG32(RLC_CGCG_CGLS_CTRL, data);
4991 }
4992
4993 static void si_enable_mgcg(struct radeon_device *rdev,
4994                            bool enable)
4995 {
4996         u32 data, orig, tmp = 0;
4997
4998         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
4999                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5000                 data = 0x96940200;
5001                 if (orig != data)
5002                         WREG32(CGTS_SM_CTRL_REG, data);
5003
5004                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5005                         orig = data = RREG32(CP_MEM_SLP_CNTL);
5006                         data |= CP_MEM_LS_EN;
5007                         if (orig != data)
5008                                 WREG32(CP_MEM_SLP_CNTL, data);
5009                 }
5010
5011                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5012                 data &= 0xffffffc0;
5013                 if (orig != data)
5014                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5015
5016                 tmp = si_halt_rlc(rdev);
5017
5018                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5019                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5020                 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5021
5022                 si_update_rlc(rdev, tmp);
5023         } else {
5024                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5025                 data |= 0x00000003;
5026                 if (orig != data)
5027                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5028
5029                 data = RREG32(CP_MEM_SLP_CNTL);
5030                 if (data & CP_MEM_LS_EN) {
5031                         data &= ~CP_MEM_LS_EN;
5032                         WREG32(CP_MEM_SLP_CNTL, data);
5033                 }
5034                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5035                 data |= LS_OVERRIDE | OVERRIDE;
5036                 if (orig != data)
5037                         WREG32(CGTS_SM_CTRL_REG, data);
5038
5039                 tmp = si_halt_rlc(rdev);
5040
5041                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5042                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5043                 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5044
5045                 si_update_rlc(rdev, tmp);
5046         }
5047 }
5048
5049 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5050                                bool enable)
5051 {
5052         u32 orig, data, tmp;
5053
5054         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5055                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5056                 tmp |= 0x3fff;
5057                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5058
5059                 orig = data = RREG32(UVD_CGC_CTRL);
5060                 data |= DCM;
5061                 if (orig != data)
5062                         WREG32(UVD_CGC_CTRL, data);
5063
5064                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5065                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5066         } else {
5067                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5068                 tmp &= ~0x3fff;
5069                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5070
5071                 orig = data = RREG32(UVD_CGC_CTRL);
5072                 data &= ~DCM;
5073                 if (orig != data)
5074                         WREG32(UVD_CGC_CTRL, data);
5075
5076                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5077                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5078         }
5079 }
5080
5081 static const u32 mc_cg_registers[] =
5082 {
5083         MC_HUB_MISC_HUB_CG,
5084         MC_HUB_MISC_SIP_CG,
5085         MC_HUB_MISC_VM_CG,
5086         MC_XPB_CLK_GAT,
5087         ATC_MISC_CG,
5088         MC_CITF_MISC_WR_CG,
5089         MC_CITF_MISC_RD_CG,
5090         MC_CITF_MISC_VM_CG,
5091         VM_L2_CG,
5092 };
5093
5094 static void si_enable_mc_ls(struct radeon_device *rdev,
5095                             bool enable)
5096 {
5097         int i;
5098         u32 orig, data;
5099
5100         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5101                 orig = data = RREG32(mc_cg_registers[i]);
5102                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5103                         data |= MC_LS_ENABLE;
5104                 else
5105                         data &= ~MC_LS_ENABLE;
5106                 if (data != orig)
5107                         WREG32(mc_cg_registers[i], data);
5108         }
5109 }
5110
5111 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5112                                bool enable)
5113 {
5114         int i;
5115         u32 orig, data;
5116
5117         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5118                 orig = data = RREG32(mc_cg_registers[i]);
5119                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5120                         data |= MC_CG_ENABLE;
5121                 else
5122                         data &= ~MC_CG_ENABLE;
5123                 if (data != orig)
5124                         WREG32(mc_cg_registers[i], data);
5125         }
5126 }
5127
5128 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5129                                bool enable)
5130 {
5131         u32 orig, data, offset;
5132         int i;
5133
5134         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5135                 for (i = 0; i < 2; i++) {
5136                         if (i == 0)
5137                                 offset = DMA0_REGISTER_OFFSET;
5138                         else
5139                                 offset = DMA1_REGISTER_OFFSET;
5140                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5141                         data &= ~MEM_POWER_OVERRIDE;
5142                         if (data != orig)
5143                                 WREG32(DMA_POWER_CNTL + offset, data);
5144                         WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5145                 }
5146         } else {
5147                 for (i = 0; i < 2; i++) {
5148                         if (i == 0)
5149                                 offset = DMA0_REGISTER_OFFSET;
5150                         else
5151                                 offset = DMA1_REGISTER_OFFSET;
5152                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5153                         data |= MEM_POWER_OVERRIDE;
5154                         if (data != orig)
5155                                 WREG32(DMA_POWER_CNTL + offset, data);
5156
5157                         orig = data = RREG32(DMA_CLK_CTRL + offset);
5158                         data = 0xff000000;
5159                         if (data != orig)
5160                                 WREG32(DMA_CLK_CTRL + offset, data);
5161                 }
5162         }
5163 }
5164
5165 static void si_enable_bif_mgls(struct radeon_device *rdev,
5166                                bool enable)
5167 {
5168         u32 orig, data;
5169
5170         orig = data = RREG32_PCIE(PCIE_CNTL2);
5171
5172         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5173                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5174                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5175         else
5176                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5177                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5178
5179         if (orig != data)
5180                 WREG32_PCIE(PCIE_CNTL2, data);
5181 }
5182
5183 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5184                                bool enable)
5185 {
5186         u32 orig, data;
5187
5188         orig = data = RREG32(HDP_HOST_PATH_CNTL);
5189
5190         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5191                 data &= ~CLOCK_GATING_DIS;
5192         else
5193                 data |= CLOCK_GATING_DIS;
5194
5195         if (orig != data)
5196                 WREG32(HDP_HOST_PATH_CNTL, data);
5197 }
5198
5199 static void si_enable_hdp_ls(struct radeon_device *rdev,
5200                              bool enable)
5201 {
5202         u32 orig, data;
5203
5204         orig = data = RREG32(HDP_MEM_POWER_LS);
5205
5206         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5207                 data |= HDP_LS_ENABLE;
5208         else
5209                 data &= ~HDP_LS_ENABLE;
5210
5211         if (orig != data)
5212                 WREG32(HDP_MEM_POWER_LS, data);
5213 }
5214
5215 void si_update_cg(struct radeon_device *rdev,
5216                   u32 block, bool enable)
5217 {
5218         if (block & RADEON_CG_BLOCK_GFX) {
5219                 si_enable_gui_idle_interrupt(rdev, false);
5220                 /* order matters! */
5221                 if (enable) {
5222                         si_enable_mgcg(rdev, true);
5223                         si_enable_cgcg(rdev, true);
5224                 } else {
5225                         si_enable_cgcg(rdev, false);
5226                         si_enable_mgcg(rdev, false);
5227                 }
5228                 si_enable_gui_idle_interrupt(rdev, true);
5229         }
5230
5231         if (block & RADEON_CG_BLOCK_MC) {
5232                 si_enable_mc_mgcg(rdev, enable);
5233                 si_enable_mc_ls(rdev, enable);
5234         }
5235
5236         if (block & RADEON_CG_BLOCK_SDMA) {
5237                 si_enable_dma_mgcg(rdev, enable);
5238         }
5239
5240         if (block & RADEON_CG_BLOCK_BIF) {
5241                 si_enable_bif_mgls(rdev, enable);
5242         }
5243
5244         if (block & RADEON_CG_BLOCK_UVD) {
5245                 if (rdev->has_uvd) {
5246                         si_enable_uvd_mgcg(rdev, enable);
5247                 }
5248         }
5249
5250         if (block & RADEON_CG_BLOCK_HDP) {
5251                 si_enable_hdp_mgcg(rdev, enable);
5252                 si_enable_hdp_ls(rdev, enable);
5253         }
5254 }
5255
5256 static void si_init_cg(struct radeon_device *rdev)
5257 {
5258         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5259                             RADEON_CG_BLOCK_MC |
5260                             RADEON_CG_BLOCK_SDMA |
5261                             RADEON_CG_BLOCK_BIF |
5262                             RADEON_CG_BLOCK_HDP), true);
5263         if (rdev->has_uvd) {
5264                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5265                 si_init_uvd_internal_cg(rdev);
5266         }
5267 }
5268
5269 static void si_fini_cg(struct radeon_device *rdev)
5270 {
5271         if (rdev->has_uvd) {
5272                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5273         }
5274         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5275                             RADEON_CG_BLOCK_MC |
5276                             RADEON_CG_BLOCK_SDMA |
5277                             RADEON_CG_BLOCK_BIF |
5278                             RADEON_CG_BLOCK_HDP), false);
5279 }
5280
5281 u32 si_get_csb_size(struct radeon_device *rdev)
5282 {
5283         u32 count = 0;
5284         const struct cs_section_def *sect = NULL;
5285         const struct cs_extent_def *ext = NULL;
5286
5287         if (rdev->rlc.cs_data == NULL)
5288                 return 0;
5289
5290         /* begin clear state */
5291         count += 2;
5292         /* context control state */
5293         count += 3;
5294
5295         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5296                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5297                         if (sect->id == SECT_CONTEXT)
5298                                 count += 2 + ext->reg_count;
5299                         else
5300                                 return 0;
5301                 }
5302         }
5303         /* pa_sc_raster_config */
5304         count += 3;
5305         /* end clear state */
5306         count += 2;
5307         /* clear state */
5308         count += 2;
5309
5310         return count;
5311 }
5312
5313 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5314 {
5315         u32 count = 0, i;
5316         const struct cs_section_def *sect = NULL;
5317         const struct cs_extent_def *ext = NULL;
5318
5319         if (rdev->rlc.cs_data == NULL)
5320                 return;
5321         if (buffer == NULL)
5322                 return;
5323
5324         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5325         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5326
5327         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5328         buffer[count++] = cpu_to_le32(0x80000000);
5329         buffer[count++] = cpu_to_le32(0x80000000);
5330
5331         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5332                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5333                         if (sect->id == SECT_CONTEXT) {
5334                                 buffer[count++] =
5335                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5336                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5337                                 for (i = 0; i < ext->reg_count; i++)
5338                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
5339                         } else {
5340                                 return;
5341                         }
5342                 }
5343         }
5344
5345         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5346         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5347         switch (rdev->family) {
5348         case CHIP_TAHITI:
5349         case CHIP_PITCAIRN:
5350                 buffer[count++] = cpu_to_le32(0x2a00126a);
5351                 break;
5352         case CHIP_VERDE:
5353                 buffer[count++] = cpu_to_le32(0x0000124a);
5354                 break;
5355         case CHIP_OLAND:
5356                 buffer[count++] = cpu_to_le32(0x00000082);
5357                 break;
5358         case CHIP_HAINAN:
5359                 buffer[count++] = cpu_to_le32(0x00000000);
5360                 break;
5361         default:
5362                 buffer[count++] = cpu_to_le32(0x00000000);
5363                 break;
5364         }
5365
5366         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5367         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5368
5369         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5370         buffer[count++] = cpu_to_le32(0);
5371 }
5372
5373 static void si_init_pg(struct radeon_device *rdev)
5374 {
5375         if (rdev->pg_flags) {
5376                 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5377                         si_init_dma_pg(rdev);
5378                 }
5379                 si_init_ao_cu_mask(rdev);
5380                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5381                         si_init_gfx_cgpg(rdev);
5382                 }
5383                 si_enable_dma_pg(rdev, true);
5384                 si_enable_gfx_cgpg(rdev, true);
5385         } else {
5386                 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5387                 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5388         }
5389 }
5390
5391 static void si_fini_pg(struct radeon_device *rdev)
5392 {
5393         if (rdev->pg_flags) {
5394                 si_enable_dma_pg(rdev, false);
5395                 si_enable_gfx_cgpg(rdev, false);
5396         }
5397 }
5398
5399 /*
5400  * RLC
5401  */
5402 void si_rlc_reset(struct radeon_device *rdev)
5403 {
5404         u32 tmp = RREG32(GRBM_SOFT_RESET);
5405
5406         tmp |= SOFT_RESET_RLC;
5407         WREG32(GRBM_SOFT_RESET, tmp);
5408         udelay(50);
5409         tmp &= ~SOFT_RESET_RLC;
5410         WREG32(GRBM_SOFT_RESET, tmp);
5411         udelay(50);
5412 }
5413
5414 static void si_rlc_stop(struct radeon_device *rdev)
5415 {
5416         WREG32(RLC_CNTL, 0);
5417
5418         si_enable_gui_idle_interrupt(rdev, false);
5419
5420         si_wait_for_rlc_serdes(rdev);
5421 }
5422
5423 static void si_rlc_start(struct radeon_device *rdev)
5424 {
5425         WREG32(RLC_CNTL, RLC_ENABLE);
5426
5427         si_enable_gui_idle_interrupt(rdev, true);
5428
5429         udelay(50);
5430 }
5431
5432 static bool si_lbpw_supported(struct radeon_device *rdev)
5433 {
5434         u32 tmp;
5435
5436         /* Enable LBPW only for DDR3 */
5437         tmp = RREG32(MC_SEQ_MISC0);
5438         if ((tmp & 0xF0000000) == 0xB0000000)
5439                 return true;
5440         return false;
5441 }
5442
5443 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5444 {
5445         u32 tmp;
5446
5447         tmp = RREG32(RLC_LB_CNTL);
5448         if (enable)
5449                 tmp |= LOAD_BALANCE_ENABLE;
5450         else
5451                 tmp &= ~LOAD_BALANCE_ENABLE;
5452         WREG32(RLC_LB_CNTL, tmp);
5453
5454         if (!enable) {
5455                 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5456                 WREG32(SPI_LB_CU_MASK, 0x00ff);
5457         }
5458 }
5459
5460 static int si_rlc_resume(struct radeon_device *rdev)
5461 {
5462         u32 i;
5463         const __be32 *fw_data;
5464
5465         if (!rdev->rlc_fw)
5466                 return -EINVAL;
5467
5468         si_rlc_stop(rdev);
5469
5470         si_rlc_reset(rdev);
5471
5472         si_init_pg(rdev);
5473
5474         si_init_cg(rdev);
5475
5476         WREG32(RLC_RL_BASE, 0);
5477         WREG32(RLC_RL_SIZE, 0);
5478         WREG32(RLC_LB_CNTL, 0);
5479         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5480         WREG32(RLC_LB_CNTR_INIT, 0);
5481         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5482
5483         WREG32(RLC_MC_CNTL, 0);
5484         WREG32(RLC_UCODE_CNTL, 0);
5485
5486         fw_data = (const __be32 *)rdev->rlc_fw->data;
5487         for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5488                 WREG32(RLC_UCODE_ADDR, i);
5489                 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5490         }
5491         WREG32(RLC_UCODE_ADDR, 0);
5492
5493         si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5494
5495         si_rlc_start(rdev);
5496
5497         return 0;
5498 }
5499
5500 static void si_enable_interrupts(struct radeon_device *rdev)
5501 {
5502         u32 ih_cntl = RREG32(IH_CNTL);
5503         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5504
5505         ih_cntl |= ENABLE_INTR;
5506         ih_rb_cntl |= IH_RB_ENABLE;
5507         WREG32(IH_CNTL, ih_cntl);
5508         WREG32(IH_RB_CNTL, ih_rb_cntl);
5509         rdev->ih.enabled = true;
5510 }
5511
5512 static void si_disable_interrupts(struct radeon_device *rdev)
5513 {
5514         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5515         u32 ih_cntl = RREG32(IH_CNTL);
5516
5517         ih_rb_cntl &= ~IH_RB_ENABLE;
5518         ih_cntl &= ~ENABLE_INTR;
5519         WREG32(IH_RB_CNTL, ih_rb_cntl);
5520         WREG32(IH_CNTL, ih_cntl);
5521         /* set rptr, wptr to 0 */
5522         WREG32(IH_RB_RPTR, 0);
5523         WREG32(IH_RB_WPTR, 0);
5524         rdev->ih.enabled = false;
5525         rdev->ih.rptr = 0;
5526 }
5527
5528 static void si_disable_interrupt_state(struct radeon_device *rdev)
5529 {
5530         u32 tmp;
5531
5532         tmp = RREG32(CP_INT_CNTL_RING0) &
5533                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5534         WREG32(CP_INT_CNTL_RING0, tmp);
5535         WREG32(CP_INT_CNTL_RING1, 0);
5536         WREG32(CP_INT_CNTL_RING2, 0);
5537         tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5538         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5539         tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5540         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5541         WREG32(GRBM_INT_CNTL, 0);
5542         if (rdev->num_crtc >= 2) {
5543                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5544                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5545         }
5546         if (rdev->num_crtc >= 4) {
5547                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5548                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5549         }
5550         if (rdev->num_crtc >= 6) {
5551                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5552                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5553         }
5554
5555         if (rdev->num_crtc >= 2) {
5556                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5557                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5558         }
5559         if (rdev->num_crtc >= 4) {
5560                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5561                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5562         }
5563         if (rdev->num_crtc >= 6) {
5564                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5565                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5566         }
5567
5568         if (!ASIC_IS_NODCE(rdev)) {
5569                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5570
5571                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5572                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5573                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5574                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5575                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5576                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5577                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5578                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5579                 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5580                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5581                 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5582                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5583         }
5584 }
5585
5586 static int si_irq_init(struct radeon_device *rdev)
5587 {
5588         int ret = 0;
5589         int rb_bufsz;
5590         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5591
5592         /* allocate ring */
5593         ret = r600_ih_ring_alloc(rdev);
5594         if (ret)
5595                 return ret;
5596
5597         /* disable irqs */
5598         si_disable_interrupts(rdev);
5599
5600         /* init rlc */
5601         ret = si_rlc_resume(rdev);
5602         if (ret) {
5603                 r600_ih_ring_fini(rdev);
5604                 return ret;
5605         }
5606
5607         /* setup interrupt control */
5608         /* set dummy read address to ring address */
5609         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5610         interrupt_cntl = RREG32(INTERRUPT_CNTL);
5611         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5612          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5613          */
5614         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5615         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5616         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5617         WREG32(INTERRUPT_CNTL, interrupt_cntl);
5618
5619         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5620         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5621
5622         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5623                       IH_WPTR_OVERFLOW_CLEAR |
5624                       (rb_bufsz << 1));
5625
5626         if (rdev->wb.enabled)
5627                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5628
5629         /* set the writeback address whether it's enabled or not */
5630         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5631         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5632
5633         WREG32(IH_RB_CNTL, ih_rb_cntl);
5634
5635         /* set rptr, wptr to 0 */
5636         WREG32(IH_RB_RPTR, 0);
5637         WREG32(IH_RB_WPTR, 0);
5638
5639         /* Default settings for IH_CNTL (disabled at first) */
5640         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5641         /* RPTR_REARM only works if msi's are enabled */
5642         if (rdev->msi_enabled)
5643                 ih_cntl |= RPTR_REARM;
5644         WREG32(IH_CNTL, ih_cntl);
5645
5646         /* force the active interrupt state to all disabled */
5647         si_disable_interrupt_state(rdev);
5648
5649         pci_set_master(rdev->pdev);
5650
5651         /* enable irqs */
5652         si_enable_interrupts(rdev);
5653
5654         return ret;
5655 }
5656
5657 int si_irq_set(struct radeon_device *rdev)
5658 {
5659         u32 cp_int_cntl;
5660         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5661         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5662         u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5663         u32 grbm_int_cntl = 0;
5664         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5665         u32 dma_cntl, dma_cntl1;
5666         u32 thermal_int = 0;
5667
5668         if (!rdev->irq.installed) {
5669                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5670                 return -EINVAL;
5671         }
5672         /* don't enable anything if the ih is disabled */
5673         if (!rdev->ih.enabled) {
5674                 si_disable_interrupts(rdev);
5675                 /* force the active interrupt state to all disabled */
5676                 si_disable_interrupt_state(rdev);
5677                 return 0;
5678         }
5679
5680         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5681                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5682
5683         if (!ASIC_IS_NODCE(rdev)) {
5684                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5685                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5686                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5687                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5688                 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5689                 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5690         }
5691
5692         dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5693         dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5694
5695         thermal_int = RREG32(CG_THERMAL_INT) &
5696                 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5697
5698         /* enable CP interrupts on all rings */
5699         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5700                 DRM_DEBUG("si_irq_set: sw int gfx\n");
5701                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5702         }
5703         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5704                 DRM_DEBUG("si_irq_set: sw int cp1\n");
5705                 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5706         }
5707         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5708                 DRM_DEBUG("si_irq_set: sw int cp2\n");
5709                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5710         }
5711         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5712                 DRM_DEBUG("si_irq_set: sw int dma\n");
5713                 dma_cntl |= TRAP_ENABLE;
5714         }
5715
5716         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5717                 DRM_DEBUG("si_irq_set: sw int dma1\n");
5718                 dma_cntl1 |= TRAP_ENABLE;
5719         }
5720         if (rdev->irq.crtc_vblank_int[0] ||
5721             atomic_read(&rdev->irq.pflip[0])) {
5722                 DRM_DEBUG("si_irq_set: vblank 0\n");
5723                 crtc1 |= VBLANK_INT_MASK;
5724         }
5725         if (rdev->irq.crtc_vblank_int[1] ||
5726             atomic_read(&rdev->irq.pflip[1])) {
5727                 DRM_DEBUG("si_irq_set: vblank 1\n");
5728                 crtc2 |= VBLANK_INT_MASK;
5729         }
5730         if (rdev->irq.crtc_vblank_int[2] ||
5731             atomic_read(&rdev->irq.pflip[2])) {
5732                 DRM_DEBUG("si_irq_set: vblank 2\n");
5733                 crtc3 |= VBLANK_INT_MASK;
5734         }
5735         if (rdev->irq.crtc_vblank_int[3] ||
5736             atomic_read(&rdev->irq.pflip[3])) {
5737                 DRM_DEBUG("si_irq_set: vblank 3\n");
5738                 crtc4 |= VBLANK_INT_MASK;
5739         }
5740         if (rdev->irq.crtc_vblank_int[4] ||
5741             atomic_read(&rdev->irq.pflip[4])) {
5742                 DRM_DEBUG("si_irq_set: vblank 4\n");
5743                 crtc5 |= VBLANK_INT_MASK;
5744         }
5745         if (rdev->irq.crtc_vblank_int[5] ||
5746             atomic_read(&rdev->irq.pflip[5])) {
5747                 DRM_DEBUG("si_irq_set: vblank 5\n");
5748                 crtc6 |= VBLANK_INT_MASK;
5749         }
5750         if (rdev->irq.hpd[0]) {
5751                 DRM_DEBUG("si_irq_set: hpd 1\n");
5752                 hpd1 |= DC_HPDx_INT_EN;
5753         }
5754         if (rdev->irq.hpd[1]) {
5755                 DRM_DEBUG("si_irq_set: hpd 2\n");
5756                 hpd2 |= DC_HPDx_INT_EN;
5757         }
5758         if (rdev->irq.hpd[2]) {
5759                 DRM_DEBUG("si_irq_set: hpd 3\n");
5760                 hpd3 |= DC_HPDx_INT_EN;
5761         }
5762         if (rdev->irq.hpd[3]) {
5763                 DRM_DEBUG("si_irq_set: hpd 4\n");
5764                 hpd4 |= DC_HPDx_INT_EN;
5765         }
5766         if (rdev->irq.hpd[4]) {
5767                 DRM_DEBUG("si_irq_set: hpd 5\n");
5768                 hpd5 |= DC_HPDx_INT_EN;
5769         }
5770         if (rdev->irq.hpd[5]) {
5771                 DRM_DEBUG("si_irq_set: hpd 6\n");
5772                 hpd6 |= DC_HPDx_INT_EN;
5773         }
5774
5775         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5776         WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5777         WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5778
5779         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5780         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5781
5782         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5783
5784         if (rdev->irq.dpm_thermal) {
5785                 DRM_DEBUG("dpm thermal\n");
5786                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5787         }
5788
5789         if (rdev->num_crtc >= 2) {
5790                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5791                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5792         }
5793         if (rdev->num_crtc >= 4) {
5794                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5795                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5796         }
5797         if (rdev->num_crtc >= 6) {
5798                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5799                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5800         }
5801
5802         if (rdev->num_crtc >= 2) {
5803                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5804                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5805         }
5806         if (rdev->num_crtc >= 4) {
5807                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5808                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5809         }
5810         if (rdev->num_crtc >= 6) {
5811                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5812                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5813         }
5814
5815         if (!ASIC_IS_NODCE(rdev)) {
5816                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5817                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5818                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5819                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5820                 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5821                 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5822         }
5823
5824         WREG32(CG_THERMAL_INT, thermal_int);
5825
5826         return 0;
5827 }
5828
5829 static inline void si_irq_ack(struct radeon_device *rdev)
5830 {
5831         u32 tmp;
5832
5833         if (ASIC_IS_NODCE(rdev))
5834                 return;
5835
5836         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5837         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5838         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5839         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5840         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5841         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5842         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5843         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5844         if (rdev->num_crtc >= 4) {
5845                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5846                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5847         }
5848         if (rdev->num_crtc >= 6) {
5849                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5850                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5851         }
5852
5853         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5854                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5855         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5856                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5857         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5858                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5859         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5860                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5861         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5862                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5863         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5864                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5865
5866         if (rdev->num_crtc >= 4) {
5867                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5868                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5869                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5870                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5871                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5872                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5873                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5874                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5875                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5876                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5877                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5878                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5879         }
5880
5881         if (rdev->num_crtc >= 6) {
5882                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5883                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5884                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5885                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5886                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5887                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5888                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5889                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5890                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5891                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5892                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5893                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5894         }
5895
5896         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5897                 tmp = RREG32(DC_HPD1_INT_CONTROL);
5898                 tmp |= DC_HPDx_INT_ACK;
5899                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5900         }
5901         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5902                 tmp = RREG32(DC_HPD2_INT_CONTROL);
5903                 tmp |= DC_HPDx_INT_ACK;
5904                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5905         }
5906         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5907                 tmp = RREG32(DC_HPD3_INT_CONTROL);
5908                 tmp |= DC_HPDx_INT_ACK;
5909                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5910         }
5911         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5912                 tmp = RREG32(DC_HPD4_INT_CONTROL);
5913                 tmp |= DC_HPDx_INT_ACK;
5914                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5915         }
5916         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5917                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5918                 tmp |= DC_HPDx_INT_ACK;
5919                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5920         }
5921         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5922                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5923                 tmp |= DC_HPDx_INT_ACK;
5924                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5925         }
5926 }
5927
5928 static void si_irq_disable(struct radeon_device *rdev)
5929 {
5930         si_disable_interrupts(rdev);
5931         /* Wait and acknowledge irq */
5932         mdelay(1);
5933         si_irq_ack(rdev);
5934         si_disable_interrupt_state(rdev);
5935 }
5936
5937 static void si_irq_suspend(struct radeon_device *rdev)
5938 {
5939         si_irq_disable(rdev);
5940         si_rlc_stop(rdev);
5941 }
5942
5943 static void si_irq_fini(struct radeon_device *rdev)
5944 {
5945         si_irq_suspend(rdev);
5946         r600_ih_ring_fini(rdev);
5947 }
5948
5949 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5950 {
5951         u32 wptr, tmp;
5952
5953         if (rdev->wb.enabled)
5954                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5955         else
5956                 wptr = RREG32(IH_RB_WPTR);
5957
5958         if (wptr & RB_OVERFLOW) {
5959                 /* When a ring buffer overflow happen start parsing interrupt
5960                  * from the last not overwritten vector (wptr + 16). Hopefully
5961                  * this should allow us to catchup.
5962                  */
5963                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5964                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5965                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5966                 tmp = RREG32(IH_RB_CNTL);
5967                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5968                 WREG32(IH_RB_CNTL, tmp);
5969         }
5970         return (wptr & rdev->ih.ptr_mask);
5971 }
5972
5973 /*        SI IV Ring
5974  * Each IV ring entry is 128 bits:
5975  * [7:0]    - interrupt source id
5976  * [31:8]   - reserved
5977  * [59:32]  - interrupt source data
5978  * [63:60]  - reserved
5979  * [71:64]  - RINGID
5980  * [79:72]  - VMID
5981  * [127:80] - reserved
5982  */
5983 int si_irq_process(struct radeon_device *rdev)
5984 {
5985         u32 wptr;
5986         u32 rptr;
5987         u32 src_id, src_data, ring_id;
5988         u32 ring_index;
5989         bool queue_hotplug = false;
5990         bool queue_thermal = false;
5991         u32 status, addr;
5992
5993         if (!rdev->ih.enabled || rdev->shutdown)
5994                 return IRQ_NONE;
5995
5996         wptr = si_get_ih_wptr(rdev);
5997
5998 restart_ih:
5999         /* is somebody else already processing irqs? */
6000         if (atomic_xchg(&rdev->ih.lock, 1))
6001                 return IRQ_NONE;
6002
6003         rptr = rdev->ih.rptr;
6004         DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6005
6006         /* Order reading of wptr vs. reading of IH ring data */
6007         rmb();
6008
6009         /* display interrupts */
6010         si_irq_ack(rdev);
6011
6012         while (rptr != wptr) {
6013                 /* wptr/rptr are in bytes! */
6014                 ring_index = rptr / 4;
6015                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6016                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6017                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6018
6019                 switch (src_id) {
6020                 case 1: /* D1 vblank/vline */
6021                         switch (src_data) {
6022                         case 0: /* D1 vblank */
6023                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6024                                         if (rdev->irq.crtc_vblank_int[0]) {
6025                                                 drm_handle_vblank(rdev->ddev, 0);
6026                                                 rdev->pm.vblank_sync = true;
6027                                                 wake_up(&rdev->irq.vblank_queue);
6028                                         }
6029                                         if (atomic_read(&rdev->irq.pflip[0]))
6030                                                 radeon_crtc_handle_flip(rdev, 0);
6031                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6032                                         DRM_DEBUG("IH: D1 vblank\n");
6033                                 }
6034                                 break;
6035                         case 1: /* D1 vline */
6036                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6037                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6038                                         DRM_DEBUG("IH: D1 vline\n");
6039                                 }
6040                                 break;
6041                         default:
6042                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6043                                 break;
6044                         }
6045                         break;
6046                 case 2: /* D2 vblank/vline */
6047                         switch (src_data) {
6048                         case 0: /* D2 vblank */
6049                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6050                                         if (rdev->irq.crtc_vblank_int[1]) {
6051                                                 drm_handle_vblank(rdev->ddev, 1);
6052                                                 rdev->pm.vblank_sync = true;
6053                                                 wake_up(&rdev->irq.vblank_queue);
6054                                         }
6055                                         if (atomic_read(&rdev->irq.pflip[1]))
6056                                                 radeon_crtc_handle_flip(rdev, 1);
6057                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6058                                         DRM_DEBUG("IH: D2 vblank\n");
6059                                 }
6060                                 break;
6061                         case 1: /* D2 vline */
6062                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6063                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6064                                         DRM_DEBUG("IH: D2 vline\n");
6065                                 }
6066                                 break;
6067                         default:
6068                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6069                                 break;
6070                         }
6071                         break;
6072                 case 3: /* D3 vblank/vline */
6073                         switch (src_data) {
6074                         case 0: /* D3 vblank */
6075                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6076                                         if (rdev->irq.crtc_vblank_int[2]) {
6077                                                 drm_handle_vblank(rdev->ddev, 2);
6078                                                 rdev->pm.vblank_sync = true;
6079                                                 wake_up(&rdev->irq.vblank_queue);
6080                                         }
6081                                         if (atomic_read(&rdev->irq.pflip[2]))
6082                                                 radeon_crtc_handle_flip(rdev, 2);
6083                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6084                                         DRM_DEBUG("IH: D3 vblank\n");
6085                                 }
6086                                 break;
6087                         case 1: /* D3 vline */
6088                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6089                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6090                                         DRM_DEBUG("IH: D3 vline\n");
6091                                 }
6092                                 break;
6093                         default:
6094                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6095                                 break;
6096                         }
6097                         break;
6098                 case 4: /* D4 vblank/vline */
6099                         switch (src_data) {
6100                         case 0: /* D4 vblank */
6101                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6102                                         if (rdev->irq.crtc_vblank_int[3]) {
6103                                                 drm_handle_vblank(rdev->ddev, 3);
6104                                                 rdev->pm.vblank_sync = true;
6105                                                 wake_up(&rdev->irq.vblank_queue);
6106                                         }
6107                                         if (atomic_read(&rdev->irq.pflip[3]))
6108                                                 radeon_crtc_handle_flip(rdev, 3);
6109                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6110                                         DRM_DEBUG("IH: D4 vblank\n");
6111                                 }
6112                                 break;
6113                         case 1: /* D4 vline */
6114                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6115                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6116                                         DRM_DEBUG("IH: D4 vline\n");
6117                                 }
6118                                 break;
6119                         default:
6120                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6121                                 break;
6122                         }
6123                         break;
6124                 case 5: /* D5 vblank/vline */
6125                         switch (src_data) {
6126                         case 0: /* D5 vblank */
6127                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6128                                         if (rdev->irq.crtc_vblank_int[4]) {
6129                                                 drm_handle_vblank(rdev->ddev, 4);
6130                                                 rdev->pm.vblank_sync = true;
6131                                                 wake_up(&rdev->irq.vblank_queue);
6132                                         }
6133                                         if (atomic_read(&rdev->irq.pflip[4]))
6134                                                 radeon_crtc_handle_flip(rdev, 4);
6135                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6136                                         DRM_DEBUG("IH: D5 vblank\n");
6137                                 }
6138                                 break;
6139                         case 1: /* D5 vline */
6140                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6141                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6142                                         DRM_DEBUG("IH: D5 vline\n");
6143                                 }
6144                                 break;
6145                         default:
6146                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6147                                 break;
6148                         }
6149                         break;
6150                 case 6: /* D6 vblank/vline */
6151                         switch (src_data) {
6152                         case 0: /* D6 vblank */
6153                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6154                                         if (rdev->irq.crtc_vblank_int[5]) {
6155                                                 drm_handle_vblank(rdev->ddev, 5);
6156                                                 rdev->pm.vblank_sync = true;
6157                                                 wake_up(&rdev->irq.vblank_queue);
6158                                         }
6159                                         if (atomic_read(&rdev->irq.pflip[5]))
6160                                                 radeon_crtc_handle_flip(rdev, 5);
6161                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6162                                         DRM_DEBUG("IH: D6 vblank\n");
6163                                 }
6164                                 break;
6165                         case 1: /* D6 vline */
6166                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6167                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6168                                         DRM_DEBUG("IH: D6 vline\n");
6169                                 }
6170                                 break;
6171                         default:
6172                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6173                                 break;
6174                         }
6175                         break;
6176                 case 42: /* HPD hotplug */
6177                         switch (src_data) {
6178                         case 0:
6179                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6180                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6181                                         queue_hotplug = true;
6182                                         DRM_DEBUG("IH: HPD1\n");
6183                                 }
6184                                 break;
6185                         case 1:
6186                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6187                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6188                                         queue_hotplug = true;
6189                                         DRM_DEBUG("IH: HPD2\n");
6190                                 }
6191                                 break;
6192                         case 2:
6193                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6194                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6195                                         queue_hotplug = true;
6196                                         DRM_DEBUG("IH: HPD3\n");
6197                                 }
6198                                 break;
6199                         case 3:
6200                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6201                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6202                                         queue_hotplug = true;
6203                                         DRM_DEBUG("IH: HPD4\n");
6204                                 }
6205                                 break;
6206                         case 4:
6207                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6208                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6209                                         queue_hotplug = true;
6210                                         DRM_DEBUG("IH: HPD5\n");
6211                                 }
6212                                 break;
6213                         case 5:
6214                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6215                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6216                                         queue_hotplug = true;
6217                                         DRM_DEBUG("IH: HPD6\n");
6218                                 }
6219                                 break;
6220                         default:
6221                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6222                                 break;
6223                         }
6224                         break;
6225                 case 146:
6226                 case 147:
6227                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6228                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6229                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6230                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6231                                 addr);
6232                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6233                                 status);
6234                         si_vm_decode_fault(rdev, status, addr);
6235                         /* reset addr and status */
6236                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6237                         break;
6238                 case 176: /* RINGID0 CP_INT */
6239                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6240                         break;
6241                 case 177: /* RINGID1 CP_INT */
6242                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6243                         break;
6244                 case 178: /* RINGID2 CP_INT */
6245                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6246                         break;
6247                 case 181: /* CP EOP event */
6248                         DRM_DEBUG("IH: CP EOP\n");
6249                         switch (ring_id) {
6250                         case 0:
6251                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6252                                 break;
6253                         case 1:
6254                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6255                                 break;
6256                         case 2:
6257                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6258                                 break;
6259                         }
6260                         break;
6261                 case 224: /* DMA trap event */
6262                         DRM_DEBUG("IH: DMA trap\n");
6263                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6264                         break;
6265                 case 230: /* thermal low to high */
6266                         DRM_DEBUG("IH: thermal low to high\n");
6267                         rdev->pm.dpm.thermal.high_to_low = false;
6268                         queue_thermal = true;
6269                         break;
6270                 case 231: /* thermal high to low */
6271                         DRM_DEBUG("IH: thermal high to low\n");
6272                         rdev->pm.dpm.thermal.high_to_low = true;
6273                         queue_thermal = true;
6274                         break;
6275                 case 233: /* GUI IDLE */
6276                         DRM_DEBUG("IH: GUI idle\n");
6277                         break;
6278                 case 244: /* DMA trap event */
6279                         DRM_DEBUG("IH: DMA1 trap\n");
6280                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6281                         break;
6282                 default:
6283                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6284                         break;
6285                 }
6286
6287                 /* wptr/rptr are in bytes! */
6288                 rptr += 16;
6289                 rptr &= rdev->ih.ptr_mask;
6290         }
6291         if (queue_hotplug)
6292                 schedule_work(&rdev->hotplug_work);
6293         if (queue_thermal && rdev->pm.dpm_enabled)
6294                 schedule_work(&rdev->pm.dpm.thermal.work);
6295         rdev->ih.rptr = rptr;
6296         WREG32(IH_RB_RPTR, rdev->ih.rptr);
6297         atomic_set(&rdev->ih.lock, 0);
6298
6299         /* make sure wptr hasn't changed while processing */
6300         wptr = si_get_ih_wptr(rdev);
6301         if (wptr != rptr)
6302                 goto restart_ih;
6303
6304         return IRQ_HANDLED;
6305 }
6306
6307 /*
6308  * startup/shutdown callbacks
6309  */
6310 static int si_startup(struct radeon_device *rdev)
6311 {
6312         struct radeon_ring *ring;
6313         int r;
6314
6315         /* enable pcie gen2/3 link */
6316         si_pcie_gen3_enable(rdev);
6317         /* enable aspm */
6318         si_program_aspm(rdev);
6319
6320         /* scratch needs to be initialized before MC */
6321         r = r600_vram_scratch_init(rdev);
6322         if (r)
6323                 return r;
6324
6325         si_mc_program(rdev);
6326
6327         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6328             !rdev->rlc_fw || !rdev->mc_fw) {
6329                 r = si_init_microcode(rdev);
6330                 if (r) {
6331                         DRM_ERROR("Failed to load firmware!\n");
6332                         return r;
6333                 }
6334         }
6335
6336         r = si_mc_load_microcode(rdev);
6337         if (r) {
6338                 DRM_ERROR("Failed to load MC firmware!\n");
6339                 return r;
6340         }
6341
6342         r = si_pcie_gart_enable(rdev);
6343         if (r)
6344                 return r;
6345         si_gpu_init(rdev);
6346
6347         /* allocate rlc buffers */
6348         if (rdev->family == CHIP_VERDE) {
6349                 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6350                 rdev->rlc.reg_list_size =
6351                         (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6352         }
6353         rdev->rlc.cs_data = si_cs_data;
6354         r = sumo_rlc_init(rdev);
6355         if (r) {
6356                 DRM_ERROR("Failed to init rlc BOs!\n");
6357                 return r;
6358         }
6359
6360         /* allocate wb buffer */
6361         r = radeon_wb_init(rdev);
6362         if (r)
6363                 return r;
6364
6365         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6366         if (r) {
6367                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6368                 return r;
6369         }
6370
6371         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6372         if (r) {
6373                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6374                 return r;
6375         }
6376
6377         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6378         if (r) {
6379                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6380                 return r;
6381         }
6382
6383         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6384         if (r) {
6385                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6386                 return r;
6387         }
6388
6389         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6390         if (r) {
6391                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6392                 return r;
6393         }
6394
6395         if (rdev->has_uvd) {
6396                 r = uvd_v2_2_resume(rdev);
6397                 if (!r) {
6398                         r = radeon_fence_driver_start_ring(rdev,
6399                                                            R600_RING_TYPE_UVD_INDEX);
6400                         if (r)
6401                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6402                 }
6403                 if (r)
6404                         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6405         }
6406
6407         /* Enable IRQ */
6408         if (!rdev->irq.installed) {
6409                 r = radeon_irq_kms_init(rdev);
6410                 if (r)
6411                         return r;
6412         }
6413
6414         r = si_irq_init(rdev);
6415         if (r) {
6416                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6417                 radeon_irq_kms_fini(rdev);
6418                 return r;
6419         }
6420         si_irq_set(rdev);
6421
6422         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6423         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6424                              CP_RB0_RPTR, CP_RB0_WPTR,
6425                              RADEON_CP_PACKET2);
6426         if (r)
6427                 return r;
6428
6429         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6430         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6431                              CP_RB1_RPTR, CP_RB1_WPTR,
6432                              RADEON_CP_PACKET2);
6433         if (r)
6434                 return r;
6435
6436         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6437         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6438                              CP_RB2_RPTR, CP_RB2_WPTR,
6439                              RADEON_CP_PACKET2);
6440         if (r)
6441                 return r;
6442
6443         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6444         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6445                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6446                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6447                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6448         if (r)
6449                 return r;
6450
6451         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6452         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6453                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6454                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6455                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6456         if (r)
6457                 return r;
6458
6459         r = si_cp_load_microcode(rdev);
6460         if (r)
6461                 return r;
6462         r = si_cp_resume(rdev);
6463         if (r)
6464                 return r;
6465
6466         r = cayman_dma_resume(rdev);
6467         if (r)
6468                 return r;
6469
6470         if (rdev->has_uvd) {
6471                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6472                 if (ring->ring_size) {
6473                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6474                                              UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6475                                              RADEON_CP_PACKET2);
6476                         if (!r)
6477                                 r = uvd_v1_0_init(rdev);
6478                         if (r)
6479                                 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6480                 }
6481         }
6482
6483         r = radeon_ib_pool_init(rdev);
6484         if (r) {
6485                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6486                 return r;
6487         }
6488
6489         r = radeon_vm_manager_init(rdev);
6490         if (r) {
6491                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6492                 return r;
6493         }
6494
6495         r = dce6_audio_init(rdev);
6496         if (r)
6497                 return r;
6498
6499         return 0;
6500 }
6501
6502 int si_resume(struct radeon_device *rdev)
6503 {
6504         int r;
6505
6506         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6507          * posting will perform necessary task to bring back GPU into good
6508          * shape.
6509          */
6510         /* post card */
6511         atom_asic_init(rdev->mode_info.atom_context);
6512
6513         /* init golden registers */
6514         si_init_golden_registers(rdev);
6515
6516         rdev->accel_working = true;
6517         r = si_startup(rdev);
6518         if (r) {
6519                 DRM_ERROR("si startup failed on resume\n");
6520                 rdev->accel_working = false;
6521                 return r;
6522         }
6523
6524         return r;
6525
6526 }
6527
6528 int si_suspend(struct radeon_device *rdev)
6529 {
6530         dce6_audio_fini(rdev);
6531         radeon_vm_manager_fini(rdev);
6532         si_cp_enable(rdev, false);
6533         cayman_dma_stop(rdev);
6534         if (rdev->has_uvd) {
6535                 uvd_v1_0_fini(rdev);
6536                 radeon_uvd_suspend(rdev);
6537         }
6538         si_fini_pg(rdev);
6539         si_fini_cg(rdev);
6540         si_irq_suspend(rdev);
6541         radeon_wb_disable(rdev);
6542         si_pcie_gart_disable(rdev);
6543         return 0;
6544 }
6545
6546 /* Plan is to move initialization in that function and use
6547  * helper function so that radeon_device_init pretty much
6548  * do nothing more than calling asic specific function. This
6549  * should also allow to remove a bunch of callback function
6550  * like vram_info.
6551  */
6552 int si_init(struct radeon_device *rdev)
6553 {
6554         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6555         int r;
6556
6557         /* Read BIOS */
6558         if (!radeon_get_bios(rdev)) {
6559                 if (ASIC_IS_AVIVO(rdev))
6560                         return -EINVAL;
6561         }
6562         /* Must be an ATOMBIOS */
6563         if (!rdev->is_atom_bios) {
6564                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6565                 return -EINVAL;
6566         }
6567         r = radeon_atombios_init(rdev);
6568         if (r)
6569                 return r;
6570
6571         /* Post card if necessary */
6572         if (!radeon_card_posted(rdev)) {
6573                 if (!rdev->bios) {
6574                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6575                         return -EINVAL;
6576                 }
6577                 DRM_INFO("GPU not posted. posting now...\n");
6578                 atom_asic_init(rdev->mode_info.atom_context);
6579         }
6580         /* init golden registers */
6581         si_init_golden_registers(rdev);
6582         /* Initialize scratch registers */
6583         si_scratch_init(rdev);
6584         /* Initialize surface registers */
6585         radeon_surface_init(rdev);
6586         /* Initialize clocks */
6587         radeon_get_clock_info(rdev->ddev);
6588
6589         /* Fence driver */
6590         r = radeon_fence_driver_init(rdev);
6591         if (r)
6592                 return r;
6593
6594         /* initialize memory controller */
6595         r = si_mc_init(rdev);
6596         if (r)
6597                 return r;
6598         /* Memory manager */
6599         r = radeon_bo_init(rdev);
6600         if (r)
6601                 return r;
6602
6603         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6604         ring->ring_obj = NULL;
6605         r600_ring_init(rdev, ring, 1024 * 1024);
6606
6607         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6608         ring->ring_obj = NULL;
6609         r600_ring_init(rdev, ring, 1024 * 1024);
6610
6611         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6612         ring->ring_obj = NULL;
6613         r600_ring_init(rdev, ring, 1024 * 1024);
6614
6615         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6616         ring->ring_obj = NULL;
6617         r600_ring_init(rdev, ring, 64 * 1024);
6618
6619         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6620         ring->ring_obj = NULL;
6621         r600_ring_init(rdev, ring, 64 * 1024);
6622
6623         if (rdev->has_uvd) {
6624                 r = radeon_uvd_init(rdev);
6625                 if (!r) {
6626                         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6627                         ring->ring_obj = NULL;
6628                         r600_ring_init(rdev, ring, 4096);
6629                 }
6630         }
6631
6632         rdev->ih.ring_obj = NULL;
6633         r600_ih_ring_init(rdev, 64 * 1024);
6634
6635         r = r600_pcie_gart_init(rdev);
6636         if (r)
6637                 return r;
6638
6639         rdev->accel_working = true;
6640         r = si_startup(rdev);
6641         if (r) {
6642                 dev_err(rdev->dev, "disabling GPU acceleration\n");
6643                 si_cp_fini(rdev);
6644                 cayman_dma_fini(rdev);
6645                 si_irq_fini(rdev);
6646                 sumo_rlc_fini(rdev);
6647                 radeon_wb_fini(rdev);
6648                 radeon_ib_pool_fini(rdev);
6649                 radeon_vm_manager_fini(rdev);
6650                 radeon_irq_kms_fini(rdev);
6651                 si_pcie_gart_fini(rdev);
6652                 rdev->accel_working = false;
6653         }
6654
6655         /* Don't start up if the MC ucode is missing.
6656          * The default clocks and voltages before the MC ucode
6657          * is loaded are not suffient for advanced operations.
6658          */
6659         if (!rdev->mc_fw) {
6660                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6661                 return -EINVAL;
6662         }
6663
6664         return 0;
6665 }
6666
6667 void si_fini(struct radeon_device *rdev)
6668 {
6669         si_cp_fini(rdev);
6670         cayman_dma_fini(rdev);
6671         si_fini_pg(rdev);
6672         si_fini_cg(rdev);
6673         si_irq_fini(rdev);
6674         sumo_rlc_fini(rdev);
6675         radeon_wb_fini(rdev);
6676         radeon_vm_manager_fini(rdev);
6677         radeon_ib_pool_fini(rdev);
6678         radeon_irq_kms_fini(rdev);
6679         if (rdev->has_uvd) {
6680                 uvd_v1_0_fini(rdev);
6681                 radeon_uvd_fini(rdev);
6682         }
6683         si_pcie_gart_fini(rdev);
6684         r600_vram_scratch_fini(rdev);
6685         radeon_gem_fini(rdev);
6686         radeon_fence_driver_fini(rdev);
6687         radeon_bo_fini(rdev);
6688         radeon_atombios_fini(rdev);
6689         kfree(rdev->bios);
6690         rdev->bios = NULL;
6691 }
6692
6693 /**
6694  * si_get_gpu_clock_counter - return GPU clock counter snapshot
6695  *
6696  * @rdev: radeon_device pointer
6697  *
6698  * Fetches a GPU clock counter snapshot (SI).
6699  * Returns the 64 bit clock counter snapshot.
6700  */
6701 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6702 {
6703         uint64_t clock;
6704
6705         mutex_lock(&rdev->gpu_clock_mutex);
6706         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6707         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6708                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6709         mutex_unlock(&rdev->gpu_clock_mutex);
6710         return clock;
6711 }
6712
6713 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6714 {
6715         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6716         int r;
6717
6718         /* bypass vclk and dclk with bclk */
6719         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6720                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6721                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6722
6723         /* put PLL in bypass mode */
6724         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6725
6726         if (!vclk || !dclk) {
6727                 /* keep the Bypass mode, put PLL to sleep */
6728                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6729                 return 0;
6730         }
6731
6732         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6733                                           16384, 0x03FFFFFF, 0, 128, 5,
6734                                           &fb_div, &vclk_div, &dclk_div);
6735         if (r)
6736                 return r;
6737
6738         /* set RESET_ANTI_MUX to 0 */
6739         WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6740
6741         /* set VCO_MODE to 1 */
6742         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6743
6744         /* toggle UPLL_SLEEP to 1 then back to 0 */
6745         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6746         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6747
6748         /* deassert UPLL_RESET */
6749         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6750
6751         mdelay(1);
6752
6753         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6754         if (r)
6755                 return r;
6756
6757         /* assert UPLL_RESET again */
6758         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6759
6760         /* disable spread spectrum. */
6761         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6762
6763         /* set feedback divider */
6764         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6765
6766         /* set ref divider to 0 */
6767         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6768
6769         if (fb_div < 307200)
6770                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6771         else
6772                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6773
6774         /* set PDIV_A and PDIV_B */
6775         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6776                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6777                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6778
6779         /* give the PLL some time to settle */
6780         mdelay(15);
6781
6782         /* deassert PLL_RESET */
6783         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6784
6785         mdelay(15);
6786
6787         /* switch from bypass mode to normal mode */
6788         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6789
6790         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6791         if (r)
6792                 return r;
6793
6794         /* switch VCLK and DCLK selection */
6795         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6796                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6797                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6798
6799         mdelay(100);
6800
6801         return 0;
6802 }
6803
6804 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6805 {
6806         struct pci_dev *root = rdev->pdev->bus->self;
6807         int bridge_pos, gpu_pos;
6808         u32 speed_cntl, mask, current_data_rate;
6809         int ret, i;
6810         u16 tmp16;
6811
6812         if (radeon_pcie_gen2 == 0)
6813                 return;
6814
6815         if (rdev->flags & RADEON_IS_IGP)
6816                 return;
6817
6818         if (!(rdev->flags & RADEON_IS_PCIE))
6819                 return;
6820
6821         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6822         if (ret != 0)
6823                 return;
6824
6825         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6826                 return;
6827
6828         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6829         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6830                 LC_CURRENT_DATA_RATE_SHIFT;
6831         if (mask & DRM_PCIE_SPEED_80) {
6832                 if (current_data_rate == 2) {
6833                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6834                         return;
6835                 }
6836                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6837         } else if (mask & DRM_PCIE_SPEED_50) {
6838                 if (current_data_rate == 1) {
6839                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6840                         return;
6841                 }
6842                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6843         }
6844
6845         bridge_pos = pci_pcie_cap(root);
6846         if (!bridge_pos)
6847                 return;
6848
6849         gpu_pos = pci_pcie_cap(rdev->pdev);
6850         if (!gpu_pos)
6851                 return;
6852
6853         if (mask & DRM_PCIE_SPEED_80) {
6854                 /* re-try equalization if gen3 is not already enabled */
6855                 if (current_data_rate != 2) {
6856                         u16 bridge_cfg, gpu_cfg;
6857                         u16 bridge_cfg2, gpu_cfg2;
6858                         u32 max_lw, current_lw, tmp;
6859
6860                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6861                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6862
6863                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6864                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6865
6866                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6867                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6868
6869                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6870                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6871                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6872
6873                         if (current_lw < max_lw) {
6874                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6875                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6876                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6877                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6878                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6879                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6880                                 }
6881                         }
6882
6883                         for (i = 0; i < 10; i++) {
6884                                 /* check status */
6885                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6886                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6887                                         break;
6888
6889                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6890                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6891
6892                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6893                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6894
6895                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6896                                 tmp |= LC_SET_QUIESCE;
6897                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6898
6899                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6900                                 tmp |= LC_REDO_EQ;
6901                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6902
6903                                 mdelay(100);
6904
6905                                 /* linkctl */
6906                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6907                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6908                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6909                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6910
6911                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6912                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6913                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6914                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6915
6916                                 /* linkctl2 */
6917                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6918                                 tmp16 &= ~((1 << 4) | (7 << 9));
6919                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6920                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6921
6922                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6923                                 tmp16 &= ~((1 << 4) | (7 << 9));
6924                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6925                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6926
6927                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6928                                 tmp &= ~LC_SET_QUIESCE;
6929                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6930                         }
6931                 }
6932         }
6933
6934         /* set the link speed */
6935         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6936         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6937         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6938
6939         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6940         tmp16 &= ~0xf;
6941         if (mask & DRM_PCIE_SPEED_80)
6942                 tmp16 |= 3; /* gen3 */
6943         else if (mask & DRM_PCIE_SPEED_50)
6944                 tmp16 |= 2; /* gen2 */
6945         else
6946                 tmp16 |= 1; /* gen1 */
6947         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6948
6949         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6950         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6951         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6952
6953         for (i = 0; i < rdev->usec_timeout; i++) {
6954                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6955                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6956                         break;
6957                 udelay(1);
6958         }
6959 }
6960
6961 static void si_program_aspm(struct radeon_device *rdev)
6962 {
6963         u32 data, orig;
6964         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6965         bool disable_clkreq = false;
6966
6967         if (radeon_aspm == 0)
6968                 return;
6969
6970         if (!(rdev->flags & RADEON_IS_PCIE))
6971                 return;
6972
6973         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6974         data &= ~LC_XMIT_N_FTS_MASK;
6975         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6976         if (orig != data)
6977                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6978
6979         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6980         data |= LC_GO_TO_RECOVERY;
6981         if (orig != data)
6982                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6983
6984         orig = data = RREG32_PCIE(PCIE_P_CNTL);
6985         data |= P_IGNORE_EDB_ERR;
6986         if (orig != data)
6987                 WREG32_PCIE(PCIE_P_CNTL, data);
6988
6989         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6990         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6991         data |= LC_PMI_TO_L1_DIS;
6992         if (!disable_l0s)
6993                 data |= LC_L0S_INACTIVITY(7);
6994
6995         if (!disable_l1) {
6996                 data |= LC_L1_INACTIVITY(7);
6997                 data &= ~LC_PMI_TO_L1_DIS;
6998                 if (orig != data)
6999                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7000
7001                 if (!disable_plloff_in_l1) {
7002                         bool clk_req_support;
7003
7004                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7005                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7006                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7007                         if (orig != data)
7008                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7009
7010                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7011                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7012                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7013                         if (orig != data)
7014                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7015
7016                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7017                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7018                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7019                         if (orig != data)
7020                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7021
7022                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7023                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7024                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7025                         if (orig != data)
7026                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7027
7028                         if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7029                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7030                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7031                                 if (orig != data)
7032                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7033
7034                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7035                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7036                                 if (orig != data)
7037                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7038
7039                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7040                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7041                                 if (orig != data)
7042                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7043
7044                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7045                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7046                                 if (orig != data)
7047                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7048
7049                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7050                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7051                                 if (orig != data)
7052                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7053
7054                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7055                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7056                                 if (orig != data)
7057                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7058
7059                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7060                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7061                                 if (orig != data)
7062                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7063
7064                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7065                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7066                                 if (orig != data)
7067                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7068                         }
7069                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7070                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7071                         data |= LC_DYN_LANES_PWR_STATE(3);
7072                         if (orig != data)
7073                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7074
7075                         orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7076                         data &= ~LS2_EXIT_TIME_MASK;
7077                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7078                                 data |= LS2_EXIT_TIME(5);
7079                         if (orig != data)
7080                                 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7081
7082                         orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7083                         data &= ~LS2_EXIT_TIME_MASK;
7084                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7085                                 data |= LS2_EXIT_TIME(5);
7086                         if (orig != data)
7087                                 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7088
7089                         if (!disable_clkreq) {
7090                                 struct pci_dev *root = rdev->pdev->bus->self;
7091                                 u32 lnkcap;
7092
7093                                 clk_req_support = false;
7094                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7095                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7096                                         clk_req_support = true;
7097                         } else {
7098                                 clk_req_support = false;
7099                         }
7100
7101                         if (clk_req_support) {
7102                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7103                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7104                                 if (orig != data)
7105                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7106
7107                                 orig = data = RREG32(THM_CLK_CNTL);
7108                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7109                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7110                                 if (orig != data)
7111                                         WREG32(THM_CLK_CNTL, data);
7112
7113                                 orig = data = RREG32(MISC_CLK_CNTL);
7114                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7115                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7116                                 if (orig != data)
7117                                         WREG32(MISC_CLK_CNTL, data);
7118
7119                                 orig = data = RREG32(CG_CLKPIN_CNTL);
7120                                 data &= ~BCLK_AS_XCLK;
7121                                 if (orig != data)
7122                                         WREG32(CG_CLKPIN_CNTL, data);
7123
7124                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7125                                 data &= ~FORCE_BIF_REFCLK_EN;
7126                                 if (orig != data)
7127                                         WREG32(CG_CLKPIN_CNTL_2, data);
7128
7129                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7130                                 data &= ~MPLL_CLKOUT_SEL_MASK;
7131                                 data |= MPLL_CLKOUT_SEL(4);
7132                                 if (orig != data)
7133                                         WREG32(MPLL_BYPASSCLK_SEL, data);
7134
7135                                 orig = data = RREG32(SPLL_CNTL_MODE);
7136                                 data &= ~SPLL_REFCLK_SEL_MASK;
7137                                 if (orig != data)
7138                                         WREG32(SPLL_CNTL_MODE, data);
7139                         }
7140                 }
7141         } else {
7142                 if (orig != data)
7143                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7144         }
7145
7146         orig = data = RREG32_PCIE(PCIE_CNTL2);
7147         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7148         if (orig != data)
7149                 WREG32_PCIE(PCIE_CNTL2, data);
7150
7151         if (!disable_l0s) {
7152                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7153                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7154                         data = RREG32_PCIE(PCIE_LC_STATUS1);
7155                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7156                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7157                                 data &= ~LC_L0S_INACTIVITY_MASK;
7158                                 if (orig != data)
7159                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7160                         }
7161                 }
7162         }
7163 }