drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "nid.h"
  32 #include "atom.h"
  33 #include "ni_reg.h"
  34 #include "cayman_blit_shaders.h"
  35 #include "radeon_ucode.h"
  36 #include "clearstate_cayman.h"
  37
  38 static const u32 tn_rlc_save_restore_register_list[] =
  39 {
  40         0x98fc,
  41         0x98f0,
  42         0x9834,
  43         0x9838,
  44         0x9870,
  45         0x9874,
  46         0x8a14,
  47         0x8b24,
  48         0x8bcc,
  49         0x8b10,
  50         0x8c30,
  51         0x8d00,
  52         0x8d04,
  53         0x8c00,
  54         0x8c04,
  55         0x8c10,
  56         0x8c14,
  57         0x8d8c,
  58         0x8cf0,
  59         0x8e38,
  60         0x9508,
  61         0x9688,
  62         0x9608,
  63         0x960c,
  64         0x9610,
  65         0x9614,
  66         0x88c4,
  67         0x8978,
  68         0x88d4,
  69         0x900c,
  70         0x9100,
  71         0x913c,
  72         0x90e8,
  73         0x9354,
  74         0xa008,
  75         0x98f8,
  76         0x9148,
  77         0x914c,
  78         0x3f94,
  79         0x98f4,
  80         0x9b7c,
  81         0x3f8c,
  82         0x8950,
  83         0x8954,
  84         0x8a18,
  85         0x8b28,
  86         0x9144,
  87         0x3f90,
  88         0x915c,
  89         0x9160,
  90         0x9178,
  91         0x917c,
  92         0x9180,
  93         0x918c,
  94         0x9190,
  95         0x9194,
  96         0x9198,
  97         0x919c,
  98         0x91a8,
  99         0x91ac,
 100         0x91b0,
 101         0x91b4,
 102         0x91b8,
 103         0x91c4,
 104         0x91c8,
 105         0x91cc,
 106         0x91d0,
 107         0x91d4,
 108         0x91e0,
 109         0x91e4,
 110         0x91ec,
 111         0x91f0,
 112         0x91f4,
 113         0x9200,
 114         0x9204,
 115         0x929c,
 116         0x8030,
 117         0x9150,
 118         0x9a60,
 119         0x920c,
 120         0x9210,
 121         0x9228,
 122         0x922c,
 123         0x9244,
 124         0x9248,
 125         0x91e8,
 126         0x9294,
 127         0x9208,
 128         0x9224,
 129         0x9240,
 130         0x9220,
 131         0x923c,
 132         0x9258,
 133         0x9744,
 134         0xa200,
 135         0xa204,
 136         0xa208,
 137         0xa20c,
 138         0x8d58,
 139         0x9030,
 140         0x9034,
 141         0x9038,
 142         0x903c,
 143         0x9040,
 144         0x9654,
 145         0x897c,
 146         0xa210,
 147         0xa214,
 148         0x9868,
 149         0xa02c,
 150         0x9664,
 151         0x9698,
 152         0x949c,
 153         0x8e10,
 154         0x8e18,
 155         0x8c50,
 156         0x8c58,
 157         0x8c60,
 158         0x8c68,
 159         0x89b4,
 160         0x9830,
 161         0x802c,
 162 };
 163
 164 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 165 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 166 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 167 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 168 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 169 extern void evergreen_mc_program(struct radeon_device *rdev);
 170 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 171 extern int evergreen_mc_init(struct radeon_device *rdev);
 172 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 173 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 174 extern void evergreen_program_aspm(struct radeon_device *rdev);
 175 extern void sumo_rlc_fini(struct radeon_device *rdev);
 176 extern int sumo_rlc_init(struct radeon_device *rdev);
 177 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 178
 179 /* Firmware Names */
 180 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 181 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 183 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 184 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 185 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 188 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 189 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 192 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 193 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 197 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 198 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 200 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 201
 202
 203 static const u32 cayman_golden_registers2[] =
 204 {
 205         0x3e5c, 0xffffffff, 0x00000000,
 206         0x3e48, 0xffffffff, 0x00000000,
 207         0x3e4c, 0xffffffff, 0x00000000,
 208         0x3e64, 0xffffffff, 0x00000000,
 209         0x3e50, 0xffffffff, 0x00000000,
 210         0x3e60, 0xffffffff, 0x00000000
 211 };
 212
 213 static const u32 cayman_golden_registers[] =
 214 {
 215         0x5eb4, 0xffffffff, 0x00000002,
 216         0x5e78, 0x8f311ff1, 0x001000f0,
 217         0x3f90, 0xffff0000, 0xff000000,
 218         0x9148, 0xffff0000, 0xff000000,
 219         0x3f94, 0xffff0000, 0xff000000,
 220         0x914c, 0xffff0000, 0xff000000,
 221         0xc78, 0x00000080, 0x00000080,
 222         0xbd4, 0x70073777, 0x00011003,
 223         0xd02c, 0xbfffff1f, 0x08421000,
 224         0xd0b8, 0x73773777, 0x02011003,
 225         0x5bc0, 0x00200000, 0x50100000,
 226         0x98f8, 0x33773777, 0x02011003,
 227         0x98fc, 0xffffffff, 0x76541032,
 228         0x7030, 0x31000311, 0x00000011,
 229         0x2f48, 0x33773777, 0x42010001,
 230         0x6b28, 0x00000010, 0x00000012,
 231         0x7728, 0x00000010, 0x00000012,
 232         0x10328, 0x00000010, 0x00000012,
 233         0x10f28, 0x00000010, 0x00000012,
 234         0x11b28, 0x00000010, 0x00000012,
 235         0x12728, 0x00000010, 0x00000012,
 236         0x240c, 0x000007ff, 0x00000000,
 237         0x8a14, 0xf000001f, 0x00000007,
 238         0x8b24, 0x3fff3fff, 0x00ff0fff,
 239         0x8b10, 0x0000ff0f, 0x00000000,
 240         0x28a4c, 0x07ffffff, 0x06000000,
 241         0x10c, 0x00000001, 0x00010003,
 242         0xa02c, 0xffffffff, 0x0000009b,
 243         0x913c, 0x0000010f, 0x01000100,
 244         0x8c04, 0xf8ff00ff, 0x40600060,
 245         0x28350, 0x00000f01, 0x00000000,
 246         0x9508, 0x3700001f, 0x00000002,
 247         0x960c, 0xffffffff, 0x54763210,
 248         0x88c4, 0x001f3ae3, 0x00000082,
 249         0x88d0, 0xffffffff, 0x0f40df40,
 250         0x88d4, 0x0000001f, 0x00000010,
 251         0x8974, 0xffffffff, 0x00000000
 252 };
 253
 254 static const u32 dvst_golden_registers2[] =
 255 {
 256         0x8f8, 0xffffffff, 0,
 257         0x8fc, 0x00380000, 0,
 258         0x8f8, 0xffffffff, 1,
 259         0x8fc, 0x0e000000, 0
 260 };
 261
 262 static const u32 dvst_golden_registers[] =
 263 {
 264         0x690, 0x3fff3fff, 0x20c00033,
 265         0x918c, 0x0fff0fff, 0x00010006,
 266         0x91a8, 0x0fff0fff, 0x00010006,
 267         0x9150, 0xffffdfff, 0x6e944040,
 268         0x917c, 0x0fff0fff, 0x00030002,
 269         0x9198, 0x0fff0fff, 0x00030002,
 270         0x915c, 0x0fff0fff, 0x00010000,
 271         0x3f90, 0xffff0001, 0xff000000,
 272         0x9178, 0x0fff0fff, 0x00070000,
 273         0x9194, 0x0fff0fff, 0x00070000,
 274         0x9148, 0xffff0001, 0xff000000,
 275         0x9190, 0x0fff0fff, 0x00090008,
 276         0x91ac, 0x0fff0fff, 0x00090008,
 277         0x3f94, 0xffff0000, 0xff000000,
 278         0x914c, 0xffff0000, 0xff000000,
 279         0x929c, 0x00000fff, 0x00000001,
 280         0x55e4, 0xff607fff, 0xfc000100,
 281         0x8a18, 0xff000fff, 0x00000100,
 282         0x8b28, 0xff000fff, 0x00000100,
 283         0x9144, 0xfffc0fff, 0x00000100,
 284         0x6ed8, 0x00010101, 0x00010000,
 285         0x9830, 0xffffffff, 0x00000000,
 286         0x9834, 0xf00fffff, 0x00000400,
 287         0x9838, 0xfffffffe, 0x00000000,
 288         0xd0c0, 0xff000fff, 0x00000100,
 289         0xd02c, 0xbfffff1f, 0x08421000,
 290         0xd0b8, 0x73773777, 0x12010001,
 291         0x5bb0, 0x000000f0, 0x00000070,
 292         0x98f8, 0x73773777, 0x12010001,
 293         0x98fc, 0xffffffff, 0x00000010,
 294         0x9b7c, 0x00ff0000, 0x00fc0000,
 295         0x8030, 0x00001f0f, 0x0000100a,
 296         0x2f48, 0x73773777, 0x12010001,
 297         0x2408, 0x00030000, 0x000c007f,
 298         0x8a14, 0xf000003f, 0x00000007,
 299         0x8b24, 0x3fff3fff, 0x00ff0fff,
 300         0x8b10, 0x0000ff0f, 0x00000000,
 301         0x28a4c, 0x07ffffff, 0x06000000,
 302         0x4d8, 0x00000fff, 0x00000100,
 303         0xa008, 0xffffffff, 0x00010000,
 304         0x913c, 0xffff03ff, 0x01000100,
 305         0x8c00, 0x000000ff, 0x00000003,
 306         0x8c04, 0xf8ff00ff, 0x40600060,
 307         0x8cf0, 0x1fff1fff, 0x08e00410,
 308         0x28350, 0x00000f01, 0x00000000,
 309         0x9508, 0xf700071f, 0x00000002,
 310         0x960c, 0xffffffff, 0x54763210,
 311         0x20ef8, 0x01ff01ff, 0x00000002,
 312         0x20e98, 0xfffffbff, 0x00200000,
 313         0x2015c, 0xffffffff, 0x00000f40,
 314         0x88c4, 0x001f3ae3, 0x00000082,
 315         0x8978, 0x3fffffff, 0x04050140,
 316         0x88d4, 0x0000001f, 0x00000010,
 317         0x8974, 0xffffffff, 0x00000000
 318 };
 319
 320 static const u32 scrapper_golden_registers[] =
 321 {
 322         0x690, 0x3fff3fff, 0x20c00033,
 323         0x918c, 0x0fff0fff, 0x00010006,
 324         0x918c, 0x0fff0fff, 0x00010006,
 325         0x91a8, 0x0fff0fff, 0x00010006,
 326         0x91a8, 0x0fff0fff, 0x00010006,
 327         0x9150, 0xffffdfff, 0x6e944040,
 328         0x9150, 0xffffdfff, 0x6e944040,
 329         0x917c, 0x0fff0fff, 0x00030002,
 330         0x917c, 0x0fff0fff, 0x00030002,
 331         0x9198, 0x0fff0fff, 0x00030002,
 332         0x9198, 0x0fff0fff, 0x00030002,
 333         0x915c, 0x0fff0fff, 0x00010000,
 334         0x915c, 0x0fff0fff, 0x00010000,
 335         0x3f90, 0xffff0001, 0xff000000,
 336         0x3f90, 0xffff0001, 0xff000000,
 337         0x9178, 0x0fff0fff, 0x00070000,
 338         0x9178, 0x0fff0fff, 0x00070000,
 339         0x9194, 0x0fff0fff, 0x00070000,
 340         0x9194, 0x0fff0fff, 0x00070000,
 341         0x9148, 0xffff0001, 0xff000000,
 342         0x9148, 0xffff0001, 0xff000000,
 343         0x9190, 0x0fff0fff, 0x00090008,
 344         0x9190, 0x0fff0fff, 0x00090008,
 345         0x91ac, 0x0fff0fff, 0x00090008,
 346         0x91ac, 0x0fff0fff, 0x00090008,
 347         0x3f94, 0xffff0000, 0xff000000,
 348         0x3f94, 0xffff0000, 0xff000000,
 349         0x914c, 0xffff0000, 0xff000000,
 350         0x914c, 0xffff0000, 0xff000000,
 351         0x929c, 0x00000fff, 0x00000001,
 352         0x929c, 0x00000fff, 0x00000001,
 353         0x55e4, 0xff607fff, 0xfc000100,
 354         0x8a18, 0xff000fff, 0x00000100,
 355         0x8a18, 0xff000fff, 0x00000100,
 356         0x8b28, 0xff000fff, 0x00000100,
 357         0x8b28, 0xff000fff, 0x00000100,
 358         0x9144, 0xfffc0fff, 0x00000100,
 359         0x9144, 0xfffc0fff, 0x00000100,
 360         0x6ed8, 0x00010101, 0x00010000,
 361         0x9830, 0xffffffff, 0x00000000,
 362         0x9830, 0xffffffff, 0x00000000,
 363         0x9834, 0xf00fffff, 0x00000400,
 364         0x9834, 0xf00fffff, 0x00000400,
 365         0x9838, 0xfffffffe, 0x00000000,
 366         0x9838, 0xfffffffe, 0x00000000,
 367         0xd0c0, 0xff000fff, 0x00000100,
 368         0xd02c, 0xbfffff1f, 0x08421000,
 369         0xd02c, 0xbfffff1f, 0x08421000,
 370         0xd0b8, 0x73773777, 0x12010001,
 371         0xd0b8, 0x73773777, 0x12010001,
 372         0x5bb0, 0x000000f0, 0x00000070,
 373         0x98f8, 0x73773777, 0x12010001,
 374         0x98f8, 0x73773777, 0x12010001,
 375         0x98fc, 0xffffffff, 0x00000010,
 376         0x98fc, 0xffffffff, 0x00000010,
 377         0x9b7c, 0x00ff0000, 0x00fc0000,
 378         0x9b7c, 0x00ff0000, 0x00fc0000,
 379         0x8030, 0x00001f0f, 0x0000100a,
 380         0x8030, 0x00001f0f, 0x0000100a,
 381         0x2f48, 0x73773777, 0x12010001,
 382         0x2f48, 0x73773777, 0x12010001,
 383         0x2408, 0x00030000, 0x000c007f,
 384         0x8a14, 0xf000003f, 0x00000007,
 385         0x8a14, 0xf000003f, 0x00000007,
 386         0x8b24, 0x3fff3fff, 0x00ff0fff,
 387         0x8b24, 0x3fff3fff, 0x00ff0fff,
 388         0x8b10, 0x0000ff0f, 0x00000000,
 389         0x8b10, 0x0000ff0f, 0x00000000,
 390         0x28a4c, 0x07ffffff, 0x06000000,
 391         0x28a4c, 0x07ffffff, 0x06000000,
 392         0x4d8, 0x00000fff, 0x00000100,
 393         0x4d8, 0x00000fff, 0x00000100,
 394         0xa008, 0xffffffff, 0x00010000,
 395         0xa008, 0xffffffff, 0x00010000,
 396         0x913c, 0xffff03ff, 0x01000100,
 397         0x913c, 0xffff03ff, 0x01000100,
 398         0x90e8, 0x001fffff, 0x010400c0,
 399         0x8c00, 0x000000ff, 0x00000003,
 400         0x8c00, 0x000000ff, 0x00000003,
 401         0x8c04, 0xf8ff00ff, 0x40600060,
 402         0x8c04, 0xf8ff00ff, 0x40600060,
 403         0x8c30, 0x0000000f, 0x00040005,
 404         0x8cf0, 0x1fff1fff, 0x08e00410,
 405         0x8cf0, 0x1fff1fff, 0x08e00410,
 406         0x900c, 0x00ffffff, 0x0017071f,
 407         0x28350, 0x00000f01, 0x00000000,
 408         0x28350, 0x00000f01, 0x00000000,
 409         0x9508, 0xf700071f, 0x00000002,
 410         0x9508, 0xf700071f, 0x00000002,
 411         0x9688, 0x00300000, 0x0017000f,
 412         0x960c, 0xffffffff, 0x54763210,
 413         0x960c, 0xffffffff, 0x54763210,
 414         0x20ef8, 0x01ff01ff, 0x00000002,
 415         0x20e98, 0xfffffbff, 0x00200000,
 416         0x2015c, 0xffffffff, 0x00000f40,
 417         0x88c4, 0x001f3ae3, 0x00000082,
 418         0x88c4, 0x001f3ae3, 0x00000082,
 419         0x8978, 0x3fffffff, 0x04050140,
 420         0x8978, 0x3fffffff, 0x04050140,
 421         0x88d4, 0x0000001f, 0x00000010,
 422         0x88d4, 0x0000001f, 0x00000010,
 423         0x8974, 0xffffffff, 0x00000000,
 424         0x8974, 0xffffffff, 0x00000000
 425 };
 426
 427 static void ni_init_golden_registers(struct radeon_device *rdev)
 428 {
 429         switch (rdev->family) {
 430         case CHIP_CAYMAN:
 431                 radeon_program_register_sequence(rdev,
 432                                                  cayman_golden_registers,
 433                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 434                 radeon_program_register_sequence(rdev,
 435                                                  cayman_golden_registers2,
 436                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 437                 break;
 438         case CHIP_ARUBA:
 439                 if ((rdev->pdev->device == 0x9900) ||
 440                     (rdev->pdev->device == 0x9901) ||
 441                     (rdev->pdev->device == 0x9903) ||
 442                     (rdev->pdev->device == 0x9904) ||
 443                     (rdev->pdev->device == 0x9905) ||
 444                     (rdev->pdev->device == 0x9906) ||
 445                     (rdev->pdev->device == 0x9907) ||
 446                     (rdev->pdev->device == 0x9908) ||
 447                     (rdev->pdev->device == 0x9909) ||
 448                     (rdev->pdev->device == 0x990A) ||
 449                     (rdev->pdev->device == 0x990B) ||
 450                     (rdev->pdev->device == 0x990C) ||
 451                     (rdev->pdev->device == 0x990D) ||
 452                     (rdev->pdev->device == 0x990E) ||
 453                     (rdev->pdev->device == 0x990F) ||
 454                     (rdev->pdev->device == 0x9910) ||
 455                     (rdev->pdev->device == 0x9913) ||
 456                     (rdev->pdev->device == 0x9917) ||
 457                     (rdev->pdev->device == 0x9918)) {
 458                         radeon_program_register_sequence(rdev,
 459                                                          dvst_golden_registers,
 460                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 461                         radeon_program_register_sequence(rdev,
 462                                                          dvst_golden_registers2,
 463                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 464                 } else {
 465                         radeon_program_register_sequence(rdev,
 466                                                          scrapper_golden_registers,
 467                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 468                         radeon_program_register_sequence(rdev,
 469                                                          dvst_golden_registers2,
 470                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 471                 }
 472                 break;
 473         default:
 474                 break;
 475         }
 476 }
 477
 478 #define BTC_IO_MC_REGS_SIZE 29
 479
 480 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 481         {0x00000077, 0xff010100},
 482         {0x00000078, 0x00000000},
 483         {0x00000079, 0x00001434},
 484         {0x0000007a, 0xcc08ec08},
 485         {0x0000007b, 0x00040000},
 486         {0x0000007c, 0x000080c0},
 487         {0x0000007d, 0x09000000},
 488         {0x0000007e, 0x00210404},
 489         {0x00000081, 0x08a8e800},
 490         {0x00000082, 0x00030444},
 491         {0x00000083, 0x00000000},
 492         {0x00000085, 0x00000001},
 493         {0x00000086, 0x00000002},
 494         {0x00000087, 0x48490000},
 495         {0x00000088, 0x20244647},
 496         {0x00000089, 0x00000005},
 497         {0x0000008b, 0x66030000},
 498         {0x0000008c, 0x00006603},
 499         {0x0000008d, 0x00000100},
 500         {0x0000008f, 0x00001c0a},
 501         {0x00000090, 0xff000001},
 502         {0x00000094, 0x00101101},
 503         {0x00000095, 0x00000fff},
 504         {0x00000096, 0x00116fff},
 505         {0x00000097, 0x60010000},
 506         {0x00000098, 0x10010000},
 507         {0x00000099, 0x00006000},
 508         {0x0000009a, 0x00001000},
 509         {0x0000009f, 0x00946a00}
 510 };
 511
 512 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 513         {0x00000077, 0xff010100},
 514         {0x00000078, 0x00000000},
 515         {0x00000079, 0x00001434},
 516         {0x0000007a, 0xcc08ec08},
 517         {0x0000007b, 0x00040000},
 518         {0x0000007c, 0x000080c0},
 519         {0x0000007d, 0x09000000},
 520         {0x0000007e, 0x00210404},
 521         {0x00000081, 0x08a8e800},
 522         {0x00000082, 0x00030444},
 523         {0x00000083, 0x00000000},
 524         {0x00000085, 0x00000001},
 525         {0x00000086, 0x00000002},
 526         {0x00000087, 0x48490000},
 527         {0x00000088, 0x20244647},
 528         {0x00000089, 0x00000005},
 529         {0x0000008b, 0x66030000},
 530         {0x0000008c, 0x00006603},
 531         {0x0000008d, 0x00000100},
 532         {0x0000008f, 0x00001c0a},
 533         {0x00000090, 0xff000001},
 534         {0x00000094, 0x00101101},
 535         {0x00000095, 0x00000fff},
 536         {0x00000096, 0x00116fff},
 537         {0x00000097, 0x60010000},
 538         {0x00000098, 0x10010000},
 539         {0x00000099, 0x00006000},
 540         {0x0000009a, 0x00001000},
 541         {0x0000009f, 0x00936a00}
 542 };
 543
 544 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 545         {0x00000077, 0xff010100},
 546         {0x00000078, 0x00000000},
 547         {0x00000079, 0x00001434},
 548         {0x0000007a, 0xcc08ec08},
 549         {0x0000007b, 0x00040000},
 550         {0x0000007c, 0x000080c0},
 551         {0x0000007d, 0x09000000},
 552         {0x0000007e, 0x00210404},
 553         {0x00000081, 0x08a8e800},
 554         {0x00000082, 0x00030444},
 555         {0x00000083, 0x00000000},
 556         {0x00000085, 0x00000001},
 557         {0x00000086, 0x00000002},
 558         {0x00000087, 0x48490000},
 559         {0x00000088, 0x20244647},
 560         {0x00000089, 0x00000005},
 561         {0x0000008b, 0x66030000},
 562         {0x0000008c, 0x00006603},
 563         {0x0000008d, 0x00000100},
 564         {0x0000008f, 0x00001c0a},
 565         {0x00000090, 0xff000001},
 566         {0x00000094, 0x00101101},
 567         {0x00000095, 0x00000fff},
 568         {0x00000096, 0x00116fff},
 569         {0x00000097, 0x60010000},
 570         {0x00000098, 0x10010000},
 571         {0x00000099, 0x00006000},
 572         {0x0000009a, 0x00001000},
 573         {0x0000009f, 0x00916a00}
 574 };
 575
 576 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 577         {0x00000077, 0xff010100},
 578         {0x00000078, 0x00000000},
 579         {0x00000079, 0x00001434},
 580         {0x0000007a, 0xcc08ec08},
 581         {0x0000007b, 0x00040000},
 582         {0x0000007c, 0x000080c0},
 583         {0x0000007d, 0x09000000},
 584         {0x0000007e, 0x00210404},
 585         {0x00000081, 0x08a8e800},
 586         {0x00000082, 0x00030444},
 587         {0x00000083, 0x00000000},
 588         {0x00000085, 0x00000001},
 589         {0x00000086, 0x00000002},
 590         {0x00000087, 0x48490000},
 591         {0x00000088, 0x20244647},
 592         {0x00000089, 0x00000005},
 593         {0x0000008b, 0x66030000},
 594         {0x0000008c, 0x00006603},
 595         {0x0000008d, 0x00000100},
 596         {0x0000008f, 0x00001c0a},
 597         {0x00000090, 0xff000001},
 598         {0x00000094, 0x00101101},
 599         {0x00000095, 0x00000fff},
 600         {0x00000096, 0x00116fff},
 601         {0x00000097, 0x60010000},
 602         {0x00000098, 0x10010000},
 603         {0x00000099, 0x00006000},
 604         {0x0000009a, 0x00001000},
 605         {0x0000009f, 0x00976b00}
 606 };
 607
 608 int ni_mc_load_microcode(struct radeon_device *rdev)
 609 {
 610         const __be32 *fw_data;
 611         u32 mem_type, running, blackout = 0;
 612         u32 *io_mc_regs;
 613         int i, ucode_size, regs_size;
 614
 615         if (!rdev->mc_fw)
 616                 return -EINVAL;
 617
 618         switch (rdev->family) {
 619         case CHIP_BARTS:
 620                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 621                 ucode_size = BTC_MC_UCODE_SIZE;
 622                 regs_size = BTC_IO_MC_REGS_SIZE;
 623                 break;
 624         case CHIP_TURKS:
 625                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 626                 ucode_size = BTC_MC_UCODE_SIZE;
 627                 regs_size = BTC_IO_MC_REGS_SIZE;
 628                 break;
 629         case CHIP_CAICOS:
 630         default:
 631                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 632                 ucode_size = BTC_MC_UCODE_SIZE;
 633                 regs_size = BTC_IO_MC_REGS_SIZE;
 634                 break;
 635         case CHIP_CAYMAN:
 636                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 637                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 638                 regs_size = BTC_IO_MC_REGS_SIZE;
 639                 break;
 640         }
 641
 642         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 643         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 644
 645         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 646                 if (running) {
 647                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 648                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 649                 }
 650
 651                 /* reset the engine and set to writable */
 652                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 653                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 654
 655                 /* load mc io regs */
 656                 for (i = 0; i < regs_size; i++) {
 657                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 658                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 659                 }
 660                 /* load the MC ucode */
 661                 fw_data = (const __be32 *)rdev->mc_fw->data;
 662                 for (i = 0; i < ucode_size; i++)
 663                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 664
 665                 /* put the engine back into the active state */
 666                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 668                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 669
 670                 /* wait for training to complete */
 671                 for (i = 0; i < rdev->usec_timeout; i++) {
 672                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 673                                 break;
 674                         udelay(1);
 675                 }
 676
 677                 if (running)
 678                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 679         }
 680
 681         return 0;
 682 }
 683
 684 int ni_init_microcode(struct radeon_device *rdev)
 685 {
 686         const char *chip_name;
 687         const char *rlc_chip_name;
 688         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 689         size_t smc_req_size = 0;
 690         char fw_name[30];
 691         int err;
 692
 693         DRM_DEBUG("\n");
 694
 695         switch (rdev->family) {
 696         case CHIP_BARTS:
 697                 chip_name = "BARTS";
 698                 rlc_chip_name = "BTC";
 699                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 700                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 701                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 702                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 703                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 704                 break;
 705         case CHIP_TURKS:
 706                 chip_name = "TURKS";
 707                 rlc_chip_name = "BTC";
 708                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 709                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 710                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 711                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 712                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 713                 break;
 714         case CHIP_CAICOS:
 715                 chip_name = "CAICOS";
 716                 rlc_chip_name = "BTC";
 717                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 718                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 719                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 720                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 721                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 722                 break;
 723         case CHIP_CAYMAN:
 724                 chip_name = "CAYMAN";
 725                 rlc_chip_name = "CAYMAN";
 726                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 727                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 728                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 729                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 730                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 731                 break;
 732         case CHIP_ARUBA:
 733                 chip_name = "ARUBA";
 734                 rlc_chip_name = "ARUBA";
 735                 /* pfp/me same size as CAYMAN */
 736                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 737                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 738                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 739                 mc_req_size = 0;
 740                 break;
 741         default: BUG();
 742         }
 743
 744         DRM_INFO("Loading %s Microcode\n", chip_name);
 745
 746         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 747         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 748         if (err)
 749                 goto out;
 750         if (rdev->pfp_fw->size != pfp_req_size) {
 751                 printk(KERN_ERR
 752                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 753                        rdev->pfp_fw->size, fw_name);
 754                 err = -EINVAL;
 755                 goto out;
 756         }
 757
 758         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 759         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 760         if (err)
 761                 goto out;
 762         if (rdev->me_fw->size != me_req_size) {
 763                 printk(KERN_ERR
 764                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 765                        rdev->me_fw->size, fw_name);
 766                 err = -EINVAL;
 767         }
 768
 769         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 770         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 771         if (err)
 772                 goto out;
 773         if (rdev->rlc_fw->size != rlc_req_size) {
 774                 printk(KERN_ERR
 775                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 776                        rdev->rlc_fw->size, fw_name);
 777                 err = -EINVAL;
 778         }
 779
 780         /* no MC ucode on TN */
 781         if (!(rdev->flags & RADEON_IS_IGP)) {
 782                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 783                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 784                 if (err)
 785                         goto out;
 786                 if (rdev->mc_fw->size != mc_req_size) {
 787                         printk(KERN_ERR
 788                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 789                                rdev->mc_fw->size, fw_name);
 790                         err = -EINVAL;
 791                 }
 792         }
 793
 794         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 795                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 796                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 797                 if (err) {
 798                         printk(KERN_ERR
 799                                "smc: error loading firmware \"%s\"\n",
 800                                fw_name);
 801                         release_firmware(rdev->smc_fw);
 802                         rdev->smc_fw = NULL;
 803                         err = 0;
 804                 } else if (rdev->smc_fw->size != smc_req_size) {
 805                         printk(KERN_ERR
 806                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 807                                rdev->mc_fw->size, fw_name);
 808                         err = -EINVAL;
 809                 }
 810         }
 811
 812 out:
 813         if (err) {
 814                 if (err != -EINVAL)
 815                         printk(KERN_ERR
 816                                "ni_cp: Failed to load firmware \"%s\"\n",
 817                                fw_name);
 818                 release_firmware(rdev->pfp_fw);
 819                 rdev->pfp_fw = NULL;
 820                 release_firmware(rdev->me_fw);
 821                 rdev->me_fw = NULL;
 822                 release_firmware(rdev->rlc_fw);
 823                 rdev->rlc_fw = NULL;
 824                 release_firmware(rdev->mc_fw);
 825                 rdev->mc_fw = NULL;
 826         }
 827         return err;
 828 }
 829
 830 int tn_get_temp(struct radeon_device *rdev)
 831 {
 832         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 833         int actual_temp = (temp / 8) - 49;
 834
 835         return actual_temp * 1000;
 836 }
 837
 838 /*
 839  * Core functions
 840  */
 841 static void cayman_gpu_init(struct radeon_device *rdev)
 842 {
 843         u32 gb_addr_config = 0;
 844         u32 mc_shared_chmap, mc_arb_ramcfg;
 845         u32 cgts_tcc_disable;
 846         u32 sx_debug_1;
 847         u32 smx_dc_ctl0;
 848         u32 cgts_sm_ctrl_reg;
 849         u32 hdp_host_path_cntl;
 850         u32 tmp;
 851         u32 disabled_rb_mask;
 852         int i, j;
 853
 854         switch (rdev->family) {
 855         case CHIP_CAYMAN:
 856                 rdev->config.cayman.max_shader_engines = 2;
 857                 rdev->config.cayman.max_pipes_per_simd = 4;
 858                 rdev->config.cayman.max_tile_pipes = 8;
 859                 rdev->config.cayman.max_simds_per_se = 12;
 860                 rdev->config.cayman.max_backends_per_se = 4;
 861                 rdev->config.cayman.max_texture_channel_caches = 8;
 862                 rdev->config.cayman.max_gprs = 256;
 863                 rdev->config.cayman.max_threads = 256;
 864                 rdev->config.cayman.max_gs_threads = 32;
 865                 rdev->config.cayman.max_stack_entries = 512;
 866                 rdev->config.cayman.sx_num_of_sets = 8;
 867                 rdev->config.cayman.sx_max_export_size = 256;
 868                 rdev->config.cayman.sx_max_export_pos_size = 64;
 869                 rdev->config.cayman.sx_max_export_smx_size = 192;
 870                 rdev->config.cayman.max_hw_contexts = 8;
 871                 rdev->config.cayman.sq_num_cf_insts = 2;
 872
 873                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 874                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 875                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 876                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 877                 break;
 878         case CHIP_ARUBA:
 879         default:
 880                 rdev->config.cayman.max_shader_engines = 1;
 881                 rdev->config.cayman.max_pipes_per_simd = 4;
 882                 rdev->config.cayman.max_tile_pipes = 2;
 883                 if ((rdev->pdev->device == 0x9900) ||
 884                     (rdev->pdev->device == 0x9901) ||
 885                     (rdev->pdev->device == 0x9905) ||
 886                     (rdev->pdev->device == 0x9906) ||
 887                     (rdev->pdev->device == 0x9907) ||
 888                     (rdev->pdev->device == 0x9908) ||
 889                     (rdev->pdev->device == 0x9909) ||
 890                     (rdev->pdev->device == 0x990B) ||
 891                     (rdev->pdev->device == 0x990C) ||
 892                     (rdev->pdev->device == 0x990F) ||
 893                     (rdev->pdev->device == 0x9910) ||
 894                     (rdev->pdev->device == 0x9917) ||
 895                     (rdev->pdev->device == 0x9999) ||
 896                     (rdev->pdev->device == 0x999C)) {
 897                         rdev->config.cayman.max_simds_per_se = 6;
 898                         rdev->config.cayman.max_backends_per_se = 2;
 899                         rdev->config.cayman.max_hw_contexts = 8;
 900                         rdev->config.cayman.sx_max_export_size = 256;
 901                         rdev->config.cayman.sx_max_export_pos_size = 64;
 902                         rdev->config.cayman.sx_max_export_smx_size = 192;
 903                 } else if ((rdev->pdev->device == 0x9903) ||
 904                            (rdev->pdev->device == 0x9904) ||
 905                            (rdev->pdev->device == 0x990A) ||
 906                            (rdev->pdev->device == 0x990D) ||
 907                            (rdev->pdev->device == 0x990E) ||
 908                            (rdev->pdev->device == 0x9913) ||
 909                            (rdev->pdev->device == 0x9918) ||
 910                            (rdev->pdev->device == 0x999D)) {
 911                         rdev->config.cayman.max_simds_per_se = 4;
 912                         rdev->config.cayman.max_backends_per_se = 2;
 913                         rdev->config.cayman.max_hw_contexts = 8;
 914                         rdev->config.cayman.sx_max_export_size = 256;
 915                         rdev->config.cayman.sx_max_export_pos_size = 64;
 916                         rdev->config.cayman.sx_max_export_smx_size = 192;
 917                 } else if ((rdev->pdev->device == 0x9919) ||
 918                            (rdev->pdev->device == 0x9990) ||
 919                            (rdev->pdev->device == 0x9991) ||
 920                            (rdev->pdev->device == 0x9994) ||
 921                            (rdev->pdev->device == 0x9995) ||
 922                            (rdev->pdev->device == 0x9996) ||
 923                            (rdev->pdev->device == 0x999A) ||
 924                            (rdev->pdev->device == 0x99A0)) {
 925                         rdev->config.cayman.max_simds_per_se = 3;
 926                         rdev->config.cayman.max_backends_per_se = 1;
 927                         rdev->config.cayman.max_hw_contexts = 4;
 928                         rdev->config.cayman.sx_max_export_size = 128;
 929                         rdev->config.cayman.sx_max_export_pos_size = 32;
 930                         rdev->config.cayman.sx_max_export_smx_size = 96;
 931                 } else {
 932                         rdev->config.cayman.max_simds_per_se = 2;
 933                         rdev->config.cayman.max_backends_per_se = 1;
 934                         rdev->config.cayman.max_hw_contexts = 4;
 935                         rdev->config.cayman.sx_max_export_size = 128;
 936                         rdev->config.cayman.sx_max_export_pos_size = 32;
 937                         rdev->config.cayman.sx_max_export_smx_size = 96;
 938                 }
 939                 rdev->config.cayman.max_texture_channel_caches = 2;
 940                 rdev->config.cayman.max_gprs = 256;
 941                 rdev->config.cayman.max_threads = 256;
 942                 rdev->config.cayman.max_gs_threads = 32;
 943                 rdev->config.cayman.max_stack_entries = 512;
 944                 rdev->config.cayman.sx_num_of_sets = 8;
 945                 rdev->config.cayman.sq_num_cf_insts = 2;
 946
 947                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 948                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 949                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 950                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 951                 break;
 952         }
 953
 954         /* Initialize HDP */
 955         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 956                 WREG32((0x2c14 + j), 0x00000000);
 957                 WREG32((0x2c18 + j), 0x00000000);
 958                 WREG32((0x2c1c + j), 0x00000000);
 959                 WREG32((0x2c20 + j), 0x00000000);
 960                 WREG32((0x2c24 + j), 0x00000000);
 961         }
 962
 963         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 964
 965         evergreen_fix_pci_max_read_req_size(rdev);
 966
 967         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 968         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 969
 970         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 971         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 972         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 973                 rdev->config.cayman.mem_row_size_in_kb = 4;
 974         /* XXX use MC settings? */
 975         rdev->config.cayman.shader_engine_tile_size = 32;
 976         rdev->config.cayman.num_gpus = 1;
 977         rdev->config.cayman.multi_gpu_tile_size = 64;
 978
 979         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 980         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 981         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 982         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 983         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 984         rdev->config.cayman.num_shader_engines = tmp + 1;
 985         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 986         rdev->config.cayman.num_gpus = tmp + 1;
 987         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 988         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 989         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 990         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 991
 992
 993         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 994          * not have bank info, so create a custom tiling dword.
 995          * bits 3:0   num_pipes
 996          * bits 7:4   num_banks
 997          * bits 11:8  group_size
 998          * bits 15:12 row_size
 999          */
1000         rdev->config.cayman.tile_config = 0;
1001         switch (rdev->config.cayman.num_tile_pipes) {
1002         case 1:
1003         default:
1004                 rdev->config.cayman.tile_config |= (0 << 0);
1005                 break;
1006         case 2:
1007                 rdev->config.cayman.tile_config |= (1 << 0);
1008                 break;
1009         case 4:
1010                 rdev->config.cayman.tile_config |= (2 << 0);
1011                 break;
1012         case 8:
1013                 rdev->config.cayman.tile_config |= (3 << 0);
1014                 break;
1015         }
1016
1017         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1018         if (rdev->flags & RADEON_IS_IGP)
1019                 rdev->config.cayman.tile_config |= 1 << 4;
1020         else {
1021                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1022                 case 0: /* four banks */
1023                         rdev->config.cayman.tile_config |= 0 << 4;
1024                         break;
1025                 case 1: /* eight banks */
1026                         rdev->config.cayman.tile_config |= 1 << 4;
1027                         break;
1028                 case 2: /* sixteen banks */
1029                 default:
1030                         rdev->config.cayman.tile_config |= 2 << 4;
1031                         break;
1032                 }
1033         }
1034         rdev->config.cayman.tile_config |=
1035                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1036         rdev->config.cayman.tile_config |=
1037                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1038
1039         tmp = 0;
1040         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1041                 u32 rb_disable_bitmap;
1042
1043                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1044                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1045                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1046                 tmp <<= 4;
1047                 tmp |= rb_disable_bitmap;
1048         }
1049         /* enabled rb are just the one not disabled :) */
1050         disabled_rb_mask = tmp;
1051         tmp = 0;
1052         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1053                 tmp |= (1 << i);
1054         /* if all the backends are disabled, fix it up here */
1055         if ((disabled_rb_mask & tmp) == tmp) {
1056                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1057                         disabled_rb_mask &= ~(1 << i);
1058         }
1059
1060         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1061                 u32 simd_disable_bitmap;
1062
1063                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1064                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1065                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1066                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1067                 tmp <<= 16;
1068                 tmp |= simd_disable_bitmap;
1069         }
1070         rdev->config.cayman.active_simds = hweight32(~tmp);
1071
1072         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1073         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1074
1075         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1076         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1077         if (ASIC_IS_DCE6(rdev))
1078                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1079         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1080         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1081         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1082         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1083         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1084         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1085
1086         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1087             (rdev->flags & RADEON_IS_IGP)) {
1088                 if ((disabled_rb_mask & 3) == 1) {
1089                         /* RB0 disabled, RB1 enabled */
1090                         tmp = 0x11111111;
1091                 } else {
1092                         /* RB1 disabled, RB0 enabled */
1093                         tmp = 0x00000000;
1094                 }
1095         } else {
1096                 tmp = gb_addr_config & NUM_PIPES_MASK;
1097                 tmp = r6xx_remap_render_backend(rdev, tmp,
1098                                                 rdev->config.cayman.max_backends_per_se *
1099                                                 rdev->config.cayman.max_shader_engines,
1100                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1101         }
1102         WREG32(GB_BACKEND_MAP, tmp);
1103
1104         cgts_tcc_disable = 0xffff0000;
1105         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1106                 cgts_tcc_disable &= ~(1 << (16 + i));
1107         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1108         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1109         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1110         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1111
1112         /* reprogram the shader complex */
1113         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1114         for (i = 0; i < 16; i++)
1115                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1116         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1117
1118         /* set HW defaults for 3D engine */
1119         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1120
1121         sx_debug_1 = RREG32(SX_DEBUG_1);
1122         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1123         WREG32(SX_DEBUG_1, sx_debug_1);
1124
1125         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1126         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1127         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1128         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1129
1130         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1131
1132         /* need to be explicitly zero-ed */
1133         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1134         WREG32(SQ_LSTMP_RING_BASE, 0);
1135         WREG32(SQ_HSTMP_RING_BASE, 0);
1136         WREG32(SQ_ESTMP_RING_BASE, 0);
1137         WREG32(SQ_GSTMP_RING_BASE, 0);
1138         WREG32(SQ_VSTMP_RING_BASE, 0);
1139         WREG32(SQ_PSTMP_RING_BASE, 0);
1140
1141         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1142
1143         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1144                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1145                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1146
1147         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1148                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1149                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1150
1151
1152         WREG32(VGT_NUM_INSTANCES, 1);
1153
1154         WREG32(CP_PERFMON_CNTL, 0);
1155
1156         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1157                                   FETCH_FIFO_HIWATER(0x4) |
1158                                   DONE_FIFO_HIWATER(0xe0) |
1159                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1160
1161         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1162         WREG32(SQ_CONFIG, (VC_ENABLE |
1163                            EXPORT_SRC_C |
1164                            GFX_PRIO(0) |
1165                            CS1_PRIO(0) |
1166                            CS2_PRIO(1)));
1167         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1168
1169         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1170                                           FORCE_EOV_MAX_REZ_CNT(255)));
1171
1172         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1173                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1174
1175         WREG32(VGT_GS_VERTEX_REUSE, 16);
1176         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1177
1178         WREG32(CB_PERF_CTR0_SEL_0, 0);
1179         WREG32(CB_PERF_CTR0_SEL_1, 0);
1180         WREG32(CB_PERF_CTR1_SEL_0, 0);
1181         WREG32(CB_PERF_CTR1_SEL_1, 0);
1182         WREG32(CB_PERF_CTR2_SEL_0, 0);
1183         WREG32(CB_PERF_CTR2_SEL_1, 0);
1184         WREG32(CB_PERF_CTR3_SEL_0, 0);
1185         WREG32(CB_PERF_CTR3_SEL_1, 0);
1186
1187         tmp = RREG32(HDP_MISC_CNTL);
1188         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1189         WREG32(HDP_MISC_CNTL, tmp);
1190
1191         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1192         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1193
1194         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1195
1196         udelay(50);
1197
1198         /* set clockgating golden values on TN */
1199         if (rdev->family == CHIP_ARUBA) {
1200                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1201                 tmp &= ~0x00380000;
1202                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1203                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1204                 tmp &= ~0x0e000000;
1205                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1206         }
1207 }
1208
1209 /*
1210  * GART
1211  */
1212 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1213 {
1214         /* flush hdp cache */
1215         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1216
1217         /* bits 0-7 are the VM contexts0-7 */
1218         WREG32(VM_INVALIDATE_REQUEST, 1);
1219 }
1220
1221 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1222 {
1223         int i, r;
1224
1225         if (rdev->gart.robj == NULL) {
1226                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1227                 return -EINVAL;
1228         }
1229         r = radeon_gart_table_vram_pin(rdev);
1230         if (r)
1231                 return r;
1232         /* Setup TLB control */
1233         WREG32(MC_VM_MX_L1_TLB_CNTL,
1234                (0xA << 7) |
1235                ENABLE_L1_TLB |
1236                ENABLE_L1_FRAGMENT_PROCESSING |
1237                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1238                ENABLE_ADVANCED_DRIVER_MODEL |
1239                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1240         /* Setup L2 cache */
1241         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1242                ENABLE_L2_FRAGMENT_PROCESSING |
1243                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1244                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1245                EFFECTIVE_L2_QUEUE_SIZE(7) |
1246                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1247         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1248         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1249                BANK_SELECT(6) |
1250                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1251         /* setup context0 */
1252         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1253         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1254         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1255         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1256                         (u32)(rdev->dummy_page.addr >> 12));
1257         WREG32(VM_CONTEXT0_CNTL2, 0);
1258         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1259                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1260
1261         WREG32(0x15D4, 0);
1262         WREG32(0x15D8, 0);
1263         WREG32(0x15DC, 0);
1264
1265         /* empty context1-7 */
1266         /* Assign the pt base to something valid for now; the pts used for
1267          * the VMs are determined by the application and setup and assigned
1268          * on the fly in the vm part of radeon_gart.c
1269          */
1270         for (i = 1; i < 8; i++) {
1271                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1272                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1273                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1274                        rdev->vm_manager.saved_table_addr[i]);
1275         }
1276
1277         /* enable context1-7 */
1278         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1279                (u32)(rdev->dummy_page.addr >> 12));
1280         WREG32(VM_CONTEXT1_CNTL2, 4);
1281         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1282                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1283                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1284                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1285                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1286                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1287                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1288                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1289                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1290                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1291                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1292                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1293                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1294                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1295
1296         cayman_pcie_gart_tlb_flush(rdev);
1297         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1298                  (unsigned)(rdev->mc.gtt_size >> 20),
1299                  (unsigned long long)rdev->gart.table_addr);
1300         rdev->gart.ready = true;
1301         return 0;
1302 }
1303
1304 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1305 {
1306         unsigned i;
1307
1308         for (i = 1; i < 8; ++i) {
1309                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1310                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1311         }
1312
1313         /* Disable all tables */
1314         WREG32(VM_CONTEXT0_CNTL, 0);
1315         WREG32(VM_CONTEXT1_CNTL, 0);
1316         /* Setup TLB control */
1317         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1318                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1319                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1320         /* Setup L2 cache */
1321         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1322                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1323                EFFECTIVE_L2_QUEUE_SIZE(7) |
1324                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1325         WREG32(VM_L2_CNTL2, 0);
1326         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1327                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1328         radeon_gart_table_vram_unpin(rdev);
1329 }
1330
1331 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1332 {
1333         cayman_pcie_gart_disable(rdev);
1334         radeon_gart_table_vram_free(rdev);
1335         radeon_gart_fini(rdev);
1336 }
1337
1338 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1339                               int ring, u32 cp_int_cntl)
1340 {
1341         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1342
1343         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1344         WREG32(CP_INT_CNTL, cp_int_cntl);
1345 }
1346
1347 /*
1348  * CP.
1349  */
1350 void cayman_fence_ring_emit(struct radeon_device *rdev,
1351                             struct radeon_fence *fence)
1352 {
1353         struct radeon_ring *ring = &rdev->ring[fence->ring];
1354         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1355         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1356                 PACKET3_SH_ACTION_ENA;
1357
1358         /* flush read cache over gart for this vmid */
1359         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1360         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1361         radeon_ring_write(ring, 0xFFFFFFFF);
1362         radeon_ring_write(ring, 0);
1363         radeon_ring_write(ring, 10); /* poll interval */
1364         /* EVENT_WRITE_EOP - flush caches, send int */
1365         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1366         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1367         radeon_ring_write(ring, lower_32_bits(addr));
1368         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1369         radeon_ring_write(ring, fence->seq);
1370         radeon_ring_write(ring, 0);
1371 }
1372
1373 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1374 {
1375         struct radeon_ring *ring = &rdev->ring[ib->ring];
1376         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1377         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1378                 PACKET3_SH_ACTION_ENA;
1379
1380         /* set to DX10/11 mode */
1381         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1382         radeon_ring_write(ring, 1);
1383
1384         if (ring->rptr_save_reg) {
1385                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1386                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1387                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1388                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1389                 radeon_ring_write(ring, next_rptr);
1390         }
1391
1392         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1393         radeon_ring_write(ring,
1394 #ifdef __BIG_ENDIAN
1395                           (2 << 0) |
1396 #endif
1397                           (ib->gpu_addr & 0xFFFFFFFC));
1398         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1399         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1400
1401         /* flush read cache over gart for this vmid */
1402         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1403         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1404         radeon_ring_write(ring, 0xFFFFFFFF);
1405         radeon_ring_write(ring, 0);
1406         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1407 }
1408
1409 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1410 {
1411         if (enable)
1412                 WREG32(CP_ME_CNTL, 0);
1413         else {
1414                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1415                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1416                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1417                 WREG32(SCRATCH_UMSK, 0);
1418                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1419         }
1420 }
1421
1422 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1423                         struct radeon_ring *ring)
1424 {
1425         u32 rptr;
1426
1427         if (rdev->wb.enabled)
1428                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1429         else {
1430                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1431                         rptr = RREG32(CP_RB0_RPTR);
1432                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1433                         rptr = RREG32(CP_RB1_RPTR);
1434                 else
1435                         rptr = RREG32(CP_RB2_RPTR);
1436         }
1437
1438         return rptr;
1439 }
1440
1441 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1442                         struct radeon_ring *ring)
1443 {
1444         u32 wptr;
1445
1446         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1447                 wptr = RREG32(CP_RB0_WPTR);
1448         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1449                 wptr = RREG32(CP_RB1_WPTR);
1450         else
1451                 wptr = RREG32(CP_RB2_WPTR);
1452
1453         return wptr;
1454 }
1455
1456 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1457                          struct radeon_ring *ring)
1458 {
1459         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1460                 WREG32(CP_RB0_WPTR, ring->wptr);
1461                 (void)RREG32(CP_RB0_WPTR);
1462         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1463                 WREG32(CP_RB1_WPTR, ring->wptr);
1464                 (void)RREG32(CP_RB1_WPTR);
1465         } else {
1466                 WREG32(CP_RB2_WPTR, ring->wptr);
1467                 (void)RREG32(CP_RB2_WPTR);
1468         }
1469 }
1470
1471 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1472 {
1473         const __be32 *fw_data;
1474         int i;
1475
1476         if (!rdev->me_fw || !rdev->pfp_fw)
1477                 return -EINVAL;
1478
1479         cayman_cp_enable(rdev, false);
1480
1481         fw_data = (const __be32 *)rdev->pfp_fw->data;
1482         WREG32(CP_PFP_UCODE_ADDR, 0);
1483         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1484                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1485         WREG32(CP_PFP_UCODE_ADDR, 0);
1486
1487         fw_data = (const __be32 *)rdev->me_fw->data;
1488         WREG32(CP_ME_RAM_WADDR, 0);
1489         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1490                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1491
1492         WREG32(CP_PFP_UCODE_ADDR, 0);
1493         WREG32(CP_ME_RAM_WADDR, 0);
1494         WREG32(CP_ME_RAM_RADDR, 0);
1495         return 0;
1496 }
1497
1498 static int cayman_cp_start(struct radeon_device *rdev)
1499 {
1500         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1501         int r, i;
1502
1503         r = radeon_ring_lock(rdev, ring, 7);
1504         if (r) {
1505                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1506                 return r;
1507         }
1508         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1509         radeon_ring_write(ring, 0x1);
1510         radeon_ring_write(ring, 0x0);
1511         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1512         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1513         radeon_ring_write(ring, 0);
1514         radeon_ring_write(ring, 0);
1515         radeon_ring_unlock_commit(rdev, ring, false);
1516
1517         cayman_cp_enable(rdev, true);
1518
1519         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1520         if (r) {
1521                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1522                 return r;
1523         }
1524
1525         /* setup clear context state */
1526         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1527         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1528
1529         for (i = 0; i < cayman_default_size; i++)
1530                 radeon_ring_write(ring, cayman_default_state[i]);
1531
1532         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1533         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1534
1535         /* set clear context state */
1536         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1537         radeon_ring_write(ring, 0);
1538
1539         /* SQ_VTX_BASE_VTX_LOC */
1540         radeon_ring_write(ring, 0xc0026f00);
1541         radeon_ring_write(ring, 0x00000000);
1542         radeon_ring_write(ring, 0x00000000);
1543         radeon_ring_write(ring, 0x00000000);
1544
1545         /* Clear consts */
1546         radeon_ring_write(ring, 0xc0036f00);
1547         radeon_ring_write(ring, 0x00000bc4);
1548         radeon_ring_write(ring, 0xffffffff);
1549         radeon_ring_write(ring, 0xffffffff);
1550         radeon_ring_write(ring, 0xffffffff);
1551
1552         radeon_ring_write(ring, 0xc0026900);
1553         radeon_ring_write(ring, 0x00000316);
1554         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1555         radeon_ring_write(ring, 0x00000010); /*  */
1556
1557         radeon_ring_unlock_commit(rdev, ring, false);
1558
1559         /* XXX init other rings */
1560
1561         return 0;
1562 }
1563
1564 static void cayman_cp_fini(struct radeon_device *rdev)
1565 {
1566         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1567         cayman_cp_enable(rdev, false);
1568         radeon_ring_fini(rdev, ring);
1569         radeon_scratch_free(rdev, ring->rptr_save_reg);
1570 }
1571
1572 static int cayman_cp_resume(struct radeon_device *rdev)
1573 {
1574         static const int ridx[] = {
1575                 RADEON_RING_TYPE_GFX_INDEX,
1576                 CAYMAN_RING_TYPE_CP1_INDEX,
1577                 CAYMAN_RING_TYPE_CP2_INDEX
1578         };
1579         static const unsigned cp_rb_cntl[] = {
1580                 CP_RB0_CNTL,
1581                 CP_RB1_CNTL,
1582                 CP_RB2_CNTL,
1583         };
1584         static const unsigned cp_rb_rptr_addr[] = {
1585                 CP_RB0_RPTR_ADDR,
1586                 CP_RB1_RPTR_ADDR,
1587                 CP_RB2_RPTR_ADDR
1588         };
1589         static const unsigned cp_rb_rptr_addr_hi[] = {
1590                 CP_RB0_RPTR_ADDR_HI,
1591                 CP_RB1_RPTR_ADDR_HI,
1592                 CP_RB2_RPTR_ADDR_HI
1593         };
1594         static const unsigned cp_rb_base[] = {
1595                 CP_RB0_BASE,
1596                 CP_RB1_BASE,
1597                 CP_RB2_BASE
1598         };
1599         static const unsigned cp_rb_rptr[] = {
1600                 CP_RB0_RPTR,
1601                 CP_RB1_RPTR,
1602                 CP_RB2_RPTR
1603         };
1604         static const unsigned cp_rb_wptr[] = {
1605                 CP_RB0_WPTR,
1606                 CP_RB1_WPTR,
1607                 CP_RB2_WPTR
1608         };
1609         struct radeon_ring *ring;
1610         int i, r;
1611
1612         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1613         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1614                                  SOFT_RESET_PA |
1615                                  SOFT_RESET_SH |
1616                                  SOFT_RESET_VGT |
1617                                  SOFT_RESET_SPI |
1618                                  SOFT_RESET_SX));
1619         RREG32(GRBM_SOFT_RESET);
1620         mdelay(15);
1621         WREG32(GRBM_SOFT_RESET, 0);
1622         RREG32(GRBM_SOFT_RESET);
1623
1624         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1625         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1626
1627         /* Set the write pointer delay */
1628         WREG32(CP_RB_WPTR_DELAY, 0);
1629
1630         WREG32(CP_DEBUG, (1 << 27));
1631
1632         /* set the wb address whether it's enabled or not */
1633         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1634         WREG32(SCRATCH_UMSK, 0xff);
1635
1636         for (i = 0; i < 3; ++i) {
1637                 uint32_t rb_cntl;
1638                 uint64_t addr;
1639
1640                 /* Set ring buffer size */
1641                 ring = &rdev->ring[ridx[i]];
1642                 rb_cntl = order_base_2(ring->ring_size / 8);
1643                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1644 #ifdef __BIG_ENDIAN
1645                 rb_cntl |= BUF_SWAP_32BIT;
1646 #endif
1647                 WREG32(cp_rb_cntl[i], rb_cntl);
1648
1649                 /* set the wb address whether it's enabled or not */
1650                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1651                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1652                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1653         }
1654
1655         /* set the rb base addr, this causes an internal reset of ALL rings */
1656         for (i = 0; i < 3; ++i) {
1657                 ring = &rdev->ring[ridx[i]];
1658                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1659         }
1660
1661         for (i = 0; i < 3; ++i) {
1662                 /* Initialize the ring buffer's read and write pointers */
1663                 ring = &rdev->ring[ridx[i]];
1664                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1665
1666                 ring->wptr = 0;
1667                 WREG32(cp_rb_rptr[i], 0);
1668                 WREG32(cp_rb_wptr[i], ring->wptr);
1669
1670                 mdelay(1);
1671                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1672         }
1673
1674         /* start the rings */
1675         cayman_cp_start(rdev);
1676         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1677         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1678         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1679         /* this only test cp0 */
1680         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1681         if (r) {
1682                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1683                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1684                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1685                 return r;
1686         }
1687
1688         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1689                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1690
1691         return 0;
1692 }
1693
1694 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1695 {
1696         u32 reset_mask = 0;
1697         u32 tmp;
1698
1699         /* GRBM_STATUS */
1700         tmp = RREG32(GRBM_STATUS);
1701         if (tmp & (PA_BUSY | SC_BUSY |
1702                    SH_BUSY | SX_BUSY |
1703                    TA_BUSY | VGT_BUSY |
1704                    DB_BUSY | CB_BUSY |
1705                    GDS_BUSY | SPI_BUSY |
1706                    IA_BUSY | IA_BUSY_NO_DMA))
1707                 reset_mask |= RADEON_RESET_GFX;
1708
1709         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1710                    CP_BUSY | CP_COHERENCY_BUSY))
1711                 reset_mask |= RADEON_RESET_CP;
1712
1713         if (tmp & GRBM_EE_BUSY)
1714                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1715
1716         /* DMA_STATUS_REG 0 */
1717         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1718         if (!(tmp & DMA_IDLE))
1719                 reset_mask |= RADEON_RESET_DMA;
1720
1721         /* DMA_STATUS_REG 1 */
1722         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1723         if (!(tmp & DMA_IDLE))
1724                 reset_mask |= RADEON_RESET_DMA1;
1725
1726         /* SRBM_STATUS2 */
1727         tmp = RREG32(SRBM_STATUS2);
1728         if (tmp & DMA_BUSY)
1729                 reset_mask |= RADEON_RESET_DMA;
1730
1731         if (tmp & DMA1_BUSY)
1732                 reset_mask |= RADEON_RESET_DMA1;
1733
1734         /* SRBM_STATUS */
1735         tmp = RREG32(SRBM_STATUS);
1736         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1737                 reset_mask |= RADEON_RESET_RLC;
1738
1739         if (tmp & IH_BUSY)
1740                 reset_mask |= RADEON_RESET_IH;
1741
1742         if (tmp & SEM_BUSY)
1743                 reset_mask |= RADEON_RESET_SEM;
1744
1745         if (tmp & GRBM_RQ_PENDING)
1746                 reset_mask |= RADEON_RESET_GRBM;
1747
1748         if (tmp & VMC_BUSY)
1749                 reset_mask |= RADEON_RESET_VMC;
1750
1751         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1752                    MCC_BUSY | MCD_BUSY))
1753                 reset_mask |= RADEON_RESET_MC;
1754
1755         if (evergreen_is_display_hung(rdev))
1756                 reset_mask |= RADEON_RESET_DISPLAY;
1757
1758         /* VM_L2_STATUS */
1759         tmp = RREG32(VM_L2_STATUS);
1760         if (tmp & L2_BUSY)
1761                 reset_mask |= RADEON_RESET_VMC;
1762
1763         /* Skip MC reset as it's mostly likely not hung, just busy */
1764         if (reset_mask & RADEON_RESET_MC) {
1765                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1766                 reset_mask &= ~RADEON_RESET_MC;
1767         }
1768
1769         return reset_mask;
1770 }
1771
1772 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1773 {
1774         struct evergreen_mc_save save;
1775         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1776         u32 tmp;
1777
1778         if (reset_mask == 0)
1779                 return;
1780
1781         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1782
1783         evergreen_print_gpu_status_regs(rdev);
1784         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1785                  RREG32(0x14F8));
1786         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1787                  RREG32(0x14D8));
1788         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1789                  RREG32(0x14FC));
1790         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1791                  RREG32(0x14DC));
1792
1793         /* Disable CP parsing/prefetching */
1794         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1795
1796         if (reset_mask & RADEON_RESET_DMA) {
1797                 /* dma0 */
1798                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1799                 tmp &= ~DMA_RB_ENABLE;
1800                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1801         }
1802
1803         if (reset_mask & RADEON_RESET_DMA1) {
1804                 /* dma1 */
1805                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1806                 tmp &= ~DMA_RB_ENABLE;
1807                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1808         }
1809
1810         udelay(50);
1811
1812         evergreen_mc_stop(rdev, &save);
1813         if (evergreen_mc_wait_for_idle(rdev)) {
1814                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1815         }
1816
1817         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1818                 grbm_soft_reset = SOFT_RESET_CB |
1819                         SOFT_RESET_DB |
1820                         SOFT_RESET_GDS |
1821                         SOFT_RESET_PA |
1822                         SOFT_RESET_SC |
1823                         SOFT_RESET_SPI |
1824                         SOFT_RESET_SH |
1825                         SOFT_RESET_SX |
1826                         SOFT_RESET_TC |
1827                         SOFT_RESET_TA |
1828                         SOFT_RESET_VGT |
1829                         SOFT_RESET_IA;
1830         }
1831
1832         if (reset_mask & RADEON_RESET_CP) {
1833                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1834
1835                 srbm_soft_reset |= SOFT_RESET_GRBM;
1836         }
1837
1838         if (reset_mask & RADEON_RESET_DMA)
1839                 srbm_soft_reset |= SOFT_RESET_DMA;
1840
1841         if (reset_mask & RADEON_RESET_DMA1)
1842                 srbm_soft_reset |= SOFT_RESET_DMA1;
1843
1844         if (reset_mask & RADEON_RESET_DISPLAY)
1845                 srbm_soft_reset |= SOFT_RESET_DC;
1846
1847         if (reset_mask & RADEON_RESET_RLC)
1848                 srbm_soft_reset |= SOFT_RESET_RLC;
1849
1850         if (reset_mask & RADEON_RESET_SEM)
1851                 srbm_soft_reset |= SOFT_RESET_SEM;
1852
1853         if (reset_mask & RADEON_RESET_IH)
1854                 srbm_soft_reset |= SOFT_RESET_IH;
1855
1856         if (reset_mask & RADEON_RESET_GRBM)
1857                 srbm_soft_reset |= SOFT_RESET_GRBM;
1858
1859         if (reset_mask & RADEON_RESET_VMC)
1860                 srbm_soft_reset |= SOFT_RESET_VMC;
1861
1862         if (!(rdev->flags & RADEON_IS_IGP)) {
1863                 if (reset_mask & RADEON_RESET_MC)
1864                         srbm_soft_reset |= SOFT_RESET_MC;
1865         }
1866
1867         if (grbm_soft_reset) {
1868                 tmp = RREG32(GRBM_SOFT_RESET);
1869                 tmp |= grbm_soft_reset;
1870                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1871                 WREG32(GRBM_SOFT_RESET, tmp);
1872                 tmp = RREG32(GRBM_SOFT_RESET);
1873
1874                 udelay(50);
1875
1876                 tmp &= ~grbm_soft_reset;
1877                 WREG32(GRBM_SOFT_RESET, tmp);
1878                 tmp = RREG32(GRBM_SOFT_RESET);
1879         }
1880
1881         if (srbm_soft_reset) {
1882                 tmp = RREG32(SRBM_SOFT_RESET);
1883                 tmp |= srbm_soft_reset;
1884                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1885                 WREG32(SRBM_SOFT_RESET, tmp);
1886                 tmp = RREG32(SRBM_SOFT_RESET);
1887
1888                 udelay(50);
1889
1890                 tmp &= ~srbm_soft_reset;
1891                 WREG32(SRBM_SOFT_RESET, tmp);
1892                 tmp = RREG32(SRBM_SOFT_RESET);
1893         }
1894
1895         /* Wait a little for things to settle down */
1896         udelay(50);
1897
1898         evergreen_mc_resume(rdev, &save);
1899         udelay(50);
1900
1901         evergreen_print_gpu_status_regs(rdev);
1902 }
1903
1904 int cayman_asic_reset(struct radeon_device *rdev)
1905 {
1906         u32 reset_mask;
1907
1908         reset_mask = cayman_gpu_check_soft_reset(rdev);
1909
1910         if (reset_mask)
1911                 r600_set_bios_scratch_engine_hung(rdev, true);
1912
1913         cayman_gpu_soft_reset(rdev, reset_mask);
1914
1915         reset_mask = cayman_gpu_check_soft_reset(rdev);
1916
1917         if (reset_mask)
1918                 evergreen_gpu_pci_config_reset(rdev);
1919
1920         r600_set_bios_scratch_engine_hung(rdev, false);
1921
1922         return 0;
1923 }
1924
1925 /**
1926  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1927  *
1928  * @rdev: radeon_device pointer
1929  * @ring: radeon_ring structure holding ring information
1930  *
1931  * Check if the GFX engine is locked up.
1932  * Returns true if the engine appears to be locked up, false if not.
1933  */
1934 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1935 {
1936         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1937
1938         if (!(reset_mask & (RADEON_RESET_GFX |
1939                             RADEON_RESET_COMPUTE |
1940                             RADEON_RESET_CP))) {
1941                 radeon_ring_lockup_update(rdev, ring);
1942                 return false;
1943         }
1944         return radeon_ring_test_lockup(rdev, ring);
1945 }
1946
1947 static int cayman_startup(struct radeon_device *rdev)
1948 {
1949         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1950         int r;
1951
1952         /* enable pcie gen2 link */
1953         evergreen_pcie_gen2_enable(rdev);
1954         /* enable aspm */
1955         evergreen_program_aspm(rdev);
1956
1957         /* scratch needs to be initialized before MC */
1958         r = r600_vram_scratch_init(rdev);
1959         if (r)
1960                 return r;
1961
1962         evergreen_mc_program(rdev);
1963
1964         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1965                 r = ni_mc_load_microcode(rdev);
1966                 if (r) {
1967                         DRM_ERROR("Failed to load MC firmware!\n");
1968                         return r;
1969                 }
1970         }
1971
1972         r = cayman_pcie_gart_enable(rdev);
1973         if (r)
1974                 return r;
1975         cayman_gpu_init(rdev);
1976
1977         /* allocate rlc buffers */
1978         if (rdev->flags & RADEON_IS_IGP) {
1979                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1980                 rdev->rlc.reg_list_size =
1981                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1982                 rdev->rlc.cs_data = cayman_cs_data;
1983                 r = sumo_rlc_init(rdev);
1984                 if (r) {
1985                         DRM_ERROR("Failed to init rlc BOs!\n");
1986                         return r;
1987                 }
1988         }
1989
1990         /* allocate wb buffer */
1991         r = radeon_wb_init(rdev);
1992         if (r)
1993                 return r;
1994
1995         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1996         if (r) {
1997                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1998                 return r;
1999         }
2000
2001         r = uvd_v2_2_resume(rdev);
2002         if (!r) {
2003                 r = radeon_fence_driver_start_ring(rdev,
2004                                                    R600_RING_TYPE_UVD_INDEX);
2005                 if (r)
2006                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2007         }
2008         if (r)
2009                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2010
2011         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2012         if (r) {
2013                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2014                 return r;
2015         }
2016
2017         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2018         if (r) {
2019                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2020                 return r;
2021         }
2022
2023         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2024         if (r) {
2025                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2026                 return r;
2027         }
2028
2029         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2030         if (r) {
2031                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2032                 return r;
2033         }
2034
2035         /* Enable IRQ */
2036         if (!rdev->irq.installed) {
2037                 r = radeon_irq_kms_init(rdev);
2038                 if (r)
2039                         return r;
2040         }
2041
2042         r = r600_irq_init(rdev);
2043         if (r) {
2044                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2045                 radeon_irq_kms_fini(rdev);
2046                 return r;
2047         }
2048         evergreen_irq_set(rdev);
2049
2050         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2051                              RADEON_CP_PACKET2);
2052         if (r)
2053                 return r;
2054
2055         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2056         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2057                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2058         if (r)
2059                 return r;
2060
2061         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2062         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2063                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2064         if (r)
2065                 return r;
2066
2067         r = cayman_cp_load_microcode(rdev);
2068         if (r)
2069                 return r;
2070         r = cayman_cp_resume(rdev);
2071         if (r)
2072                 return r;
2073
2074         r = cayman_dma_resume(rdev);
2075         if (r)
2076                 return r;
2077
2078         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2079         if (ring->ring_size) {
2080                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2081                                      RADEON_CP_PACKET2);
2082                 if (!r)
2083                         r = uvd_v1_0_init(rdev);
2084                 if (r)
2085                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2086         }
2087
2088         r = radeon_ib_pool_init(rdev);
2089         if (r) {
2090                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2091                 return r;
2092         }
2093
2094         r = radeon_vm_manager_init(rdev);
2095         if (r) {
2096                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2097                 return r;
2098         }
2099
2100         if (ASIC_IS_DCE6(rdev)) {
2101                 r = dce6_audio_init(rdev);
2102                 if (r)
2103                         return r;
2104         } else {
2105                 r = r600_audio_init(rdev);
2106                 if (r)
2107                         return r;
2108         }
2109
2110         return 0;
2111 }
2112
2113 int cayman_resume(struct radeon_device *rdev)
2114 {
2115         int r;
2116
2117         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2118          * posting will perform necessary task to bring back GPU into good
2119          * shape.
2120          */
2121         /* post card */
2122         atom_asic_init(rdev->mode_info.atom_context);
2123
2124         /* init golden registers */
2125         ni_init_golden_registers(rdev);
2126
2127         if (rdev->pm.pm_method == PM_METHOD_DPM)
2128                 radeon_pm_resume(rdev);
2129
2130         rdev->accel_working = true;
2131         r = cayman_startup(rdev);
2132         if (r) {
2133                 DRM_ERROR("cayman startup failed on resume\n");
2134                 rdev->accel_working = false;
2135                 return r;
2136         }
2137         return r;
2138 }
2139
2140 int cayman_suspend(struct radeon_device *rdev)
2141 {
2142         radeon_pm_suspend(rdev);
2143         if (ASIC_IS_DCE6(rdev))
2144                 dce6_audio_fini(rdev);
2145         else
2146                 r600_audio_fini(rdev);
2147         radeon_vm_manager_fini(rdev);
2148         cayman_cp_enable(rdev, false);
2149         cayman_dma_stop(rdev);
2150         uvd_v1_0_fini(rdev);
2151         radeon_uvd_suspend(rdev);
2152         evergreen_irq_suspend(rdev);
2153         radeon_wb_disable(rdev);
2154         cayman_pcie_gart_disable(rdev);
2155         return 0;
2156 }
2157
2158 /* Plan is to move initialization in that function and use
2159  * helper function so that radeon_device_init pretty much
2160  * do nothing more than calling asic specific function. This
2161  * should also allow to remove a bunch of callback function
2162  * like vram_info.
2163  */
2164 int cayman_init(struct radeon_device *rdev)
2165 {
2166         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2167         int r;
2168
2169         /* Read BIOS */
2170         if (!radeon_get_bios(rdev)) {
2171                 if (ASIC_IS_AVIVO(rdev))
2172                         return -EINVAL;
2173         }
2174         /* Must be an ATOMBIOS */
2175         if (!rdev->is_atom_bios) {
2176                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2177                 return -EINVAL;
2178         }
2179         r = radeon_atombios_init(rdev);
2180         if (r)
2181                 return r;
2182
2183         /* Post card if necessary */
2184         if (!radeon_card_posted(rdev)) {
2185                 if (!rdev->bios) {
2186                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2187                         return -EINVAL;
2188                 }
2189                 DRM_INFO("GPU not posted. posting now...\n");
2190                 atom_asic_init(rdev->mode_info.atom_context);
2191         }
2192         /* init golden registers */
2193         ni_init_golden_registers(rdev);
2194         /* Initialize scratch registers */
2195         r600_scratch_init(rdev);
2196         /* Initialize surface registers */
2197         radeon_surface_init(rdev);
2198         /* Initialize clocks */
2199         radeon_get_clock_info(rdev->ddev);
2200         /* Fence driver */
2201         r = radeon_fence_driver_init(rdev);
2202         if (r)
2203                 return r;
2204         /* initialize memory controller */
2205         r = evergreen_mc_init(rdev);
2206         if (r)
2207                 return r;
2208         /* Memory manager */
2209         r = radeon_bo_init(rdev);
2210         if (r)
2211                 return r;
2212
2213         if (rdev->flags & RADEON_IS_IGP) {
2214                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2215                         r = ni_init_microcode(rdev);
2216                         if (r) {
2217                                 DRM_ERROR("Failed to load firmware!\n");
2218                                 return r;
2219                         }
2220                 }
2221         } else {
2222                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2223                         r = ni_init_microcode(rdev);
2224                         if (r) {
2225                                 DRM_ERROR("Failed to load firmware!\n");
2226                                 return r;
2227                         }
2228                 }
2229         }
2230
2231         /* Initialize power management */
2232         radeon_pm_init(rdev);
2233
2234         ring->ring_obj = NULL;
2235         r600_ring_init(rdev, ring, 1024 * 1024);
2236
2237         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2238         ring->ring_obj = NULL;
2239         r600_ring_init(rdev, ring, 64 * 1024);
2240
2241         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2242         ring->ring_obj = NULL;
2243         r600_ring_init(rdev, ring, 64 * 1024);
2244
2245         r = radeon_uvd_init(rdev);
2246         if (!r) {
2247                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2248                 ring->ring_obj = NULL;
2249                 r600_ring_init(rdev, ring, 4096);
2250         }
2251
2252         rdev->ih.ring_obj = NULL;
2253         r600_ih_ring_init(rdev, 64 * 1024);
2254
2255         r = r600_pcie_gart_init(rdev);
2256         if (r)
2257                 return r;
2258
2259         rdev->accel_working = true;
2260         r = cayman_startup(rdev);
2261         if (r) {
2262                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2263                 cayman_cp_fini(rdev);
2264                 cayman_dma_fini(rdev);
2265                 r600_irq_fini(rdev);
2266                 if (rdev->flags & RADEON_IS_IGP)
2267                         sumo_rlc_fini(rdev);
2268                 radeon_wb_fini(rdev);
2269                 radeon_ib_pool_fini(rdev);
2270                 radeon_vm_manager_fini(rdev);
2271                 radeon_irq_kms_fini(rdev);
2272                 cayman_pcie_gart_fini(rdev);
2273                 rdev->accel_working = false;
2274         }
2275
2276         /* Don't start up if the MC ucode is missing.
2277          * The default clocks and voltages before the MC ucode
2278          * is loaded are not suffient for advanced operations.
2279          *
2280          * We can skip this check for TN, because there is no MC
2281          * ucode.
2282          */
2283         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2284                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2285                 return -EINVAL;
2286         }
2287
2288         return 0;
2289 }
2290
2291 void cayman_fini(struct radeon_device *rdev)
2292 {
2293         radeon_pm_fini(rdev);
2294         cayman_cp_fini(rdev);
2295         cayman_dma_fini(rdev);
2296         r600_irq_fini(rdev);
2297         if (rdev->flags & RADEON_IS_IGP)
2298                 sumo_rlc_fini(rdev);
2299         radeon_wb_fini(rdev);
2300         radeon_vm_manager_fini(rdev);
2301         radeon_ib_pool_fini(rdev);
2302         radeon_irq_kms_fini(rdev);
2303         uvd_v1_0_fini(rdev);
2304         radeon_uvd_fini(rdev);
2305         cayman_pcie_gart_fini(rdev);
2306         r600_vram_scratch_fini(rdev);
2307         radeon_gem_fini(rdev);
2308         radeon_fence_driver_fini(rdev);
2309         radeon_bo_fini(rdev);
2310         radeon_atombios_fini(rdev);
2311         kfree(rdev->bios);
2312         rdev->bios = NULL;
2313 }
2314
2315 /*
2316  * vm
2317  */
2318 int cayman_vm_init(struct radeon_device *rdev)
2319 {
2320         /* number of VMs */
2321         rdev->vm_manager.nvm = 8;
2322         /* base offset of vram pages */
2323         if (rdev->flags & RADEON_IS_IGP) {
2324                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2325                 tmp <<= 22;
2326                 rdev->vm_manager.vram_base_offset = tmp;
2327         } else
2328                 rdev->vm_manager.vram_base_offset = 0;
2329         return 0;
2330 }
2331
2332 void cayman_vm_fini(struct radeon_device *rdev)
2333 {
2334 }
2335
2336 /**
2337  * cayman_vm_decode_fault - print human readable fault info
2338  *
2339  * @rdev: radeon_device pointer
2340  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2341  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2342  *
2343  * Print human readable fault information (cayman/TN).
2344  */
2345 void cayman_vm_decode_fault(struct radeon_device *rdev,
2346                             u32 status, u32 addr)
2347 {
2348         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2349         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2350         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2351         char *block;
2352
2353         switch (mc_id) {
2354         case 32:
2355         case 16:
2356         case 96:
2357         case 80:
2358         case 160:
2359         case 144:
2360         case 224:
2361         case 208:
2362                 block = "CB";
2363                 break;
2364         case 33:
2365         case 17:
2366         case 97:
2367         case 81:
2368         case 161:
2369         case 145:
2370         case 225:
2371         case 209:
2372                 block = "CB_FMASK";
2373                 break;
2374         case 34:
2375         case 18:
2376         case 98:
2377         case 82:
2378         case 162:
2379         case 146:
2380         case 226:
2381         case 210:
2382                 block = "CB_CMASK";
2383                 break;
2384         case 35:
2385         case 19:
2386         case 99:
2387         case 83:
2388         case 163:
2389         case 147:
2390         case 227:
2391         case 211:
2392                 block = "CB_IMMED";
2393                 break;
2394         case 36:
2395         case 20:
2396         case 100:
2397         case 84:
2398         case 164:
2399         case 148:
2400         case 228:
2401         case 212:
2402                 block = "DB";
2403                 break;
2404         case 37:
2405         case 21:
2406         case 101:
2407         case 85:
2408         case 165:
2409         case 149:
2410         case 229:
2411         case 213:
2412                 block = "DB_HTILE";
2413                 break;
2414         case 38:
2415         case 22:
2416         case 102:
2417         case 86:
2418         case 166:
2419         case 150:
2420         case 230:
2421         case 214:
2422                 block = "SX";
2423                 break;
2424         case 39:
2425         case 23:
2426         case 103:
2427         case 87:
2428         case 167:
2429         case 151:
2430         case 231:
2431         case 215:
2432                 block = "DB_STEN";
2433                 break;
2434         case 40:
2435         case 24:
2436         case 104:
2437         case 88:
2438         case 232:
2439         case 216:
2440         case 168:
2441         case 152:
2442                 block = "TC_TFETCH";
2443                 break;
2444         case 41:
2445         case 25:
2446         case 105:
2447         case 89:
2448         case 233:
2449         case 217:
2450         case 169:
2451         case 153:
2452                 block = "TC_VFETCH";
2453                 break;
2454         case 42:
2455         case 26:
2456         case 106:
2457         case 90:
2458         case 234:
2459         case 218:
2460         case 170:
2461         case 154:
2462                 block = "VC";
2463                 break;
2464         case 112:
2465                 block = "CP";
2466                 break;
2467         case 113:
2468         case 114:
2469                 block = "SH";
2470                 break;
2471         case 115:
2472                 block = "VGT";
2473                 break;
2474         case 178:
2475                 block = "IH";
2476                 break;
2477         case 51:
2478                 block = "RLC";
2479                 break;
2480         case 55:
2481                 block = "DMA";
2482                 break;
2483         case 56:
2484                 block = "HDP";
2485                 break;
2486         default:
2487                 block = "unknown";
2488                 break;
2489         }
2490
2491         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2492                protections, vmid, addr,
2493                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2494                block, mc_id);
2495 }
2496
2497 /**
2498  * cayman_vm_flush - vm flush using the CP
2499  *
2500  * @rdev: radeon_device pointer
2501  *
2502  * Update the page table base and flush the VM TLB
2503  * using the CP (cayman-si).
2504  */
2505 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2506                      unsigned vm_id, uint64_t pd_addr)
2507 {
2508         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2509         radeon_ring_write(ring, pd_addr >> 12);
2510
2511         /* flush hdp cache */
2512         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2513         radeon_ring_write(ring, 0x1);
2514
2515         /* bits 0-7 are the VM contexts0-7 */
2516         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2517         radeon_ring_write(ring, 1 << vm_id);
2518
2519         /* wait for the invalidate to complete */
2520         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2521         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2522                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2523         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2524         radeon_ring_write(ring, 0);
2525         radeon_ring_write(ring, 0); /* ref */
2526         radeon_ring_write(ring, 0); /* mask */
2527         radeon_ring_write(ring, 0x20); /* poll interval */
2528
2529         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2530         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2531         radeon_ring_write(ring, 0x0);
2532 }