drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "nid.h"
  32 #include "atom.h"
  33 #include "ni_reg.h"
  34 #include "cayman_blit_shaders.h"
  35 #include "radeon_ucode.h"
  36 #include "clearstate_cayman.h"
  37
  38 static const u32 tn_rlc_save_restore_register_list[] =
  39 {
  40         0x98fc,
  41         0x98f0,
  42         0x9834,
  43         0x9838,
  44         0x9870,
  45         0x9874,
  46         0x8a14,
  47         0x8b24,
  48         0x8bcc,
  49         0x8b10,
  50         0x8c30,
  51         0x8d00,
  52         0x8d04,
  53         0x8c00,
  54         0x8c04,
  55         0x8c10,
  56         0x8c14,
  57         0x8d8c,
  58         0x8cf0,
  59         0x8e38,
  60         0x9508,
  61         0x9688,
  62         0x9608,
  63         0x960c,
  64         0x9610,
  65         0x9614,
  66         0x88c4,
  67         0x8978,
  68         0x88d4,
  69         0x900c,
  70         0x9100,
  71         0x913c,
  72         0x90e8,
  73         0x9354,
  74         0xa008,
  75         0x98f8,
  76         0x9148,
  77         0x914c,
  78         0x3f94,
  79         0x98f4,
  80         0x9b7c,
  81         0x3f8c,
  82         0x8950,
  83         0x8954,
  84         0x8a18,
  85         0x8b28,
  86         0x9144,
  87         0x3f90,
  88         0x915c,
  89         0x9160,
  90         0x9178,
  91         0x917c,
  92         0x9180,
  93         0x918c,
  94         0x9190,
  95         0x9194,
  96         0x9198,
  97         0x919c,
  98         0x91a8,
  99         0x91ac,
 100         0x91b0,
 101         0x91b4,
 102         0x91b8,
 103         0x91c4,
 104         0x91c8,
 105         0x91cc,
 106         0x91d0,
 107         0x91d4,
 108         0x91e0,
 109         0x91e4,
 110         0x91ec,
 111         0x91f0,
 112         0x91f4,
 113         0x9200,
 114         0x9204,
 115         0x929c,
 116         0x8030,
 117         0x9150,
 118         0x9a60,
 119         0x920c,
 120         0x9210,
 121         0x9228,
 122         0x922c,
 123         0x9244,
 124         0x9248,
 125         0x91e8,
 126         0x9294,
 127         0x9208,
 128         0x9224,
 129         0x9240,
 130         0x9220,
 131         0x923c,
 132         0x9258,
 133         0x9744,
 134         0xa200,
 135         0xa204,
 136         0xa208,
 137         0xa20c,
 138         0x8d58,
 139         0x9030,
 140         0x9034,
 141         0x9038,
 142         0x903c,
 143         0x9040,
 144         0x9654,
 145         0x897c,
 146         0xa210,
 147         0xa214,
 148         0x9868,
 149         0xa02c,
 150         0x9664,
 151         0x9698,
 152         0x949c,
 153         0x8e10,
 154         0x8e18,
 155         0x8c50,
 156         0x8c58,
 157         0x8c60,
 158         0x8c68,
 159         0x89b4,
 160         0x9830,
 161         0x802c,
 162 };
 163
 164 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 165 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 166 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 167 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 168 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 169 extern void evergreen_mc_program(struct radeon_device *rdev);
 170 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 171 extern int evergreen_mc_init(struct radeon_device *rdev);
 172 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 173 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 174 extern void evergreen_program_aspm(struct radeon_device *rdev);
 175 extern void sumo_rlc_fini(struct radeon_device *rdev);
 176 extern int sumo_rlc_init(struct radeon_device *rdev);
 177
 178 /* Firmware Names */
 179 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 180 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 181 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 183 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 184 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 185 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 188 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 189 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 192 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 193 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 197 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 198 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 200
 201
 202 static const u32 cayman_golden_registers2[] =
 203 {
 204         0x3e5c, 0xffffffff, 0x00000000,
 205         0x3e48, 0xffffffff, 0x00000000,
 206         0x3e4c, 0xffffffff, 0x00000000,
 207         0x3e64, 0xffffffff, 0x00000000,
 208         0x3e50, 0xffffffff, 0x00000000,
 209         0x3e60, 0xffffffff, 0x00000000
 210 };
 211
 212 static const u32 cayman_golden_registers[] =
 213 {
 214         0x5eb4, 0xffffffff, 0x00000002,
 215         0x5e78, 0x8f311ff1, 0x001000f0,
 216         0x3f90, 0xffff0000, 0xff000000,
 217         0x9148, 0xffff0000, 0xff000000,
 218         0x3f94, 0xffff0000, 0xff000000,
 219         0x914c, 0xffff0000, 0xff000000,
 220         0xc78, 0x00000080, 0x00000080,
 221         0xbd4, 0x70073777, 0x00011003,
 222         0xd02c, 0xbfffff1f, 0x08421000,
 223         0xd0b8, 0x73773777, 0x02011003,
 224         0x5bc0, 0x00200000, 0x50100000,
 225         0x98f8, 0x33773777, 0x02011003,
 226         0x98fc, 0xffffffff, 0x76541032,
 227         0x7030, 0x31000311, 0x00000011,
 228         0x2f48, 0x33773777, 0x42010001,
 229         0x6b28, 0x00000010, 0x00000012,
 230         0x7728, 0x00000010, 0x00000012,
 231         0x10328, 0x00000010, 0x00000012,
 232         0x10f28, 0x00000010, 0x00000012,
 233         0x11b28, 0x00000010, 0x00000012,
 234         0x12728, 0x00000010, 0x00000012,
 235         0x240c, 0x000007ff, 0x00000000,
 236         0x8a14, 0xf000001f, 0x00000007,
 237         0x8b24, 0x3fff3fff, 0x00ff0fff,
 238         0x8b10, 0x0000ff0f, 0x00000000,
 239         0x28a4c, 0x07ffffff, 0x06000000,
 240         0x10c, 0x00000001, 0x00010003,
 241         0xa02c, 0xffffffff, 0x0000009b,
 242         0x913c, 0x0000010f, 0x01000100,
 243         0x8c04, 0xf8ff00ff, 0x40600060,
 244         0x28350, 0x00000f01, 0x00000000,
 245         0x9508, 0x3700001f, 0x00000002,
 246         0x960c, 0xffffffff, 0x54763210,
 247         0x88c4, 0x001f3ae3, 0x00000082,
 248         0x88d0, 0xffffffff, 0x0f40df40,
 249         0x88d4, 0x0000001f, 0x00000010,
 250         0x8974, 0xffffffff, 0x00000000
 251 };
 252
 253 static const u32 dvst_golden_registers2[] =
 254 {
 255         0x8f8, 0xffffffff, 0,
 256         0x8fc, 0x00380000, 0,
 257         0x8f8, 0xffffffff, 1,
 258         0x8fc, 0x0e000000, 0
 259 };
 260
 261 static const u32 dvst_golden_registers[] =
 262 {
 263         0x690, 0x3fff3fff, 0x20c00033,
 264         0x918c, 0x0fff0fff, 0x00010006,
 265         0x91a8, 0x0fff0fff, 0x00010006,
 266         0x9150, 0xffffdfff, 0x6e944040,
 267         0x917c, 0x0fff0fff, 0x00030002,
 268         0x9198, 0x0fff0fff, 0x00030002,
 269         0x915c, 0x0fff0fff, 0x00010000,
 270         0x3f90, 0xffff0001, 0xff000000,
 271         0x9178, 0x0fff0fff, 0x00070000,
 272         0x9194, 0x0fff0fff, 0x00070000,
 273         0x9148, 0xffff0001, 0xff000000,
 274         0x9190, 0x0fff0fff, 0x00090008,
 275         0x91ac, 0x0fff0fff, 0x00090008,
 276         0x3f94, 0xffff0000, 0xff000000,
 277         0x914c, 0xffff0000, 0xff000000,
 278         0x929c, 0x00000fff, 0x00000001,
 279         0x55e4, 0xff607fff, 0xfc000100,
 280         0x8a18, 0xff000fff, 0x00000100,
 281         0x8b28, 0xff000fff, 0x00000100,
 282         0x9144, 0xfffc0fff, 0x00000100,
 283         0x6ed8, 0x00010101, 0x00010000,
 284         0x9830, 0xffffffff, 0x00000000,
 285         0x9834, 0xf00fffff, 0x00000400,
 286         0x9838, 0xfffffffe, 0x00000000,
 287         0xd0c0, 0xff000fff, 0x00000100,
 288         0xd02c, 0xbfffff1f, 0x08421000,
 289         0xd0b8, 0x73773777, 0x12010001,
 290         0x5bb0, 0x000000f0, 0x00000070,
 291         0x98f8, 0x73773777, 0x12010001,
 292         0x98fc, 0xffffffff, 0x00000010,
 293         0x9b7c, 0x00ff0000, 0x00fc0000,
 294         0x8030, 0x00001f0f, 0x0000100a,
 295         0x2f48, 0x73773777, 0x12010001,
 296         0x2408, 0x00030000, 0x000c007f,
 297         0x8a14, 0xf000003f, 0x00000007,
 298         0x8b24, 0x3fff3fff, 0x00ff0fff,
 299         0x8b10, 0x0000ff0f, 0x00000000,
 300         0x28a4c, 0x07ffffff, 0x06000000,
 301         0x4d8, 0x00000fff, 0x00000100,
 302         0xa008, 0xffffffff, 0x00010000,
 303         0x913c, 0xffff03ff, 0x01000100,
 304         0x8c00, 0x000000ff, 0x00000003,
 305         0x8c04, 0xf8ff00ff, 0x40600060,
 306         0x8cf0, 0x1fff1fff, 0x08e00410,
 307         0x28350, 0x00000f01, 0x00000000,
 308         0x9508, 0xf700071f, 0x00000002,
 309         0x960c, 0xffffffff, 0x54763210,
 310         0x20ef8, 0x01ff01ff, 0x00000002,
 311         0x20e98, 0xfffffbff, 0x00200000,
 312         0x2015c, 0xffffffff, 0x00000f40,
 313         0x88c4, 0x001f3ae3, 0x00000082,
 314         0x8978, 0x3fffffff, 0x04050140,
 315         0x88d4, 0x0000001f, 0x00000010,
 316         0x8974, 0xffffffff, 0x00000000
 317 };
 318
 319 static const u32 scrapper_golden_registers[] =
 320 {
 321         0x690, 0x3fff3fff, 0x20c00033,
 322         0x918c, 0x0fff0fff, 0x00010006,
 323         0x918c, 0x0fff0fff, 0x00010006,
 324         0x91a8, 0x0fff0fff, 0x00010006,
 325         0x91a8, 0x0fff0fff, 0x00010006,
 326         0x9150, 0xffffdfff, 0x6e944040,
 327         0x9150, 0xffffdfff, 0x6e944040,
 328         0x917c, 0x0fff0fff, 0x00030002,
 329         0x917c, 0x0fff0fff, 0x00030002,
 330         0x9198, 0x0fff0fff, 0x00030002,
 331         0x9198, 0x0fff0fff, 0x00030002,
 332         0x915c, 0x0fff0fff, 0x00010000,
 333         0x915c, 0x0fff0fff, 0x00010000,
 334         0x3f90, 0xffff0001, 0xff000000,
 335         0x3f90, 0xffff0001, 0xff000000,
 336         0x9178, 0x0fff0fff, 0x00070000,
 337         0x9178, 0x0fff0fff, 0x00070000,
 338         0x9194, 0x0fff0fff, 0x00070000,
 339         0x9194, 0x0fff0fff, 0x00070000,
 340         0x9148, 0xffff0001, 0xff000000,
 341         0x9148, 0xffff0001, 0xff000000,
 342         0x9190, 0x0fff0fff, 0x00090008,
 343         0x9190, 0x0fff0fff, 0x00090008,
 344         0x91ac, 0x0fff0fff, 0x00090008,
 345         0x91ac, 0x0fff0fff, 0x00090008,
 346         0x3f94, 0xffff0000, 0xff000000,
 347         0x3f94, 0xffff0000, 0xff000000,
 348         0x914c, 0xffff0000, 0xff000000,
 349         0x914c, 0xffff0000, 0xff000000,
 350         0x929c, 0x00000fff, 0x00000001,
 351         0x929c, 0x00000fff, 0x00000001,
 352         0x55e4, 0xff607fff, 0xfc000100,
 353         0x8a18, 0xff000fff, 0x00000100,
 354         0x8a18, 0xff000fff, 0x00000100,
 355         0x8b28, 0xff000fff, 0x00000100,
 356         0x8b28, 0xff000fff, 0x00000100,
 357         0x9144, 0xfffc0fff, 0x00000100,
 358         0x9144, 0xfffc0fff, 0x00000100,
 359         0x6ed8, 0x00010101, 0x00010000,
 360         0x9830, 0xffffffff, 0x00000000,
 361         0x9830, 0xffffffff, 0x00000000,
 362         0x9834, 0xf00fffff, 0x00000400,
 363         0x9834, 0xf00fffff, 0x00000400,
 364         0x9838, 0xfffffffe, 0x00000000,
 365         0x9838, 0xfffffffe, 0x00000000,
 366         0xd0c0, 0xff000fff, 0x00000100,
 367         0xd02c, 0xbfffff1f, 0x08421000,
 368         0xd02c, 0xbfffff1f, 0x08421000,
 369         0xd0b8, 0x73773777, 0x12010001,
 370         0xd0b8, 0x73773777, 0x12010001,
 371         0x5bb0, 0x000000f0, 0x00000070,
 372         0x98f8, 0x73773777, 0x12010001,
 373         0x98f8, 0x73773777, 0x12010001,
 374         0x98fc, 0xffffffff, 0x00000010,
 375         0x98fc, 0xffffffff, 0x00000010,
 376         0x9b7c, 0x00ff0000, 0x00fc0000,
 377         0x9b7c, 0x00ff0000, 0x00fc0000,
 378         0x8030, 0x00001f0f, 0x0000100a,
 379         0x8030, 0x00001f0f, 0x0000100a,
 380         0x2f48, 0x73773777, 0x12010001,
 381         0x2f48, 0x73773777, 0x12010001,
 382         0x2408, 0x00030000, 0x000c007f,
 383         0x8a14, 0xf000003f, 0x00000007,
 384         0x8a14, 0xf000003f, 0x00000007,
 385         0x8b24, 0x3fff3fff, 0x00ff0fff,
 386         0x8b24, 0x3fff3fff, 0x00ff0fff,
 387         0x8b10, 0x0000ff0f, 0x00000000,
 388         0x8b10, 0x0000ff0f, 0x00000000,
 389         0x28a4c, 0x07ffffff, 0x06000000,
 390         0x28a4c, 0x07ffffff, 0x06000000,
 391         0x4d8, 0x00000fff, 0x00000100,
 392         0x4d8, 0x00000fff, 0x00000100,
 393         0xa008, 0xffffffff, 0x00010000,
 394         0xa008, 0xffffffff, 0x00010000,
 395         0x913c, 0xffff03ff, 0x01000100,
 396         0x913c, 0xffff03ff, 0x01000100,
 397         0x90e8, 0x001fffff, 0x010400c0,
 398         0x8c00, 0x000000ff, 0x00000003,
 399         0x8c00, 0x000000ff, 0x00000003,
 400         0x8c04, 0xf8ff00ff, 0x40600060,
 401         0x8c04, 0xf8ff00ff, 0x40600060,
 402         0x8c30, 0x0000000f, 0x00040005,
 403         0x8cf0, 0x1fff1fff, 0x08e00410,
 404         0x8cf0, 0x1fff1fff, 0x08e00410,
 405         0x900c, 0x00ffffff, 0x0017071f,
 406         0x28350, 0x00000f01, 0x00000000,
 407         0x28350, 0x00000f01, 0x00000000,
 408         0x9508, 0xf700071f, 0x00000002,
 409         0x9508, 0xf700071f, 0x00000002,
 410         0x9688, 0x00300000, 0x0017000f,
 411         0x960c, 0xffffffff, 0x54763210,
 412         0x960c, 0xffffffff, 0x54763210,
 413         0x20ef8, 0x01ff01ff, 0x00000002,
 414         0x20e98, 0xfffffbff, 0x00200000,
 415         0x2015c, 0xffffffff, 0x00000f40,
 416         0x88c4, 0x001f3ae3, 0x00000082,
 417         0x88c4, 0x001f3ae3, 0x00000082,
 418         0x8978, 0x3fffffff, 0x04050140,
 419         0x8978, 0x3fffffff, 0x04050140,
 420         0x88d4, 0x0000001f, 0x00000010,
 421         0x88d4, 0x0000001f, 0x00000010,
 422         0x8974, 0xffffffff, 0x00000000,
 423         0x8974, 0xffffffff, 0x00000000
 424 };
 425
 426 static void ni_init_golden_registers(struct radeon_device *rdev)
 427 {
 428         switch (rdev->family) {
 429         case CHIP_CAYMAN:
 430                 radeon_program_register_sequence(rdev,
 431                                                  cayman_golden_registers,
 432                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 433                 radeon_program_register_sequence(rdev,
 434                                                  cayman_golden_registers2,
 435                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 436                 break;
 437         case CHIP_ARUBA:
 438                 if ((rdev->pdev->device == 0x9900) ||
 439                     (rdev->pdev->device == 0x9901) ||
 440                     (rdev->pdev->device == 0x9903) ||
 441                     (rdev->pdev->device == 0x9904) ||
 442                     (rdev->pdev->device == 0x9905) ||
 443                     (rdev->pdev->device == 0x9906) ||
 444                     (rdev->pdev->device == 0x9907) ||
 445                     (rdev->pdev->device == 0x9908) ||
 446                     (rdev->pdev->device == 0x9909) ||
 447                     (rdev->pdev->device == 0x990A) ||
 448                     (rdev->pdev->device == 0x990B) ||
 449                     (rdev->pdev->device == 0x990C) ||
 450                     (rdev->pdev->device == 0x990D) ||
 451                     (rdev->pdev->device == 0x990E) ||
 452                     (rdev->pdev->device == 0x990F) ||
 453                     (rdev->pdev->device == 0x9910) ||
 454                     (rdev->pdev->device == 0x9913) ||
 455                     (rdev->pdev->device == 0x9917) ||
 456                     (rdev->pdev->device == 0x9918)) {
 457                         radeon_program_register_sequence(rdev,
 458                                                          dvst_golden_registers,
 459                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 460                         radeon_program_register_sequence(rdev,
 461                                                          dvst_golden_registers2,
 462                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 463                 } else {
 464                         radeon_program_register_sequence(rdev,
 465                                                          scrapper_golden_registers,
 466                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 467                         radeon_program_register_sequence(rdev,
 468                                                          dvst_golden_registers2,
 469                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 470                 }
 471                 break;
 472         default:
 473                 break;
 474         }
 475 }
 476
 477 #define BTC_IO_MC_REGS_SIZE 29
 478
 479 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 480         {0x00000077, 0xff010100},
 481         {0x00000078, 0x00000000},
 482         {0x00000079, 0x00001434},
 483         {0x0000007a, 0xcc08ec08},
 484         {0x0000007b, 0x00040000},
 485         {0x0000007c, 0x000080c0},
 486         {0x0000007d, 0x09000000},
 487         {0x0000007e, 0x00210404},
 488         {0x00000081, 0x08a8e800},
 489         {0x00000082, 0x00030444},
 490         {0x00000083, 0x00000000},
 491         {0x00000085, 0x00000001},
 492         {0x00000086, 0x00000002},
 493         {0x00000087, 0x48490000},
 494         {0x00000088, 0x20244647},
 495         {0x00000089, 0x00000005},
 496         {0x0000008b, 0x66030000},
 497         {0x0000008c, 0x00006603},
 498         {0x0000008d, 0x00000100},
 499         {0x0000008f, 0x00001c0a},
 500         {0x00000090, 0xff000001},
 501         {0x00000094, 0x00101101},
 502         {0x00000095, 0x00000fff},
 503         {0x00000096, 0x00116fff},
 504         {0x00000097, 0x60010000},
 505         {0x00000098, 0x10010000},
 506         {0x00000099, 0x00006000},
 507         {0x0000009a, 0x00001000},
 508         {0x0000009f, 0x00946a00}
 509 };
 510
 511 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 512         {0x00000077, 0xff010100},
 513         {0x00000078, 0x00000000},
 514         {0x00000079, 0x00001434},
 515         {0x0000007a, 0xcc08ec08},
 516         {0x0000007b, 0x00040000},
 517         {0x0000007c, 0x000080c0},
 518         {0x0000007d, 0x09000000},
 519         {0x0000007e, 0x00210404},
 520         {0x00000081, 0x08a8e800},
 521         {0x00000082, 0x00030444},
 522         {0x00000083, 0x00000000},
 523         {0x00000085, 0x00000001},
 524         {0x00000086, 0x00000002},
 525         {0x00000087, 0x48490000},
 526         {0x00000088, 0x20244647},
 527         {0x00000089, 0x00000005},
 528         {0x0000008b, 0x66030000},
 529         {0x0000008c, 0x00006603},
 530         {0x0000008d, 0x00000100},
 531         {0x0000008f, 0x00001c0a},
 532         {0x00000090, 0xff000001},
 533         {0x00000094, 0x00101101},
 534         {0x00000095, 0x00000fff},
 535         {0x00000096, 0x00116fff},
 536         {0x00000097, 0x60010000},
 537         {0x00000098, 0x10010000},
 538         {0x00000099, 0x00006000},
 539         {0x0000009a, 0x00001000},
 540         {0x0000009f, 0x00936a00}
 541 };
 542
 543 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 544         {0x00000077, 0xff010100},
 545         {0x00000078, 0x00000000},
 546         {0x00000079, 0x00001434},
 547         {0x0000007a, 0xcc08ec08},
 548         {0x0000007b, 0x00040000},
 549         {0x0000007c, 0x000080c0},
 550         {0x0000007d, 0x09000000},
 551         {0x0000007e, 0x00210404},
 552         {0x00000081, 0x08a8e800},
 553         {0x00000082, 0x00030444},
 554         {0x00000083, 0x00000000},
 555         {0x00000085, 0x00000001},
 556         {0x00000086, 0x00000002},
 557         {0x00000087, 0x48490000},
 558         {0x00000088, 0x20244647},
 559         {0x00000089, 0x00000005},
 560         {0x0000008b, 0x66030000},
 561         {0x0000008c, 0x00006603},
 562         {0x0000008d, 0x00000100},
 563         {0x0000008f, 0x00001c0a},
 564         {0x00000090, 0xff000001},
 565         {0x00000094, 0x00101101},
 566         {0x00000095, 0x00000fff},
 567         {0x00000096, 0x00116fff},
 568         {0x00000097, 0x60010000},
 569         {0x00000098, 0x10010000},
 570         {0x00000099, 0x00006000},
 571         {0x0000009a, 0x00001000},
 572         {0x0000009f, 0x00916a00}
 573 };
 574
 575 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 576         {0x00000077, 0xff010100},
 577         {0x00000078, 0x00000000},
 578         {0x00000079, 0x00001434},
 579         {0x0000007a, 0xcc08ec08},
 580         {0x0000007b, 0x00040000},
 581         {0x0000007c, 0x000080c0},
 582         {0x0000007d, 0x09000000},
 583         {0x0000007e, 0x00210404},
 584         {0x00000081, 0x08a8e800},
 585         {0x00000082, 0x00030444},
 586         {0x00000083, 0x00000000},
 587         {0x00000085, 0x00000001},
 588         {0x00000086, 0x00000002},
 589         {0x00000087, 0x48490000},
 590         {0x00000088, 0x20244647},
 591         {0x00000089, 0x00000005},
 592         {0x0000008b, 0x66030000},
 593         {0x0000008c, 0x00006603},
 594         {0x0000008d, 0x00000100},
 595         {0x0000008f, 0x00001c0a},
 596         {0x00000090, 0xff000001},
 597         {0x00000094, 0x00101101},
 598         {0x00000095, 0x00000fff},
 599         {0x00000096, 0x00116fff},
 600         {0x00000097, 0x60010000},
 601         {0x00000098, 0x10010000},
 602         {0x00000099, 0x00006000},
 603         {0x0000009a, 0x00001000},
 604         {0x0000009f, 0x00976b00}
 605 };
 606
 607 int ni_mc_load_microcode(struct radeon_device *rdev)
 608 {
 609         const __be32 *fw_data;
 610         u32 mem_type, running, blackout = 0;
 611         u32 *io_mc_regs;
 612         int i, ucode_size, regs_size;
 613
 614         if (!rdev->mc_fw)
 615                 return -EINVAL;
 616
 617         switch (rdev->family) {
 618         case CHIP_BARTS:
 619                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 620                 ucode_size = BTC_MC_UCODE_SIZE;
 621                 regs_size = BTC_IO_MC_REGS_SIZE;
 622                 break;
 623         case CHIP_TURKS:
 624                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 625                 ucode_size = BTC_MC_UCODE_SIZE;
 626                 regs_size = BTC_IO_MC_REGS_SIZE;
 627                 break;
 628         case CHIP_CAICOS:
 629         default:
 630                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 631                 ucode_size = BTC_MC_UCODE_SIZE;
 632                 regs_size = BTC_IO_MC_REGS_SIZE;
 633                 break;
 634         case CHIP_CAYMAN:
 635                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 636                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 637                 regs_size = BTC_IO_MC_REGS_SIZE;
 638                 break;
 639         }
 640
 641         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 642         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 643
 644         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 645                 if (running) {
 646                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 647                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 648                 }
 649
 650                 /* reset the engine and set to writable */
 651                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 652                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 653
 654                 /* load mc io regs */
 655                 for (i = 0; i < regs_size; i++) {
 656                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 657                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 658                 }
 659                 /* load the MC ucode */
 660                 fw_data = (const __be32 *)rdev->mc_fw->data;
 661                 for (i = 0; i < ucode_size; i++)
 662                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 663
 664                 /* put the engine back into the active state */
 665                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 666                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 668
 669                 /* wait for training to complete */
 670                 for (i = 0; i < rdev->usec_timeout; i++) {
 671                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 672                                 break;
 673                         udelay(1);
 674                 }
 675
 676                 if (running)
 677                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 678         }
 679
 680         return 0;
 681 }
 682
 683 int ni_init_microcode(struct radeon_device *rdev)
 684 {
 685         const char *chip_name;
 686         const char *rlc_chip_name;
 687         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 688         size_t smc_req_size = 0;
 689         char fw_name[30];
 690         int err;
 691
 692         DRM_DEBUG("\n");
 693
 694         switch (rdev->family) {
 695         case CHIP_BARTS:
 696                 chip_name = "BARTS";
 697                 rlc_chip_name = "BTC";
 698                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 699                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 700                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 701                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 702                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 703                 break;
 704         case CHIP_TURKS:
 705                 chip_name = "TURKS";
 706                 rlc_chip_name = "BTC";
 707                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 708                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 709                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 710                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 711                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 712                 break;
 713         case CHIP_CAICOS:
 714                 chip_name = "CAICOS";
 715                 rlc_chip_name = "BTC";
 716                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 717                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 718                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 719                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 720                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 721                 break;
 722         case CHIP_CAYMAN:
 723                 chip_name = "CAYMAN";
 724                 rlc_chip_name = "CAYMAN";
 725                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 726                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 727                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 728                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 729                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 730                 break;
 731         case CHIP_ARUBA:
 732                 chip_name = "ARUBA";
 733                 rlc_chip_name = "ARUBA";
 734                 /* pfp/me same size as CAYMAN */
 735                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 736                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 737                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 738                 mc_req_size = 0;
 739                 break;
 740         default: BUG();
 741         }
 742
 743         DRM_INFO("Loading %s Microcode\n", chip_name);
 744
 745         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 746         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 747         if (err)
 748                 goto out;
 749         if (rdev->pfp_fw->size != pfp_req_size) {
 750                 printk(KERN_ERR
 751                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 752                        rdev->pfp_fw->size, fw_name);
 753                 err = -EINVAL;
 754                 goto out;
 755         }
 756
 757         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 758         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 759         if (err)
 760                 goto out;
 761         if (rdev->me_fw->size != me_req_size) {
 762                 printk(KERN_ERR
 763                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 764                        rdev->me_fw->size, fw_name);
 765                 err = -EINVAL;
 766         }
 767
 768         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 769         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 770         if (err)
 771                 goto out;
 772         if (rdev->rlc_fw->size != rlc_req_size) {
 773                 printk(KERN_ERR
 774                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 775                        rdev->rlc_fw->size, fw_name);
 776                 err = -EINVAL;
 777         }
 778
 779         /* no MC ucode on TN */
 780         if (!(rdev->flags & RADEON_IS_IGP)) {
 781                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 782                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 783                 if (err)
 784                         goto out;
 785                 if (rdev->mc_fw->size != mc_req_size) {
 786                         printk(KERN_ERR
 787                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 788                                rdev->mc_fw->size, fw_name);
 789                         err = -EINVAL;
 790                 }
 791         }
 792
 793         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 794                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 795                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 796                 if (err) {
 797                         printk(KERN_ERR
 798                                "smc: error loading firmware \"%s\"\n",
 799                                fw_name);
 800                         release_firmware(rdev->smc_fw);
 801                         rdev->smc_fw = NULL;
 802                         err = 0;
 803                 } else if (rdev->smc_fw->size != smc_req_size) {
 804                         printk(KERN_ERR
 805                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 806                                rdev->mc_fw->size, fw_name);
 807                         err = -EINVAL;
 808                 }
 809         }
 810
 811 out:
 812         if (err) {
 813                 if (err != -EINVAL)
 814                         printk(KERN_ERR
 815                                "ni_cp: Failed to load firmware \"%s\"\n",
 816                                fw_name);
 817                 release_firmware(rdev->pfp_fw);
 818                 rdev->pfp_fw = NULL;
 819                 release_firmware(rdev->me_fw);
 820                 rdev->me_fw = NULL;
 821                 release_firmware(rdev->rlc_fw);
 822                 rdev->rlc_fw = NULL;
 823                 release_firmware(rdev->mc_fw);
 824                 rdev->mc_fw = NULL;
 825         }
 826         return err;
 827 }
 828
 829 int tn_get_temp(struct radeon_device *rdev)
 830 {
 831         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 832         int actual_temp = (temp / 8) - 49;
 833
 834         return actual_temp * 1000;
 835 }
 836
 837 /*
 838  * Core functions
 839  */
 840 static void cayman_gpu_init(struct radeon_device *rdev)
 841 {
 842         u32 gb_addr_config = 0;
 843         u32 mc_shared_chmap, mc_arb_ramcfg;
 844         u32 cgts_tcc_disable;
 845         u32 sx_debug_1;
 846         u32 smx_dc_ctl0;
 847         u32 cgts_sm_ctrl_reg;
 848         u32 hdp_host_path_cntl;
 849         u32 tmp;
 850         u32 disabled_rb_mask;
 851         int i, j;
 852
 853         switch (rdev->family) {
 854         case CHIP_CAYMAN:
 855                 rdev->config.cayman.max_shader_engines = 2;
 856                 rdev->config.cayman.max_pipes_per_simd = 4;
 857                 rdev->config.cayman.max_tile_pipes = 8;
 858                 rdev->config.cayman.max_simds_per_se = 12;
 859                 rdev->config.cayman.max_backends_per_se = 4;
 860                 rdev->config.cayman.max_texture_channel_caches = 8;
 861                 rdev->config.cayman.max_gprs = 256;
 862                 rdev->config.cayman.max_threads = 256;
 863                 rdev->config.cayman.max_gs_threads = 32;
 864                 rdev->config.cayman.max_stack_entries = 512;
 865                 rdev->config.cayman.sx_num_of_sets = 8;
 866                 rdev->config.cayman.sx_max_export_size = 256;
 867                 rdev->config.cayman.sx_max_export_pos_size = 64;
 868                 rdev->config.cayman.sx_max_export_smx_size = 192;
 869                 rdev->config.cayman.max_hw_contexts = 8;
 870                 rdev->config.cayman.sq_num_cf_insts = 2;
 871
 872                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 873                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 874                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 875                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 876                 break;
 877         case CHIP_ARUBA:
 878         default:
 879                 rdev->config.cayman.max_shader_engines = 1;
 880                 rdev->config.cayman.max_pipes_per_simd = 4;
 881                 rdev->config.cayman.max_tile_pipes = 2;
 882                 if ((rdev->pdev->device == 0x9900) ||
 883                     (rdev->pdev->device == 0x9901) ||
 884                     (rdev->pdev->device == 0x9905) ||
 885                     (rdev->pdev->device == 0x9906) ||
 886                     (rdev->pdev->device == 0x9907) ||
 887                     (rdev->pdev->device == 0x9908) ||
 888                     (rdev->pdev->device == 0x9909) ||
 889                     (rdev->pdev->device == 0x990B) ||
 890                     (rdev->pdev->device == 0x990C) ||
 891                     (rdev->pdev->device == 0x990F) ||
 892                     (rdev->pdev->device == 0x9910) ||
 893                     (rdev->pdev->device == 0x9917) ||
 894                     (rdev->pdev->device == 0x9999) ||
 895                     (rdev->pdev->device == 0x999C)) {
 896                         rdev->config.cayman.max_simds_per_se = 6;
 897                         rdev->config.cayman.max_backends_per_se = 2;
 898                         rdev->config.cayman.max_hw_contexts = 8;
 899                         rdev->config.cayman.sx_max_export_size = 256;
 900                         rdev->config.cayman.sx_max_export_pos_size = 64;
 901                         rdev->config.cayman.sx_max_export_smx_size = 192;
 902                 } else if ((rdev->pdev->device == 0x9903) ||
 903                            (rdev->pdev->device == 0x9904) ||
 904                            (rdev->pdev->device == 0x990A) ||
 905                            (rdev->pdev->device == 0x990D) ||
 906                            (rdev->pdev->device == 0x990E) ||
 907                            (rdev->pdev->device == 0x9913) ||
 908                            (rdev->pdev->device == 0x9918) ||
 909                            (rdev->pdev->device == 0x999D)) {
 910                         rdev->config.cayman.max_simds_per_se = 4;
 911                         rdev->config.cayman.max_backends_per_se = 2;
 912                         rdev->config.cayman.max_hw_contexts = 8;
 913                         rdev->config.cayman.sx_max_export_size = 256;
 914                         rdev->config.cayman.sx_max_export_pos_size = 64;
 915                         rdev->config.cayman.sx_max_export_smx_size = 192;
 916                 } else if ((rdev->pdev->device == 0x9919) ||
 917                            (rdev->pdev->device == 0x9990) ||
 918                            (rdev->pdev->device == 0x9991) ||
 919                            (rdev->pdev->device == 0x9994) ||
 920                            (rdev->pdev->device == 0x9995) ||
 921                            (rdev->pdev->device == 0x9996) ||
 922                            (rdev->pdev->device == 0x999A) ||
 923                            (rdev->pdev->device == 0x99A0)) {
 924                         rdev->config.cayman.max_simds_per_se = 3;
 925                         rdev->config.cayman.max_backends_per_se = 1;
 926                         rdev->config.cayman.max_hw_contexts = 4;
 927                         rdev->config.cayman.sx_max_export_size = 128;
 928                         rdev->config.cayman.sx_max_export_pos_size = 32;
 929                         rdev->config.cayman.sx_max_export_smx_size = 96;
 930                 } else {
 931                         rdev->config.cayman.max_simds_per_se = 2;
 932                         rdev->config.cayman.max_backends_per_se = 1;
 933                         rdev->config.cayman.max_hw_contexts = 4;
 934                         rdev->config.cayman.sx_max_export_size = 128;
 935                         rdev->config.cayman.sx_max_export_pos_size = 32;
 936                         rdev->config.cayman.sx_max_export_smx_size = 96;
 937                 }
 938                 rdev->config.cayman.max_texture_channel_caches = 2;
 939                 rdev->config.cayman.max_gprs = 256;
 940                 rdev->config.cayman.max_threads = 256;
 941                 rdev->config.cayman.max_gs_threads = 32;
 942                 rdev->config.cayman.max_stack_entries = 512;
 943                 rdev->config.cayman.sx_num_of_sets = 8;
 944                 rdev->config.cayman.sq_num_cf_insts = 2;
 945
 946                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 947                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 948                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 949                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 950                 break;
 951         }
 952
 953         /* Initialize HDP */
 954         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 955                 WREG32((0x2c14 + j), 0x00000000);
 956                 WREG32((0x2c18 + j), 0x00000000);
 957                 WREG32((0x2c1c + j), 0x00000000);
 958                 WREG32((0x2c20 + j), 0x00000000);
 959                 WREG32((0x2c24 + j), 0x00000000);
 960         }
 961
 962         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 963
 964         evergreen_fix_pci_max_read_req_size(rdev);
 965
 966         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 967         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 968
 969         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 970         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 971         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 972                 rdev->config.cayman.mem_row_size_in_kb = 4;
 973         /* XXX use MC settings? */
 974         rdev->config.cayman.shader_engine_tile_size = 32;
 975         rdev->config.cayman.num_gpus = 1;
 976         rdev->config.cayman.multi_gpu_tile_size = 64;
 977
 978         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 979         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 980         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 981         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 982         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 983         rdev->config.cayman.num_shader_engines = tmp + 1;
 984         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 985         rdev->config.cayman.num_gpus = tmp + 1;
 986         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 987         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 988         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 989         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 990
 991
 992         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 993          * not have bank info, so create a custom tiling dword.
 994          * bits 3:0   num_pipes
 995          * bits 7:4   num_banks
 996          * bits 11:8  group_size
 997          * bits 15:12 row_size
 998          */
 999         rdev->config.cayman.tile_config = 0;
1000         switch (rdev->config.cayman.num_tile_pipes) {
1001         case 1:
1002         default:
1003                 rdev->config.cayman.tile_config |= (0 << 0);
1004                 break;
1005         case 2:
1006                 rdev->config.cayman.tile_config |= (1 << 0);
1007                 break;
1008         case 4:
1009                 rdev->config.cayman.tile_config |= (2 << 0);
1010                 break;
1011         case 8:
1012                 rdev->config.cayman.tile_config |= (3 << 0);
1013                 break;
1014         }
1015
1016         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1017         if (rdev->flags & RADEON_IS_IGP)
1018                 rdev->config.cayman.tile_config |= 1 << 4;
1019         else {
1020                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1021                 case 0: /* four banks */
1022                         rdev->config.cayman.tile_config |= 0 << 4;
1023                         break;
1024                 case 1: /* eight banks */
1025                         rdev->config.cayman.tile_config |= 1 << 4;
1026                         break;
1027                 case 2: /* sixteen banks */
1028                 default:
1029                         rdev->config.cayman.tile_config |= 2 << 4;
1030                         break;
1031                 }
1032         }
1033         rdev->config.cayman.tile_config |=
1034                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1035         rdev->config.cayman.tile_config |=
1036                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1037
1038         tmp = 0;
1039         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1040                 u32 rb_disable_bitmap;
1041
1042                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1043                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1044                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1045                 tmp <<= 4;
1046                 tmp |= rb_disable_bitmap;
1047         }
1048         /* enabled rb are just the one not disabled :) */
1049         disabled_rb_mask = tmp;
1050         tmp = 0;
1051         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1052                 tmp |= (1 << i);
1053         /* if all the backends are disabled, fix it up here */
1054         if ((disabled_rb_mask & tmp) == tmp) {
1055                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1056                         disabled_rb_mask &= ~(1 << i);
1057         }
1058
1059         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1060         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1061
1062         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1063         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1064         if (ASIC_IS_DCE6(rdev))
1065                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1066         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1067         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1068         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1069         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1070         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1071         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1072
1073         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1074             (rdev->flags & RADEON_IS_IGP)) {
1075                 if ((disabled_rb_mask & 3) == 1) {
1076                         /* RB0 disabled, RB1 enabled */
1077                         tmp = 0x11111111;
1078                 } else {
1079                         /* RB1 disabled, RB0 enabled */
1080                         tmp = 0x00000000;
1081                 }
1082         } else {
1083                 tmp = gb_addr_config & NUM_PIPES_MASK;
1084                 tmp = r6xx_remap_render_backend(rdev, tmp,
1085                                                 rdev->config.cayman.max_backends_per_se *
1086                                                 rdev->config.cayman.max_shader_engines,
1087                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1088         }
1089         WREG32(GB_BACKEND_MAP, tmp);
1090
1091         cgts_tcc_disable = 0xffff0000;
1092         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1093                 cgts_tcc_disable &= ~(1 << (16 + i));
1094         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1095         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1096         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1097         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1098
1099         /* reprogram the shader complex */
1100         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1101         for (i = 0; i < 16; i++)
1102                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1103         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1104
1105         /* set HW defaults for 3D engine */
1106         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1107
1108         sx_debug_1 = RREG32(SX_DEBUG_1);
1109         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1110         WREG32(SX_DEBUG_1, sx_debug_1);
1111
1112         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1113         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1114         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1115         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1116
1117         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1118
1119         /* need to be explicitly zero-ed */
1120         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1121         WREG32(SQ_LSTMP_RING_BASE, 0);
1122         WREG32(SQ_HSTMP_RING_BASE, 0);
1123         WREG32(SQ_ESTMP_RING_BASE, 0);
1124         WREG32(SQ_GSTMP_RING_BASE, 0);
1125         WREG32(SQ_VSTMP_RING_BASE, 0);
1126         WREG32(SQ_PSTMP_RING_BASE, 0);
1127
1128         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1129
1130         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1131                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1132                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1133
1134         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1135                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1136                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1137
1138
1139         WREG32(VGT_NUM_INSTANCES, 1);
1140
1141         WREG32(CP_PERFMON_CNTL, 0);
1142
1143         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1144                                   FETCH_FIFO_HIWATER(0x4) |
1145                                   DONE_FIFO_HIWATER(0xe0) |
1146                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1147
1148         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1149         WREG32(SQ_CONFIG, (VC_ENABLE |
1150                            EXPORT_SRC_C |
1151                            GFX_PRIO(0) |
1152                            CS1_PRIO(0) |
1153                            CS2_PRIO(1)));
1154         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1155
1156         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1157                                           FORCE_EOV_MAX_REZ_CNT(255)));
1158
1159         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1160                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1161
1162         WREG32(VGT_GS_VERTEX_REUSE, 16);
1163         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1164
1165         WREG32(CB_PERF_CTR0_SEL_0, 0);
1166         WREG32(CB_PERF_CTR0_SEL_1, 0);
1167         WREG32(CB_PERF_CTR1_SEL_0, 0);
1168         WREG32(CB_PERF_CTR1_SEL_1, 0);
1169         WREG32(CB_PERF_CTR2_SEL_0, 0);
1170         WREG32(CB_PERF_CTR2_SEL_1, 0);
1171         WREG32(CB_PERF_CTR3_SEL_0, 0);
1172         WREG32(CB_PERF_CTR3_SEL_1, 0);
1173
1174         tmp = RREG32(HDP_MISC_CNTL);
1175         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1176         WREG32(HDP_MISC_CNTL, tmp);
1177
1178         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1179         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1180
1181         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1182
1183         udelay(50);
1184
1185         /* set clockgating golden values on TN */
1186         if (rdev->family == CHIP_ARUBA) {
1187                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1188                 tmp &= ~0x00380000;
1189                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1190                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1191                 tmp &= ~0x0e000000;
1192                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1193         }
1194 }
1195
1196 /*
1197  * GART
1198  */
1199 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1200 {
1201         /* flush hdp cache */
1202         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1203
1204         /* bits 0-7 are the VM contexts0-7 */
1205         WREG32(VM_INVALIDATE_REQUEST, 1);
1206 }
1207
1208 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1209 {
1210         int i, r;
1211
1212         if (rdev->gart.robj == NULL) {
1213                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1214                 return -EINVAL;
1215         }
1216         r = radeon_gart_table_vram_pin(rdev);
1217         if (r)
1218                 return r;
1219         radeon_gart_restore(rdev);
1220         /* Setup TLB control */
1221         WREG32(MC_VM_MX_L1_TLB_CNTL,
1222                (0xA << 7) |
1223                ENABLE_L1_TLB |
1224                ENABLE_L1_FRAGMENT_PROCESSING |
1225                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1226                ENABLE_ADVANCED_DRIVER_MODEL |
1227                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1228         /* Setup L2 cache */
1229         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1230                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1231                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1232                EFFECTIVE_L2_QUEUE_SIZE(7) |
1233                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1234         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1235         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1236                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1237         /* setup context0 */
1238         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1239         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1240         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1241         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1242                         (u32)(rdev->dummy_page.addr >> 12));
1243         WREG32(VM_CONTEXT0_CNTL2, 0);
1244         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1245                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1246
1247         WREG32(0x15D4, 0);
1248         WREG32(0x15D8, 0);
1249         WREG32(0x15DC, 0);
1250
1251         /* empty context1-7 */
1252         /* Assign the pt base to something valid for now; the pts used for
1253          * the VMs are determined by the application and setup and assigned
1254          * on the fly in the vm part of radeon_gart.c
1255          */
1256         for (i = 1; i < 8; i++) {
1257                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1258                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1259                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1260                         rdev->gart.table_addr >> 12);
1261         }
1262
1263         /* enable context1-7 */
1264         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1265                (u32)(rdev->dummy_page.addr >> 12));
1266         WREG32(VM_CONTEXT1_CNTL2, 4);
1267         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1268                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1269                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1270                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1271                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1272                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1273                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1274                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1275                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1276                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1277                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1278                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1279                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1280
1281         cayman_pcie_gart_tlb_flush(rdev);
1282         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1283                  (unsigned)(rdev->mc.gtt_size >> 20),
1284                  (unsigned long long)rdev->gart.table_addr);
1285         rdev->gart.ready = true;
1286         return 0;
1287 }
1288
1289 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1290 {
1291         /* Disable all tables */
1292         WREG32(VM_CONTEXT0_CNTL, 0);
1293         WREG32(VM_CONTEXT1_CNTL, 0);
1294         /* Setup TLB control */
1295         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1296                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1297                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1298         /* Setup L2 cache */
1299         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1300                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1301                EFFECTIVE_L2_QUEUE_SIZE(7) |
1302                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1303         WREG32(VM_L2_CNTL2, 0);
1304         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1305                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1306         radeon_gart_table_vram_unpin(rdev);
1307 }
1308
1309 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1310 {
1311         cayman_pcie_gart_disable(rdev);
1312         radeon_gart_table_vram_free(rdev);
1313         radeon_gart_fini(rdev);
1314 }
1315
1316 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1317                               int ring, u32 cp_int_cntl)
1318 {
1319         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1320
1321         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1322         WREG32(CP_INT_CNTL, cp_int_cntl);
1323 }
1324
1325 /*
1326  * CP.
1327  */
1328 void cayman_fence_ring_emit(struct radeon_device *rdev,
1329                             struct radeon_fence *fence)
1330 {
1331         struct radeon_ring *ring = &rdev->ring[fence->ring];
1332         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1333
1334         /* flush read cache over gart for this vmid */
1335         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1336         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1337         radeon_ring_write(ring, 0);
1338         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1339         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1340         radeon_ring_write(ring, 0xFFFFFFFF);
1341         radeon_ring_write(ring, 0);
1342         radeon_ring_write(ring, 10); /* poll interval */
1343         /* EVENT_WRITE_EOP - flush caches, send int */
1344         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1345         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1346         radeon_ring_write(ring, addr & 0xffffffff);
1347         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1348         radeon_ring_write(ring, fence->seq);
1349         radeon_ring_write(ring, 0);
1350 }
1351
1352 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1353 {
1354         struct radeon_ring *ring = &rdev->ring[ib->ring];
1355
1356         /* set to DX10/11 mode */
1357         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1358         radeon_ring_write(ring, 1);
1359
1360         if (ring->rptr_save_reg) {
1361                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1362                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1363                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1364                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1365                 radeon_ring_write(ring, next_rptr);
1366         }
1367
1368         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1369         radeon_ring_write(ring,
1370 #ifdef __BIG_ENDIAN
1371                           (2 << 0) |
1372 #endif
1373                           (ib->gpu_addr & 0xFFFFFFFC));
1374         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1375         radeon_ring_write(ring, ib->length_dw |
1376                           (ib->vm ? (ib->vm->id << 24) : 0));
1377
1378         /* flush read cache over gart for this vmid */
1379         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1380         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1381         radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
1382         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1383         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1384         radeon_ring_write(ring, 0xFFFFFFFF);
1385         radeon_ring_write(ring, 0);
1386         radeon_ring_write(ring, 10); /* poll interval */
1387 }
1388
1389 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1390 {
1391         if (enable)
1392                 WREG32(CP_ME_CNTL, 0);
1393         else {
1394                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1395                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1396                 WREG32(SCRATCH_UMSK, 0);
1397                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1398         }
1399 }
1400
1401 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1402 {
1403         const __be32 *fw_data;
1404         int i;
1405
1406         if (!rdev->me_fw || !rdev->pfp_fw)
1407                 return -EINVAL;
1408
1409         cayman_cp_enable(rdev, false);
1410
1411         fw_data = (const __be32 *)rdev->pfp_fw->data;
1412         WREG32(CP_PFP_UCODE_ADDR, 0);
1413         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1414                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1415         WREG32(CP_PFP_UCODE_ADDR, 0);
1416
1417         fw_data = (const __be32 *)rdev->me_fw->data;
1418         WREG32(CP_ME_RAM_WADDR, 0);
1419         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1420                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1421
1422         WREG32(CP_PFP_UCODE_ADDR, 0);
1423         WREG32(CP_ME_RAM_WADDR, 0);
1424         WREG32(CP_ME_RAM_RADDR, 0);
1425         return 0;
1426 }
1427
1428 static int cayman_cp_start(struct radeon_device *rdev)
1429 {
1430         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1431         int r, i;
1432
1433         r = radeon_ring_lock(rdev, ring, 7);
1434         if (r) {
1435                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1436                 return r;
1437         }
1438         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1439         radeon_ring_write(ring, 0x1);
1440         radeon_ring_write(ring, 0x0);
1441         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1442         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1443         radeon_ring_write(ring, 0);
1444         radeon_ring_write(ring, 0);
1445         radeon_ring_unlock_commit(rdev, ring);
1446
1447         cayman_cp_enable(rdev, true);
1448
1449         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1450         if (r) {
1451                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1452                 return r;
1453         }
1454
1455         /* setup clear context state */
1456         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1457         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1458
1459         for (i = 0; i < cayman_default_size; i++)
1460                 radeon_ring_write(ring, cayman_default_state[i]);
1461
1462         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1463         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1464
1465         /* set clear context state */
1466         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1467         radeon_ring_write(ring, 0);
1468
1469         /* SQ_VTX_BASE_VTX_LOC */
1470         radeon_ring_write(ring, 0xc0026f00);
1471         radeon_ring_write(ring, 0x00000000);
1472         radeon_ring_write(ring, 0x00000000);
1473         radeon_ring_write(ring, 0x00000000);
1474
1475         /* Clear consts */
1476         radeon_ring_write(ring, 0xc0036f00);
1477         radeon_ring_write(ring, 0x00000bc4);
1478         radeon_ring_write(ring, 0xffffffff);
1479         radeon_ring_write(ring, 0xffffffff);
1480         radeon_ring_write(ring, 0xffffffff);
1481
1482         radeon_ring_write(ring, 0xc0026900);
1483         radeon_ring_write(ring, 0x00000316);
1484         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1485         radeon_ring_write(ring, 0x00000010); /*  */
1486
1487         radeon_ring_unlock_commit(rdev, ring);
1488
1489         /* XXX init other rings */
1490
1491         return 0;
1492 }
1493
1494 static void cayman_cp_fini(struct radeon_device *rdev)
1495 {
1496         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1497         cayman_cp_enable(rdev, false);
1498         radeon_ring_fini(rdev, ring);
1499         radeon_scratch_free(rdev, ring->rptr_save_reg);
1500 }
1501
1502 static int cayman_cp_resume(struct radeon_device *rdev)
1503 {
1504         static const int ridx[] = {
1505                 RADEON_RING_TYPE_GFX_INDEX,
1506                 CAYMAN_RING_TYPE_CP1_INDEX,
1507                 CAYMAN_RING_TYPE_CP2_INDEX
1508         };
1509         static const unsigned cp_rb_cntl[] = {
1510                 CP_RB0_CNTL,
1511                 CP_RB1_CNTL,
1512                 CP_RB2_CNTL,
1513         };
1514         static const unsigned cp_rb_rptr_addr[] = {
1515                 CP_RB0_RPTR_ADDR,
1516                 CP_RB1_RPTR_ADDR,
1517                 CP_RB2_RPTR_ADDR
1518         };
1519         static const unsigned cp_rb_rptr_addr_hi[] = {
1520                 CP_RB0_RPTR_ADDR_HI,
1521                 CP_RB1_RPTR_ADDR_HI,
1522                 CP_RB2_RPTR_ADDR_HI
1523         };
1524         static const unsigned cp_rb_base[] = {
1525                 CP_RB0_BASE,
1526                 CP_RB1_BASE,
1527                 CP_RB2_BASE
1528         };
1529         struct radeon_ring *ring;
1530         int i, r;
1531
1532         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1533         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1534                                  SOFT_RESET_PA |
1535                                  SOFT_RESET_SH |
1536                                  SOFT_RESET_VGT |
1537                                  SOFT_RESET_SPI |
1538                                  SOFT_RESET_SX));
1539         RREG32(GRBM_SOFT_RESET);
1540         mdelay(15);
1541         WREG32(GRBM_SOFT_RESET, 0);
1542         RREG32(GRBM_SOFT_RESET);
1543
1544         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1545         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1546
1547         /* Set the write pointer delay */
1548         WREG32(CP_RB_WPTR_DELAY, 0);
1549
1550         WREG32(CP_DEBUG, (1 << 27));
1551
1552         /* set the wb address whether it's enabled or not */
1553         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1554         WREG32(SCRATCH_UMSK, 0xff);
1555
1556         for (i = 0; i < 3; ++i) {
1557                 uint32_t rb_cntl;
1558                 uint64_t addr;
1559
1560                 /* Set ring buffer size */
1561                 ring = &rdev->ring[ridx[i]];
1562                 rb_cntl = order_base_2(ring->ring_size / 8);
1563                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1564 #ifdef __BIG_ENDIAN
1565                 rb_cntl |= BUF_SWAP_32BIT;
1566 #endif
1567                 WREG32(cp_rb_cntl[i], rb_cntl);
1568
1569                 /* set the wb address whether it's enabled or not */
1570                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1571                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1572                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1573         }
1574
1575         /* set the rb base addr, this causes an internal reset of ALL rings */
1576         for (i = 0; i < 3; ++i) {
1577                 ring = &rdev->ring[ridx[i]];
1578                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1579         }
1580
1581         for (i = 0; i < 3; ++i) {
1582                 /* Initialize the ring buffer's read and write pointers */
1583                 ring = &rdev->ring[ridx[i]];
1584                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1585
1586                 ring->rptr = ring->wptr = 0;
1587                 WREG32(ring->rptr_reg, ring->rptr);
1588                 WREG32(ring->wptr_reg, ring->wptr);
1589
1590                 mdelay(1);
1591                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1592         }
1593
1594         /* start the rings */
1595         cayman_cp_start(rdev);
1596         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1597         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1598         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1599         /* this only test cp0 */
1600         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1601         if (r) {
1602                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1603                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1604                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1605                 return r;
1606         }
1607
1608         return 0;
1609 }
1610
1611 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1612 {
1613         u32 reset_mask = 0;
1614         u32 tmp;
1615
1616         /* GRBM_STATUS */
1617         tmp = RREG32(GRBM_STATUS);
1618         if (tmp & (PA_BUSY | SC_BUSY |
1619                    SH_BUSY | SX_BUSY |
1620                    TA_BUSY | VGT_BUSY |
1621                    DB_BUSY | CB_BUSY |
1622                    GDS_BUSY | SPI_BUSY |
1623                    IA_BUSY | IA_BUSY_NO_DMA))
1624                 reset_mask |= RADEON_RESET_GFX;
1625
1626         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1627                    CP_BUSY | CP_COHERENCY_BUSY))
1628                 reset_mask |= RADEON_RESET_CP;
1629
1630         if (tmp & GRBM_EE_BUSY)
1631                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1632
1633         /* DMA_STATUS_REG 0 */
1634         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1635         if (!(tmp & DMA_IDLE))
1636                 reset_mask |= RADEON_RESET_DMA;
1637
1638         /* DMA_STATUS_REG 1 */
1639         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1640         if (!(tmp & DMA_IDLE))
1641                 reset_mask |= RADEON_RESET_DMA1;
1642
1643         /* SRBM_STATUS2 */
1644         tmp = RREG32(SRBM_STATUS2);
1645         if (tmp & DMA_BUSY)
1646                 reset_mask |= RADEON_RESET_DMA;
1647
1648         if (tmp & DMA1_BUSY)
1649                 reset_mask |= RADEON_RESET_DMA1;
1650
1651         /* SRBM_STATUS */
1652         tmp = RREG32(SRBM_STATUS);
1653         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1654                 reset_mask |= RADEON_RESET_RLC;
1655
1656         if (tmp & IH_BUSY)
1657                 reset_mask |= RADEON_RESET_IH;
1658
1659         if (tmp & SEM_BUSY)
1660                 reset_mask |= RADEON_RESET_SEM;
1661
1662         if (tmp & GRBM_RQ_PENDING)
1663                 reset_mask |= RADEON_RESET_GRBM;
1664
1665         if (tmp & VMC_BUSY)
1666                 reset_mask |= RADEON_RESET_VMC;
1667
1668         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1669                    MCC_BUSY | MCD_BUSY))
1670                 reset_mask |= RADEON_RESET_MC;
1671
1672         if (evergreen_is_display_hung(rdev))
1673                 reset_mask |= RADEON_RESET_DISPLAY;
1674
1675         /* VM_L2_STATUS */
1676         tmp = RREG32(VM_L2_STATUS);
1677         if (tmp & L2_BUSY)
1678                 reset_mask |= RADEON_RESET_VMC;
1679
1680         /* Skip MC reset as it's mostly likely not hung, just busy */
1681         if (reset_mask & RADEON_RESET_MC) {
1682                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1683                 reset_mask &= ~RADEON_RESET_MC;
1684         }
1685
1686         return reset_mask;
1687 }
1688
1689 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1690 {
1691         struct evergreen_mc_save save;
1692         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1693         u32 tmp;
1694
1695         if (reset_mask == 0)
1696                 return;
1697
1698         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1699
1700         evergreen_print_gpu_status_regs(rdev);
1701         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1702                  RREG32(0x14F8));
1703         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1704                  RREG32(0x14D8));
1705         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1706                  RREG32(0x14FC));
1707         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1708                  RREG32(0x14DC));
1709
1710         /* Disable CP parsing/prefetching */
1711         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1712
1713         if (reset_mask & RADEON_RESET_DMA) {
1714                 /* dma0 */
1715                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1716                 tmp &= ~DMA_RB_ENABLE;
1717                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1718         }
1719
1720         if (reset_mask & RADEON_RESET_DMA1) {
1721                 /* dma1 */
1722                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1723                 tmp &= ~DMA_RB_ENABLE;
1724                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1725         }
1726
1727         udelay(50);
1728
1729         evergreen_mc_stop(rdev, &save);
1730         if (evergreen_mc_wait_for_idle(rdev)) {
1731                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1732         }
1733
1734         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1735                 grbm_soft_reset = SOFT_RESET_CB |
1736                         SOFT_RESET_DB |
1737                         SOFT_RESET_GDS |
1738                         SOFT_RESET_PA |
1739                         SOFT_RESET_SC |
1740                         SOFT_RESET_SPI |
1741                         SOFT_RESET_SH |
1742                         SOFT_RESET_SX |
1743                         SOFT_RESET_TC |
1744                         SOFT_RESET_TA |
1745                         SOFT_RESET_VGT |
1746                         SOFT_RESET_IA;
1747         }
1748
1749         if (reset_mask & RADEON_RESET_CP) {
1750                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1751
1752                 srbm_soft_reset |= SOFT_RESET_GRBM;
1753         }
1754
1755         if (reset_mask & RADEON_RESET_DMA)
1756                 srbm_soft_reset |= SOFT_RESET_DMA;
1757
1758         if (reset_mask & RADEON_RESET_DMA1)
1759                 srbm_soft_reset |= SOFT_RESET_DMA1;
1760
1761         if (reset_mask & RADEON_RESET_DISPLAY)
1762                 srbm_soft_reset |= SOFT_RESET_DC;
1763
1764         if (reset_mask & RADEON_RESET_RLC)
1765                 srbm_soft_reset |= SOFT_RESET_RLC;
1766
1767         if (reset_mask & RADEON_RESET_SEM)
1768                 srbm_soft_reset |= SOFT_RESET_SEM;
1769
1770         if (reset_mask & RADEON_RESET_IH)
1771                 srbm_soft_reset |= SOFT_RESET_IH;
1772
1773         if (reset_mask & RADEON_RESET_GRBM)
1774                 srbm_soft_reset |= SOFT_RESET_GRBM;
1775
1776         if (reset_mask & RADEON_RESET_VMC)
1777                 srbm_soft_reset |= SOFT_RESET_VMC;
1778
1779         if (!(rdev->flags & RADEON_IS_IGP)) {
1780                 if (reset_mask & RADEON_RESET_MC)
1781                         srbm_soft_reset |= SOFT_RESET_MC;
1782         }
1783
1784         if (grbm_soft_reset) {
1785                 tmp = RREG32(GRBM_SOFT_RESET);
1786                 tmp |= grbm_soft_reset;
1787                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1788                 WREG32(GRBM_SOFT_RESET, tmp);
1789                 tmp = RREG32(GRBM_SOFT_RESET);
1790
1791                 udelay(50);
1792
1793                 tmp &= ~grbm_soft_reset;
1794                 WREG32(GRBM_SOFT_RESET, tmp);
1795                 tmp = RREG32(GRBM_SOFT_RESET);
1796         }
1797
1798         if (srbm_soft_reset) {
1799                 tmp = RREG32(SRBM_SOFT_RESET);
1800                 tmp |= srbm_soft_reset;
1801                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1802                 WREG32(SRBM_SOFT_RESET, tmp);
1803                 tmp = RREG32(SRBM_SOFT_RESET);
1804
1805                 udelay(50);
1806
1807                 tmp &= ~srbm_soft_reset;
1808                 WREG32(SRBM_SOFT_RESET, tmp);
1809                 tmp = RREG32(SRBM_SOFT_RESET);
1810         }
1811
1812         /* Wait a little for things to settle down */
1813         udelay(50);
1814
1815         evergreen_mc_resume(rdev, &save);
1816         udelay(50);
1817
1818         evergreen_print_gpu_status_regs(rdev);
1819 }
1820
1821 int cayman_asic_reset(struct radeon_device *rdev)
1822 {
1823         u32 reset_mask;
1824
1825         reset_mask = cayman_gpu_check_soft_reset(rdev);
1826
1827         if (reset_mask)
1828                 r600_set_bios_scratch_engine_hung(rdev, true);
1829
1830         cayman_gpu_soft_reset(rdev, reset_mask);
1831
1832         reset_mask = cayman_gpu_check_soft_reset(rdev);
1833
1834         if (!reset_mask)
1835                 r600_set_bios_scratch_engine_hung(rdev, false);
1836
1837         return 0;
1838 }
1839
1840 /**
1841  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1842  *
1843  * @rdev: radeon_device pointer
1844  * @ring: radeon_ring structure holding ring information
1845  *
1846  * Check if the GFX engine is locked up.
1847  * Returns true if the engine appears to be locked up, false if not.
1848  */
1849 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1850 {
1851         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1852
1853         if (!(reset_mask & (RADEON_RESET_GFX |
1854                             RADEON_RESET_COMPUTE |
1855                             RADEON_RESET_CP))) {
1856                 radeon_ring_lockup_update(ring);
1857                 return false;
1858         }
1859         /* force CP activities */
1860         radeon_ring_force_activity(rdev, ring);
1861         return radeon_ring_test_lockup(rdev, ring);
1862 }
1863
1864 static int cayman_startup(struct radeon_device *rdev)
1865 {
1866         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1867         int r;
1868
1869         /* enable pcie gen2 link */
1870         evergreen_pcie_gen2_enable(rdev);
1871         /* enable aspm */
1872         evergreen_program_aspm(rdev);
1873
1874         /* scratch needs to be initialized before MC */
1875         r = r600_vram_scratch_init(rdev);
1876         if (r)
1877                 return r;
1878
1879         evergreen_mc_program(rdev);
1880
1881         if (rdev->flags & RADEON_IS_IGP) {
1882                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1883                         r = ni_init_microcode(rdev);
1884                         if (r) {
1885                                 DRM_ERROR("Failed to load firmware!\n");
1886                                 return r;
1887                         }
1888                 }
1889         } else {
1890                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
1891                         r = ni_init_microcode(rdev);
1892                         if (r) {
1893                                 DRM_ERROR("Failed to load firmware!\n");
1894                                 return r;
1895                         }
1896                 }
1897
1898                 r = ni_mc_load_microcode(rdev);
1899                 if (r) {
1900                         DRM_ERROR("Failed to load MC firmware!\n");
1901                         return r;
1902                 }
1903         }
1904
1905         r = cayman_pcie_gart_enable(rdev);
1906         if (r)
1907                 return r;
1908         cayman_gpu_init(rdev);
1909
1910         /* allocate rlc buffers */
1911         if (rdev->flags & RADEON_IS_IGP) {
1912                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1913                 rdev->rlc.reg_list_size =
1914                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1915                 rdev->rlc.cs_data = cayman_cs_data;
1916                 r = sumo_rlc_init(rdev);
1917                 if (r) {
1918                         DRM_ERROR("Failed to init rlc BOs!\n");
1919                         return r;
1920                 }
1921         }
1922
1923         /* allocate wb buffer */
1924         r = radeon_wb_init(rdev);
1925         if (r)
1926                 return r;
1927
1928         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1929         if (r) {
1930                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1931                 return r;
1932         }
1933
1934         r = uvd_v2_2_resume(rdev);
1935         if (!r) {
1936                 r = radeon_fence_driver_start_ring(rdev,
1937                                                    R600_RING_TYPE_UVD_INDEX);
1938                 if (r)
1939                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
1940         }
1941         if (r)
1942                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1943
1944         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
1945         if (r) {
1946                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1947                 return r;
1948         }
1949
1950         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
1951         if (r) {
1952                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1953                 return r;
1954         }
1955
1956         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1957         if (r) {
1958                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1959                 return r;
1960         }
1961
1962         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
1963         if (r) {
1964                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1965                 return r;
1966         }
1967
1968         /* Enable IRQ */
1969         if (!rdev->irq.installed) {
1970                 r = radeon_irq_kms_init(rdev);
1971                 if (r)
1972                         return r;
1973         }
1974
1975         r = r600_irq_init(rdev);
1976         if (r) {
1977                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1978                 radeon_irq_kms_fini(rdev);
1979                 return r;
1980         }
1981         evergreen_irq_set(rdev);
1982
1983         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1984                              CP_RB0_RPTR, CP_RB0_WPTR,
1985                              RADEON_CP_PACKET2);
1986         if (r)
1987                 return r;
1988
1989         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1990         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1991                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
1992                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
1993                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1994         if (r)
1995                 return r;
1996
1997         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1998         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
1999                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
2000                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
2001                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2002         if (r)
2003                 return r;
2004
2005         r = cayman_cp_load_microcode(rdev);
2006         if (r)
2007                 return r;
2008         r = cayman_cp_resume(rdev);
2009         if (r)
2010                 return r;
2011
2012         r = cayman_dma_resume(rdev);
2013         if (r)
2014                 return r;
2015
2016         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2017         if (ring->ring_size) {
2018                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2019                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2020                                      RADEON_CP_PACKET2);
2021                 if (!r)
2022                         r = uvd_v1_0_init(rdev);
2023                 if (r)
2024                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2025         }
2026
2027         r = radeon_ib_pool_init(rdev);
2028         if (r) {
2029                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2030                 return r;
2031         }
2032
2033         r = radeon_vm_manager_init(rdev);
2034         if (r) {
2035                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2036                 return r;
2037         }
2038
2039         if (ASIC_IS_DCE6(rdev)) {
2040                 r = dce6_audio_init(rdev);
2041                 if (r)
2042                         return r;
2043         } else {
2044                 r = r600_audio_init(rdev);
2045                 if (r)
2046                         return r;
2047         }
2048
2049         return 0;
2050 }
2051
2052 int cayman_resume(struct radeon_device *rdev)
2053 {
2054         int r;
2055
2056         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2057          * posting will perform necessary task to bring back GPU into good
2058          * shape.
2059          */
2060         /* post card */
2061         atom_asic_init(rdev->mode_info.atom_context);
2062
2063         /* init golden registers */
2064         ni_init_golden_registers(rdev);
2065
2066         rdev->accel_working = true;
2067         r = cayman_startup(rdev);
2068         if (r) {
2069                 DRM_ERROR("cayman startup failed on resume\n");
2070                 rdev->accel_working = false;
2071                 return r;
2072         }
2073         return r;
2074 }
2075
2076 int cayman_suspend(struct radeon_device *rdev)
2077 {
2078         if (ASIC_IS_DCE6(rdev))
2079                 dce6_audio_fini(rdev);
2080         else
2081                 r600_audio_fini(rdev);
2082         radeon_vm_manager_fini(rdev);
2083         cayman_cp_enable(rdev, false);
2084         cayman_dma_stop(rdev);
2085         uvd_v1_0_fini(rdev);
2086         radeon_uvd_suspend(rdev);
2087         evergreen_irq_suspend(rdev);
2088         radeon_wb_disable(rdev);
2089         cayman_pcie_gart_disable(rdev);
2090         return 0;
2091 }
2092
2093 /* Plan is to move initialization in that function and use
2094  * helper function so that radeon_device_init pretty much
2095  * do nothing more than calling asic specific function. This
2096  * should also allow to remove a bunch of callback function
2097  * like vram_info.
2098  */
2099 int cayman_init(struct radeon_device *rdev)
2100 {
2101         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2102         int r;
2103
2104         /* Read BIOS */
2105         if (!radeon_get_bios(rdev)) {
2106                 if (ASIC_IS_AVIVO(rdev))
2107                         return -EINVAL;
2108         }
2109         /* Must be an ATOMBIOS */
2110         if (!rdev->is_atom_bios) {
2111                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2112                 return -EINVAL;
2113         }
2114         r = radeon_atombios_init(rdev);
2115         if (r)
2116                 return r;
2117
2118         /* Post card if necessary */
2119         if (!radeon_card_posted(rdev)) {
2120                 if (!rdev->bios) {
2121                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2122                         return -EINVAL;
2123                 }
2124                 DRM_INFO("GPU not posted. posting now...\n");
2125                 atom_asic_init(rdev->mode_info.atom_context);
2126         }
2127         /* init golden registers */
2128         ni_init_golden_registers(rdev);
2129         /* Initialize scratch registers */
2130         r600_scratch_init(rdev);
2131         /* Initialize surface registers */
2132         radeon_surface_init(rdev);
2133         /* Initialize clocks */
2134         radeon_get_clock_info(rdev->ddev);
2135         /* Fence driver */
2136         r = radeon_fence_driver_init(rdev);
2137         if (r)
2138                 return r;
2139         /* initialize memory controller */
2140         r = evergreen_mc_init(rdev);
2141         if (r)
2142                 return r;
2143         /* Memory manager */
2144         r = radeon_bo_init(rdev);
2145         if (r)
2146                 return r;
2147
2148         ring->ring_obj = NULL;
2149         r600_ring_init(rdev, ring, 1024 * 1024);
2150
2151         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2152         ring->ring_obj = NULL;
2153         r600_ring_init(rdev, ring, 64 * 1024);
2154
2155         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2156         ring->ring_obj = NULL;
2157         r600_ring_init(rdev, ring, 64 * 1024);
2158
2159         r = radeon_uvd_init(rdev);
2160         if (!r) {
2161                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2162                 ring->ring_obj = NULL;
2163                 r600_ring_init(rdev, ring, 4096);
2164         }
2165
2166         rdev->ih.ring_obj = NULL;
2167         r600_ih_ring_init(rdev, 64 * 1024);
2168
2169         r = r600_pcie_gart_init(rdev);
2170         if (r)
2171                 return r;
2172
2173         rdev->accel_working = true;
2174         r = cayman_startup(rdev);
2175         if (r) {
2176                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2177                 cayman_cp_fini(rdev);
2178                 cayman_dma_fini(rdev);
2179                 r600_irq_fini(rdev);
2180                 if (rdev->flags & RADEON_IS_IGP)
2181                         sumo_rlc_fini(rdev);
2182                 radeon_wb_fini(rdev);
2183                 radeon_ib_pool_fini(rdev);
2184                 radeon_vm_manager_fini(rdev);
2185                 radeon_irq_kms_fini(rdev);
2186                 cayman_pcie_gart_fini(rdev);
2187                 rdev->accel_working = false;
2188         }
2189
2190         /* Don't start up if the MC ucode is missing.
2191          * The default clocks and voltages before the MC ucode
2192          * is loaded are not suffient for advanced operations.
2193          *
2194          * We can skip this check for TN, because there is no MC
2195          * ucode.
2196          */
2197         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2198                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2199                 return -EINVAL;
2200         }
2201
2202         return 0;
2203 }
2204
2205 void cayman_fini(struct radeon_device *rdev)
2206 {
2207         cayman_cp_fini(rdev);
2208         cayman_dma_fini(rdev);
2209         r600_irq_fini(rdev);
2210         if (rdev->flags & RADEON_IS_IGP)
2211                 sumo_rlc_fini(rdev);
2212         radeon_wb_fini(rdev);
2213         radeon_vm_manager_fini(rdev);
2214         radeon_ib_pool_fini(rdev);
2215         radeon_irq_kms_fini(rdev);
2216         uvd_v1_0_fini(rdev);
2217         radeon_uvd_fini(rdev);
2218         cayman_pcie_gart_fini(rdev);
2219         r600_vram_scratch_fini(rdev);
2220         radeon_gem_fini(rdev);
2221         radeon_fence_driver_fini(rdev);
2222         radeon_bo_fini(rdev);
2223         radeon_atombios_fini(rdev);
2224         kfree(rdev->bios);
2225         rdev->bios = NULL;
2226 }
2227
2228 /*
2229  * vm
2230  */
2231 int cayman_vm_init(struct radeon_device *rdev)
2232 {
2233         /* number of VMs */
2234         rdev->vm_manager.nvm = 8;
2235         /* base offset of vram pages */
2236         if (rdev->flags & RADEON_IS_IGP) {
2237                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2238                 tmp <<= 22;
2239                 rdev->vm_manager.vram_base_offset = tmp;
2240         } else
2241                 rdev->vm_manager.vram_base_offset = 0;
2242         return 0;
2243 }
2244
2245 void cayman_vm_fini(struct radeon_device *rdev)
2246 {
2247 }
2248
2249 /**
2250  * cayman_vm_decode_fault - print human readable fault info
2251  *
2252  * @rdev: radeon_device pointer
2253  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2254  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2255  *
2256  * Print human readable fault information (cayman/TN).
2257  */
2258 void cayman_vm_decode_fault(struct radeon_device *rdev,
2259                             u32 status, u32 addr)
2260 {
2261         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2262         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2263         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2264         char *block;
2265
2266         switch (mc_id) {
2267         case 32:
2268         case 16:
2269         case 96:
2270         case 80:
2271         case 160:
2272         case 144:
2273         case 224:
2274         case 208:
2275                 block = "CB";
2276                 break;
2277         case 33:
2278         case 17:
2279         case 97:
2280         case 81:
2281         case 161:
2282         case 145:
2283         case 225:
2284         case 209:
2285                 block = "CB_FMASK";
2286                 break;
2287         case 34:
2288         case 18:
2289         case 98:
2290         case 82:
2291         case 162:
2292         case 146:
2293         case 226:
2294         case 210:
2295                 block = "CB_CMASK";
2296                 break;
2297         case 35:
2298         case 19:
2299         case 99:
2300         case 83:
2301         case 163:
2302         case 147:
2303         case 227:
2304         case 211:
2305                 block = "CB_IMMED";
2306                 break;
2307         case 36:
2308         case 20:
2309         case 100:
2310         case 84:
2311         case 164:
2312         case 148:
2313         case 228:
2314         case 212:
2315                 block = "DB";
2316                 break;
2317         case 37:
2318         case 21:
2319         case 101:
2320         case 85:
2321         case 165:
2322         case 149:
2323         case 229:
2324         case 213:
2325                 block = "DB_HTILE";
2326                 break;
2327         case 38:
2328         case 22:
2329         case 102:
2330         case 86:
2331         case 166:
2332         case 150:
2333         case 230:
2334         case 214:
2335                 block = "SX";
2336                 break;
2337         case 39:
2338         case 23:
2339         case 103:
2340         case 87:
2341         case 167:
2342         case 151:
2343         case 231:
2344         case 215:
2345                 block = "DB_STEN";
2346                 break;
2347         case 40:
2348         case 24:
2349         case 104:
2350         case 88:
2351         case 232:
2352         case 216:
2353         case 168:
2354         case 152:
2355                 block = "TC_TFETCH";
2356                 break;
2357         case 41:
2358         case 25:
2359         case 105:
2360         case 89:
2361         case 233:
2362         case 217:
2363         case 169:
2364         case 153:
2365                 block = "TC_VFETCH";
2366                 break;
2367         case 42:
2368         case 26:
2369         case 106:
2370         case 90:
2371         case 234:
2372         case 218:
2373         case 170:
2374         case 154:
2375                 block = "VC";
2376                 break;
2377         case 112:
2378                 block = "CP";
2379                 break;
2380         case 113:
2381         case 114:
2382                 block = "SH";
2383                 break;
2384         case 115:
2385                 block = "VGT";
2386                 break;
2387         case 178:
2388                 block = "IH";
2389                 break;
2390         case 51:
2391                 block = "RLC";
2392                 break;
2393         case 55:
2394                 block = "DMA";
2395                 break;
2396         case 56:
2397                 block = "HDP";
2398                 break;
2399         default:
2400                 block = "unknown";
2401                 break;
2402         }
2403
2404         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2405                protections, vmid, addr,
2406                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2407                block, mc_id);
2408 }
2409
2410 /**
2411  * cayman_vm_flush - vm flush using the CP
2412  *
2413  * @rdev: radeon_device pointer
2414  *
2415  * Update the page table base and flush the VM TLB
2416  * using the CP (cayman-si).
2417  */
2418 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2419 {
2420         struct radeon_ring *ring = &rdev->ring[ridx];
2421
2422         if (vm == NULL)
2423                 return;
2424
2425         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2426         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2427
2428         /* flush hdp cache */
2429         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2430         radeon_ring_write(ring, 0x1);
2431
2432         /* bits 0-7 are the VM contexts0-7 */
2433         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2434         radeon_ring_write(ring, 1 << vm->id);
2435
2436         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2437         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2438         radeon_ring_write(ring, 0x0);
2439 }