drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "drmP.h"
  25 #include "radeon.h"
  26 #include "nid.h"
  27 #include "r600_dpm.h"
  28 #include "ni_dpm.h"
  29 #include "atom.h"
  30 #include <linux/math64.h>
  31 #include <linux/seq_file.h>
  32
  33 #define MC_CG_ARB_FREQ_F0           0x0a
  34 #define MC_CG_ARB_FREQ_F1           0x0b
  35 #define MC_CG_ARB_FREQ_F2           0x0c
  36 #define MC_CG_ARB_FREQ_F3           0x0d
  37
  38 #define SMC_RAM_END 0xC000
  39
  40 static const struct ni_cac_weights cac_weights_cayman_xt =
  41 {
  42         0x15,
  43         0x2,
  44         0x19,
  45         0x2,
  46         0x8,
  47         0x14,
  48         0x2,
  49         0x16,
  50         0xE,
  51         0x17,
  52         0x13,
  53         0x2B,
  54         0x10,
  55         0x7,
  56         0x5,
  57         0x5,
  58         0x5,
  59         0x2,
  60         0x3,
  61         0x9,
  62         0x10,
  63         0x10,
  64         0x2B,
  65         0xA,
  66         0x9,
  67         0x4,
  68         0xD,
  69         0xD,
  70         0x3E,
  71         0x18,
  72         0x14,
  73         0,
  74         0x3,
  75         0x3,
  76         0x5,
  77         0,
  78         0x2,
  79         0,
  80         0,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0x1CC,
  89         0,
  90         0x164,
  91         1,
  92         1,
  93         1,
  94         1,
  95         12,
  96         12,
  97         12,
  98         0x12,
  99         0x1F,
 100         132,
 101         5,
 102         7,
 103         0,
 104         { 0, 0, 0, 0, 0, 0, 0, 0 },
 105         { 0, 0, 0, 0 },
 106         true
 107 };
 108
 109 static const struct ni_cac_weights cac_weights_cayman_pro =
 110 {
 111         0x16,
 112         0x4,
 113         0x10,
 114         0x2,
 115         0xA,
 116         0x16,
 117         0x2,
 118         0x18,
 119         0x10,
 120         0x1A,
 121         0x16,
 122         0x2D,
 123         0x12,
 124         0xA,
 125         0x6,
 126         0x6,
 127         0x6,
 128         0x2,
 129         0x4,
 130         0xB,
 131         0x11,
 132         0x11,
 133         0x2D,
 134         0xC,
 135         0xC,
 136         0x7,
 137         0x10,
 138         0x10,
 139         0x3F,
 140         0x1A,
 141         0x16,
 142         0,
 143         0x7,
 144         0x4,
 145         0x6,
 146         1,
 147         0x2,
 148         0x1,
 149         0,
 150         0,
 151         0,
 152         0,
 153         0,
 154         0,
 155         0x30,
 156         0,
 157         0x1CF,
 158         0,
 159         0x166,
 160         1,
 161         1,
 162         1,
 163         1,
 164         12,
 165         12,
 166         12,
 167         0x15,
 168         0x1F,
 169         132,
 170         6,
 171         6,
 172         0,
 173         { 0, 0, 0, 0, 0, 0, 0, 0 },
 174         { 0, 0, 0, 0 },
 175         true
 176 };
 177
 178 static const struct ni_cac_weights cac_weights_cayman_le =
 179 {
 180         0x7,
 181         0xE,
 182         0x1,
 183         0xA,
 184         0x1,
 185         0x3F,
 186         0x2,
 187         0x18,
 188         0x10,
 189         0x1A,
 190         0x1,
 191         0x3F,
 192         0x1,
 193         0xE,
 194         0x6,
 195         0x6,
 196         0x6,
 197         0x2,
 198         0x4,
 199         0x9,
 200         0x1A,
 201         0x1A,
 202         0x2C,
 203         0xA,
 204         0x11,
 205         0x8,
 206         0x19,
 207         0x19,
 208         0x1,
 209         0x1,
 210         0x1A,
 211         0,
 212         0x8,
 213         0x5,
 214         0x8,
 215         0x1,
 216         0x3,
 217         0x1,
 218         0,
 219         0,
 220         0,
 221         0,
 222         0,
 223         0,
 224         0x38,
 225         0x38,
 226         0x239,
 227         0x3,
 228         0x18A,
 229         1,
 230         1,
 231         1,
 232         1,
 233         12,
 234         12,
 235         12,
 236         0x15,
 237         0x22,
 238         132,
 239         6,
 240         6,
 241         0,
 242         { 0, 0, 0, 0, 0, 0, 0, 0 },
 243         { 0, 0, 0, 0 },
 244         true
 245 };
 246
 247 #define NISLANDS_MGCG_SEQUENCE  300
 248
 249 static const u32 cayman_cgcg_cgls_default[] =
 250 {
 251         0x000008f8, 0x00000010, 0xffffffff,
 252         0x000008fc, 0x00000000, 0xffffffff,
 253         0x000008f8, 0x00000011, 0xffffffff,
 254         0x000008fc, 0x00000000, 0xffffffff,
 255         0x000008f8, 0x00000012, 0xffffffff,
 256         0x000008fc, 0x00000000, 0xffffffff,
 257         0x000008f8, 0x00000013, 0xffffffff,
 258         0x000008fc, 0x00000000, 0xffffffff,
 259         0x000008f8, 0x00000014, 0xffffffff,
 260         0x000008fc, 0x00000000, 0xffffffff,
 261         0x000008f8, 0x00000015, 0xffffffff,
 262         0x000008fc, 0x00000000, 0xffffffff,
 263         0x000008f8, 0x00000016, 0xffffffff,
 264         0x000008fc, 0x00000000, 0xffffffff,
 265         0x000008f8, 0x00000017, 0xffffffff,
 266         0x000008fc, 0x00000000, 0xffffffff,
 267         0x000008f8, 0x00000018, 0xffffffff,
 268         0x000008fc, 0x00000000, 0xffffffff,
 269         0x000008f8, 0x00000019, 0xffffffff,
 270         0x000008fc, 0x00000000, 0xffffffff,
 271         0x000008f8, 0x0000001a, 0xffffffff,
 272         0x000008fc, 0x00000000, 0xffffffff,
 273         0x000008f8, 0x0000001b, 0xffffffff,
 274         0x000008fc, 0x00000000, 0xffffffff,
 275         0x000008f8, 0x00000020, 0xffffffff,
 276         0x000008fc, 0x00000000, 0xffffffff,
 277         0x000008f8, 0x00000021, 0xffffffff,
 278         0x000008fc, 0x00000000, 0xffffffff,
 279         0x000008f8, 0x00000022, 0xffffffff,
 280         0x000008fc, 0x00000000, 0xffffffff,
 281         0x000008f8, 0x00000023, 0xffffffff,
 282         0x000008fc, 0x00000000, 0xffffffff,
 283         0x000008f8, 0x00000024, 0xffffffff,
 284         0x000008fc, 0x00000000, 0xffffffff,
 285         0x000008f8, 0x00000025, 0xffffffff,
 286         0x000008fc, 0x00000000, 0xffffffff,
 287         0x000008f8, 0x00000026, 0xffffffff,
 288         0x000008fc, 0x00000000, 0xffffffff,
 289         0x000008f8, 0x00000027, 0xffffffff,
 290         0x000008fc, 0x00000000, 0xffffffff,
 291         0x000008f8, 0x00000028, 0xffffffff,
 292         0x000008fc, 0x00000000, 0xffffffff,
 293         0x000008f8, 0x00000029, 0xffffffff,
 294         0x000008fc, 0x00000000, 0xffffffff,
 295         0x000008f8, 0x0000002a, 0xffffffff,
 296         0x000008fc, 0x00000000, 0xffffffff,
 297         0x000008f8, 0x0000002b, 0xffffffff,
 298         0x000008fc, 0x00000000, 0xffffffff
 299 };
 300 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 301
 302 static const u32 cayman_cgcg_cgls_disable[] =
 303 {
 304         0x000008f8, 0x00000010, 0xffffffff,
 305         0x000008fc, 0xffffffff, 0xffffffff,
 306         0x000008f8, 0x00000011, 0xffffffff,
 307         0x000008fc, 0xffffffff, 0xffffffff,
 308         0x000008f8, 0x00000012, 0xffffffff,
 309         0x000008fc, 0xffffffff, 0xffffffff,
 310         0x000008f8, 0x00000013, 0xffffffff,
 311         0x000008fc, 0xffffffff, 0xffffffff,
 312         0x000008f8, 0x00000014, 0xffffffff,
 313         0x000008fc, 0xffffffff, 0xffffffff,
 314         0x000008f8, 0x00000015, 0xffffffff,
 315         0x000008fc, 0xffffffff, 0xffffffff,
 316         0x000008f8, 0x00000016, 0xffffffff,
 317         0x000008fc, 0xffffffff, 0xffffffff,
 318         0x000008f8, 0x00000017, 0xffffffff,
 319         0x000008fc, 0xffffffff, 0xffffffff,
 320         0x000008f8, 0x00000018, 0xffffffff,
 321         0x000008fc, 0xffffffff, 0xffffffff,
 322         0x000008f8, 0x00000019, 0xffffffff,
 323         0x000008fc, 0xffffffff, 0xffffffff,
 324         0x000008f8, 0x0000001a, 0xffffffff,
 325         0x000008fc, 0xffffffff, 0xffffffff,
 326         0x000008f8, 0x0000001b, 0xffffffff,
 327         0x000008fc, 0xffffffff, 0xffffffff,
 328         0x000008f8, 0x00000020, 0xffffffff,
 329         0x000008fc, 0x00000000, 0xffffffff,
 330         0x000008f8, 0x00000021, 0xffffffff,
 331         0x000008fc, 0x00000000, 0xffffffff,
 332         0x000008f8, 0x00000022, 0xffffffff,
 333         0x000008fc, 0x00000000, 0xffffffff,
 334         0x000008f8, 0x00000023, 0xffffffff,
 335         0x000008fc, 0x00000000, 0xffffffff,
 336         0x000008f8, 0x00000024, 0xffffffff,
 337         0x000008fc, 0x00000000, 0xffffffff,
 338         0x000008f8, 0x00000025, 0xffffffff,
 339         0x000008fc, 0x00000000, 0xffffffff,
 340         0x000008f8, 0x00000026, 0xffffffff,
 341         0x000008fc, 0x00000000, 0xffffffff,
 342         0x000008f8, 0x00000027, 0xffffffff,
 343         0x000008fc, 0x00000000, 0xffffffff,
 344         0x000008f8, 0x00000028, 0xffffffff,
 345         0x000008fc, 0x00000000, 0xffffffff,
 346         0x000008f8, 0x00000029, 0xffffffff,
 347         0x000008fc, 0x00000000, 0xffffffff,
 348         0x000008f8, 0x0000002a, 0xffffffff,
 349         0x000008fc, 0x00000000, 0xffffffff,
 350         0x000008f8, 0x0000002b, 0xffffffff,
 351         0x000008fc, 0x00000000, 0xffffffff,
 352         0x00000644, 0x000f7902, 0x001f4180,
 353         0x00000644, 0x000f3802, 0x001f4180
 354 };
 355 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 356
 357 static const u32 cayman_cgcg_cgls_enable[] =
 358 {
 359         0x00000644, 0x000f7882, 0x001f4080,
 360         0x000008f8, 0x00000010, 0xffffffff,
 361         0x000008fc, 0x00000000, 0xffffffff,
 362         0x000008f8, 0x00000011, 0xffffffff,
 363         0x000008fc, 0x00000000, 0xffffffff,
 364         0x000008f8, 0x00000012, 0xffffffff,
 365         0x000008fc, 0x00000000, 0xffffffff,
 366         0x000008f8, 0x00000013, 0xffffffff,
 367         0x000008fc, 0x00000000, 0xffffffff,
 368         0x000008f8, 0x00000014, 0xffffffff,
 369         0x000008fc, 0x00000000, 0xffffffff,
 370         0x000008f8, 0x00000015, 0xffffffff,
 371         0x000008fc, 0x00000000, 0xffffffff,
 372         0x000008f8, 0x00000016, 0xffffffff,
 373         0x000008fc, 0x00000000, 0xffffffff,
 374         0x000008f8, 0x00000017, 0xffffffff,
 375         0x000008fc, 0x00000000, 0xffffffff,
 376         0x000008f8, 0x00000018, 0xffffffff,
 377         0x000008fc, 0x00000000, 0xffffffff,
 378         0x000008f8, 0x00000019, 0xffffffff,
 379         0x000008fc, 0x00000000, 0xffffffff,
 380         0x000008f8, 0x0000001a, 0xffffffff,
 381         0x000008fc, 0x00000000, 0xffffffff,
 382         0x000008f8, 0x0000001b, 0xffffffff,
 383         0x000008fc, 0x00000000, 0xffffffff,
 384         0x000008f8, 0x00000020, 0xffffffff,
 385         0x000008fc, 0xffffffff, 0xffffffff,
 386         0x000008f8, 0x00000021, 0xffffffff,
 387         0x000008fc, 0xffffffff, 0xffffffff,
 388         0x000008f8, 0x00000022, 0xffffffff,
 389         0x000008fc, 0xffffffff, 0xffffffff,
 390         0x000008f8, 0x00000023, 0xffffffff,
 391         0x000008fc, 0xffffffff, 0xffffffff,
 392         0x000008f8, 0x00000024, 0xffffffff,
 393         0x000008fc, 0xffffffff, 0xffffffff,
 394         0x000008f8, 0x00000025, 0xffffffff,
 395         0x000008fc, 0xffffffff, 0xffffffff,
 396         0x000008f8, 0x00000026, 0xffffffff,
 397         0x000008fc, 0xffffffff, 0xffffffff,
 398         0x000008f8, 0x00000027, 0xffffffff,
 399         0x000008fc, 0xffffffff, 0xffffffff,
 400         0x000008f8, 0x00000028, 0xffffffff,
 401         0x000008fc, 0xffffffff, 0xffffffff,
 402         0x000008f8, 0x00000029, 0xffffffff,
 403         0x000008fc, 0xffffffff, 0xffffffff,
 404         0x000008f8, 0x0000002a, 0xffffffff,
 405         0x000008fc, 0xffffffff, 0xffffffff,
 406         0x000008f8, 0x0000002b, 0xffffffff,
 407         0x000008fc, 0xffffffff, 0xffffffff
 408 };
 409 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 410
 411 static const u32 cayman_mgcg_default[] =
 412 {
 413         0x0000802c, 0xc0000000, 0xffffffff,
 414         0x00003fc4, 0xc0000000, 0xffffffff,
 415         0x00005448, 0x00000100, 0xffffffff,
 416         0x000055e4, 0x00000100, 0xffffffff,
 417         0x0000160c, 0x00000100, 0xffffffff,
 418         0x00008984, 0x06000100, 0xffffffff,
 419         0x0000c164, 0x00000100, 0xffffffff,
 420         0x00008a18, 0x00000100, 0xffffffff,
 421         0x0000897c, 0x06000100, 0xffffffff,
 422         0x00008b28, 0x00000100, 0xffffffff,
 423         0x00009144, 0x00800200, 0xffffffff,
 424         0x00009a60, 0x00000100, 0xffffffff,
 425         0x00009868, 0x00000100, 0xffffffff,
 426         0x00008d58, 0x00000100, 0xffffffff,
 427         0x00009510, 0x00000100, 0xffffffff,
 428         0x0000949c, 0x00000100, 0xffffffff,
 429         0x00009654, 0x00000100, 0xffffffff,
 430         0x00009030, 0x00000100, 0xffffffff,
 431         0x00009034, 0x00000100, 0xffffffff,
 432         0x00009038, 0x00000100, 0xffffffff,
 433         0x0000903c, 0x00000100, 0xffffffff,
 434         0x00009040, 0x00000100, 0xffffffff,
 435         0x0000a200, 0x00000100, 0xffffffff,
 436         0x0000a204, 0x00000100, 0xffffffff,
 437         0x0000a208, 0x00000100, 0xffffffff,
 438         0x0000a20c, 0x00000100, 0xffffffff,
 439         0x00009744, 0x00000100, 0xffffffff,
 440         0x00003f80, 0x00000100, 0xffffffff,
 441         0x0000a210, 0x00000100, 0xffffffff,
 442         0x0000a214, 0x00000100, 0xffffffff,
 443         0x000004d8, 0x00000100, 0xffffffff,
 444         0x00009664, 0x00000100, 0xffffffff,
 445         0x00009698, 0x00000100, 0xffffffff,
 446         0x000004d4, 0x00000200, 0xffffffff,
 447         0x000004d0, 0x00000000, 0xffffffff,
 448         0x000030cc, 0x00000104, 0xffffffff,
 449         0x0000d0c0, 0x00000100, 0xffffffff,
 450         0x0000d8c0, 0x00000100, 0xffffffff,
 451         0x0000802c, 0x40000000, 0xffffffff,
 452         0x00003fc4, 0x40000000, 0xffffffff,
 453         0x0000915c, 0x00010000, 0xffffffff,
 454         0x00009160, 0x00030002, 0xffffffff,
 455         0x00009164, 0x00050004, 0xffffffff,
 456         0x00009168, 0x00070006, 0xffffffff,
 457         0x00009178, 0x00070000, 0xffffffff,
 458         0x0000917c, 0x00030002, 0xffffffff,
 459         0x00009180, 0x00050004, 0xffffffff,
 460         0x0000918c, 0x00010006, 0xffffffff,
 461         0x00009190, 0x00090008, 0xffffffff,
 462         0x00009194, 0x00070000, 0xffffffff,
 463         0x00009198, 0x00030002, 0xffffffff,
 464         0x0000919c, 0x00050004, 0xffffffff,
 465         0x000091a8, 0x00010006, 0xffffffff,
 466         0x000091ac, 0x00090008, 0xffffffff,
 467         0x000091b0, 0x00070000, 0xffffffff,
 468         0x000091b4, 0x00030002, 0xffffffff,
 469         0x000091b8, 0x00050004, 0xffffffff,
 470         0x000091c4, 0x00010006, 0xffffffff,
 471         0x000091c8, 0x00090008, 0xffffffff,
 472         0x000091cc, 0x00070000, 0xffffffff,
 473         0x000091d0, 0x00030002, 0xffffffff,
 474         0x000091d4, 0x00050004, 0xffffffff,
 475         0x000091e0, 0x00010006, 0xffffffff,
 476         0x000091e4, 0x00090008, 0xffffffff,
 477         0x000091e8, 0x00000000, 0xffffffff,
 478         0x000091ec, 0x00070000, 0xffffffff,
 479         0x000091f0, 0x00030002, 0xffffffff,
 480         0x000091f4, 0x00050004, 0xffffffff,
 481         0x00009200, 0x00010006, 0xffffffff,
 482         0x00009204, 0x00090008, 0xffffffff,
 483         0x00009208, 0x00070000, 0xffffffff,
 484         0x0000920c, 0x00030002, 0xffffffff,
 485         0x00009210, 0x00050004, 0xffffffff,
 486         0x0000921c, 0x00010006, 0xffffffff,
 487         0x00009220, 0x00090008, 0xffffffff,
 488         0x00009224, 0x00070000, 0xffffffff,
 489         0x00009228, 0x00030002, 0xffffffff,
 490         0x0000922c, 0x00050004, 0xffffffff,
 491         0x00009238, 0x00010006, 0xffffffff,
 492         0x0000923c, 0x00090008, 0xffffffff,
 493         0x00009240, 0x00070000, 0xffffffff,
 494         0x00009244, 0x00030002, 0xffffffff,
 495         0x00009248, 0x00050004, 0xffffffff,
 496         0x00009254, 0x00010006, 0xffffffff,
 497         0x00009258, 0x00090008, 0xffffffff,
 498         0x0000925c, 0x00070000, 0xffffffff,
 499         0x00009260, 0x00030002, 0xffffffff,
 500         0x00009264, 0x00050004, 0xffffffff,
 501         0x00009270, 0x00010006, 0xffffffff,
 502         0x00009274, 0x00090008, 0xffffffff,
 503         0x00009278, 0x00070000, 0xffffffff,
 504         0x0000927c, 0x00030002, 0xffffffff,
 505         0x00009280, 0x00050004, 0xffffffff,
 506         0x0000928c, 0x00010006, 0xffffffff,
 507         0x00009290, 0x00090008, 0xffffffff,
 508         0x000092a8, 0x00070000, 0xffffffff,
 509         0x000092ac, 0x00030002, 0xffffffff,
 510         0x000092b0, 0x00050004, 0xffffffff,
 511         0x000092bc, 0x00010006, 0xffffffff,
 512         0x000092c0, 0x00090008, 0xffffffff,
 513         0x000092c4, 0x00070000, 0xffffffff,
 514         0x000092c8, 0x00030002, 0xffffffff,
 515         0x000092cc, 0x00050004, 0xffffffff,
 516         0x000092d8, 0x00010006, 0xffffffff,
 517         0x000092dc, 0x00090008, 0xffffffff,
 518         0x00009294, 0x00000000, 0xffffffff,
 519         0x0000802c, 0x40010000, 0xffffffff,
 520         0x00003fc4, 0x40010000, 0xffffffff,
 521         0x0000915c, 0x00010000, 0xffffffff,
 522         0x00009160, 0x00030002, 0xffffffff,
 523         0x00009164, 0x00050004, 0xffffffff,
 524         0x00009168, 0x00070006, 0xffffffff,
 525         0x00009178, 0x00070000, 0xffffffff,
 526         0x0000917c, 0x00030002, 0xffffffff,
 527         0x00009180, 0x00050004, 0xffffffff,
 528         0x0000918c, 0x00010006, 0xffffffff,
 529         0x00009190, 0x00090008, 0xffffffff,
 530         0x00009194, 0x00070000, 0xffffffff,
 531         0x00009198, 0x00030002, 0xffffffff,
 532         0x0000919c, 0x00050004, 0xffffffff,
 533         0x000091a8, 0x00010006, 0xffffffff,
 534         0x000091ac, 0x00090008, 0xffffffff,
 535         0x000091b0, 0x00070000, 0xffffffff,
 536         0x000091b4, 0x00030002, 0xffffffff,
 537         0x000091b8, 0x00050004, 0xffffffff,
 538         0x000091c4, 0x00010006, 0xffffffff,
 539         0x000091c8, 0x00090008, 0xffffffff,
 540         0x000091cc, 0x00070000, 0xffffffff,
 541         0x000091d0, 0x00030002, 0xffffffff,
 542         0x000091d4, 0x00050004, 0xffffffff,
 543         0x000091e0, 0x00010006, 0xffffffff,
 544         0x000091e4, 0x00090008, 0xffffffff,
 545         0x000091e8, 0x00000000, 0xffffffff,
 546         0x000091ec, 0x00070000, 0xffffffff,
 547         0x000091f0, 0x00030002, 0xffffffff,
 548         0x000091f4, 0x00050004, 0xffffffff,
 549         0x00009200, 0x00010006, 0xffffffff,
 550         0x00009204, 0x00090008, 0xffffffff,
 551         0x00009208, 0x00070000, 0xffffffff,
 552         0x0000920c, 0x00030002, 0xffffffff,
 553         0x00009210, 0x00050004, 0xffffffff,
 554         0x0000921c, 0x00010006, 0xffffffff,
 555         0x00009220, 0x00090008, 0xffffffff,
 556         0x00009224, 0x00070000, 0xffffffff,
 557         0x00009228, 0x00030002, 0xffffffff,
 558         0x0000922c, 0x00050004, 0xffffffff,
 559         0x00009238, 0x00010006, 0xffffffff,
 560         0x0000923c, 0x00090008, 0xffffffff,
 561         0x00009240, 0x00070000, 0xffffffff,
 562         0x00009244, 0x00030002, 0xffffffff,
 563         0x00009248, 0x00050004, 0xffffffff,
 564         0x00009254, 0x00010006, 0xffffffff,
 565         0x00009258, 0x00090008, 0xffffffff,
 566         0x0000925c, 0x00070000, 0xffffffff,
 567         0x00009260, 0x00030002, 0xffffffff,
 568         0x00009264, 0x00050004, 0xffffffff,
 569         0x00009270, 0x00010006, 0xffffffff,
 570         0x00009274, 0x00090008, 0xffffffff,
 571         0x00009278, 0x00070000, 0xffffffff,
 572         0x0000927c, 0x00030002, 0xffffffff,
 573         0x00009280, 0x00050004, 0xffffffff,
 574         0x0000928c, 0x00010006, 0xffffffff,
 575         0x00009290, 0x00090008, 0xffffffff,
 576         0x000092a8, 0x00070000, 0xffffffff,
 577         0x000092ac, 0x00030002, 0xffffffff,
 578         0x000092b0, 0x00050004, 0xffffffff,
 579         0x000092bc, 0x00010006, 0xffffffff,
 580         0x000092c0, 0x00090008, 0xffffffff,
 581         0x000092c4, 0x00070000, 0xffffffff,
 582         0x000092c8, 0x00030002, 0xffffffff,
 583         0x000092cc, 0x00050004, 0xffffffff,
 584         0x000092d8, 0x00010006, 0xffffffff,
 585         0x000092dc, 0x00090008, 0xffffffff,
 586         0x00009294, 0x00000000, 0xffffffff,
 587         0x0000802c, 0xc0000000, 0xffffffff,
 588         0x00003fc4, 0xc0000000, 0xffffffff,
 589         0x000008f8, 0x00000010, 0xffffffff,
 590         0x000008fc, 0x00000000, 0xffffffff,
 591         0x000008f8, 0x00000011, 0xffffffff,
 592         0x000008fc, 0x00000000, 0xffffffff,
 593         0x000008f8, 0x00000012, 0xffffffff,
 594         0x000008fc, 0x00000000, 0xffffffff,
 595         0x000008f8, 0x00000013, 0xffffffff,
 596         0x000008fc, 0x00000000, 0xffffffff,
 597         0x000008f8, 0x00000014, 0xffffffff,
 598         0x000008fc, 0x00000000, 0xffffffff,
 599         0x000008f8, 0x00000015, 0xffffffff,
 600         0x000008fc, 0x00000000, 0xffffffff,
 601         0x000008f8, 0x00000016, 0xffffffff,
 602         0x000008fc, 0x00000000, 0xffffffff,
 603         0x000008f8, 0x00000017, 0xffffffff,
 604         0x000008fc, 0x00000000, 0xffffffff,
 605         0x000008f8, 0x00000018, 0xffffffff,
 606         0x000008fc, 0x00000000, 0xffffffff,
 607         0x000008f8, 0x00000019, 0xffffffff,
 608         0x000008fc, 0x00000000, 0xffffffff,
 609         0x000008f8, 0x0000001a, 0xffffffff,
 610         0x000008fc, 0x00000000, 0xffffffff,
 611         0x000008f8, 0x0000001b, 0xffffffff,
 612         0x000008fc, 0x00000000, 0xffffffff
 613 };
 614 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 615
 616 static const u32 cayman_mgcg_disable[] =
 617 {
 618         0x0000802c, 0xc0000000, 0xffffffff,
 619         0x000008f8, 0x00000000, 0xffffffff,
 620         0x000008fc, 0xffffffff, 0xffffffff,
 621         0x000008f8, 0x00000001, 0xffffffff,
 622         0x000008fc, 0xffffffff, 0xffffffff,
 623         0x000008f8, 0x00000002, 0xffffffff,
 624         0x000008fc, 0xffffffff, 0xffffffff,
 625         0x000008f8, 0x00000003, 0xffffffff,
 626         0x000008fc, 0xffffffff, 0xffffffff,
 627         0x00009150, 0x00600000, 0xffffffff
 628 };
 629 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 630
 631 static const u32 cayman_mgcg_enable[] =
 632 {
 633         0x0000802c, 0xc0000000, 0xffffffff,
 634         0x000008f8, 0x00000000, 0xffffffff,
 635         0x000008fc, 0x00000000, 0xffffffff,
 636         0x000008f8, 0x00000001, 0xffffffff,
 637         0x000008fc, 0x00000000, 0xffffffff,
 638         0x000008f8, 0x00000002, 0xffffffff,
 639         0x000008fc, 0x00600000, 0xffffffff,
 640         0x000008f8, 0x00000003, 0xffffffff,
 641         0x000008fc, 0x00000000, 0xffffffff,
 642         0x00009150, 0x96944200, 0xffffffff
 643 };
 644
 645 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 646
 647 #define NISLANDS_SYSLS_SEQUENCE  100
 648
 649 static const u32 cayman_sysls_default[] =
 650 {
 651         /* Register,   Value,     Mask bits */
 652         0x000055e8, 0x00000000, 0xffffffff,
 653         0x0000d0bc, 0x00000000, 0xffffffff,
 654         0x0000d8bc, 0x00000000, 0xffffffff,
 655         0x000015c0, 0x000c1401, 0xffffffff,
 656         0x0000264c, 0x000c0400, 0xffffffff,
 657         0x00002648, 0x000c0400, 0xffffffff,
 658         0x00002650, 0x000c0400, 0xffffffff,
 659         0x000020b8, 0x000c0400, 0xffffffff,
 660         0x000020bc, 0x000c0400, 0xffffffff,
 661         0x000020c0, 0x000c0c80, 0xffffffff,
 662         0x0000f4a0, 0x000000c0, 0xffffffff,
 663         0x0000f4a4, 0x00680fff, 0xffffffff,
 664         0x00002f50, 0x00000404, 0xffffffff,
 665         0x000004c8, 0x00000001, 0xffffffff,
 666         0x000064ec, 0x00000000, 0xffffffff,
 667         0x00000c7c, 0x00000000, 0xffffffff,
 668         0x00008dfc, 0x00000000, 0xffffffff
 669 };
 670 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 671
 672 static const u32 cayman_sysls_disable[] =
 673 {
 674         /* Register,   Value,     Mask bits */
 675         0x0000d0c0, 0x00000000, 0xffffffff,
 676         0x0000d8c0, 0x00000000, 0xffffffff,
 677         0x000055e8, 0x00000000, 0xffffffff,
 678         0x0000d0bc, 0x00000000, 0xffffffff,
 679         0x0000d8bc, 0x00000000, 0xffffffff,
 680         0x000015c0, 0x00041401, 0xffffffff,
 681         0x0000264c, 0x00040400, 0xffffffff,
 682         0x00002648, 0x00040400, 0xffffffff,
 683         0x00002650, 0x00040400, 0xffffffff,
 684         0x000020b8, 0x00040400, 0xffffffff,
 685         0x000020bc, 0x00040400, 0xffffffff,
 686         0x000020c0, 0x00040c80, 0xffffffff,
 687         0x0000f4a0, 0x000000c0, 0xffffffff,
 688         0x0000f4a4, 0x00680000, 0xffffffff,
 689         0x00002f50, 0x00000404, 0xffffffff,
 690         0x000004c8, 0x00000001, 0xffffffff,
 691         0x000064ec, 0x00007ffd, 0xffffffff,
 692         0x00000c7c, 0x0000ff00, 0xffffffff,
 693         0x00008dfc, 0x0000007f, 0xffffffff
 694 };
 695 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 696
 697 static const u32 cayman_sysls_enable[] =
 698 {
 699         /* Register,   Value,     Mask bits */
 700         0x000055e8, 0x00000001, 0xffffffff,
 701         0x0000d0bc, 0x00000100, 0xffffffff,
 702         0x0000d8bc, 0x00000100, 0xffffffff,
 703         0x000015c0, 0x000c1401, 0xffffffff,
 704         0x0000264c, 0x000c0400, 0xffffffff,
 705         0x00002648, 0x000c0400, 0xffffffff,
 706         0x00002650, 0x000c0400, 0xffffffff,
 707         0x000020b8, 0x000c0400, 0xffffffff,
 708         0x000020bc, 0x000c0400, 0xffffffff,
 709         0x000020c0, 0x000c0c80, 0xffffffff,
 710         0x0000f4a0, 0x000000c0, 0xffffffff,
 711         0x0000f4a4, 0x00680fff, 0xffffffff,
 712         0x00002f50, 0x00000903, 0xffffffff,
 713         0x000004c8, 0x00000000, 0xffffffff,
 714         0x000064ec, 0x00000000, 0xffffffff,
 715         0x00000c7c, 0x00000000, 0xffffffff,
 716         0x00008dfc, 0x00000000, 0xffffffff
 717 };
 718 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 719
 720 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 721 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 722
 723 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 724 {
 725         struct ni_power_info *pi = rdev->pm.dpm.priv;
 726
 727         return pi;
 728 }
 729
 730 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 731 {
 732         struct ni_ps *ps = rps->ps_priv;
 733
 734         return ps;
 735 }
 736
 737 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 738                                                      u16 v, s32 t,
 739                                                      u32 ileakage,
 740                                                      u32 *leakage)
 741 {
 742         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 743
 744         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 745         vddc = div64_s64(drm_int2fixp(v), 1000);
 746         temperature = div64_s64(drm_int2fixp(t), 1000);
 747
 748         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 749                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 750         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 751                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 752
 753         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 754
 755         *leakage = drm_fixp2int(leakage_w * 1000);
 756 }
 757
 758 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 759                                              const struct ni_leakage_coeffients *coeff,
 760                                              u16 v,
 761                                              s32 t,
 762                                              u32 i_leakage,
 763                                              u32 *leakage)
 764 {
 765         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 766 }
 767
 768 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 769 {
 770         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 771         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 772         /* we never hit the non-gddr5 limit so disable it */
 773         u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
 774
 775         if (vblank_time < switch_limit)
 776                 return true;
 777         else
 778                 return false;
 779
 780 }
 781
 782 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 783                                         struct radeon_ps *rps)
 784 {
 785         struct ni_ps *ps = ni_get_ps(rps);
 786         struct radeon_clock_and_voltage_limits *max_limits;
 787         bool disable_mclk_switching;
 788         u32 mclk, sclk;
 789         u16 vddc, vddci;
 790         u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc;
 791         int i;
 792
 793         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 794             ni_dpm_vblank_too_short(rdev))
 795                 disable_mclk_switching = true;
 796         else
 797                 disable_mclk_switching = false;
 798
 799         if (rdev->pm.dpm.ac_power)
 800                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 801         else
 802                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 803
 804         if (rdev->pm.dpm.ac_power == false) {
 805                 for (i = 0; i < ps->performance_level_count; i++) {
 806                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 807                                 ps->performance_levels[i].mclk = max_limits->mclk;
 808                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 809                                 ps->performance_levels[i].sclk = max_limits->sclk;
 810                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 811                                 ps->performance_levels[i].vddc = max_limits->vddc;
 812                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 813                                 ps->performance_levels[i].vddci = max_limits->vddci;
 814                 }
 815         }
 816
 817         /* limit clocks to max supported clocks based on voltage dependency tables */
 818         btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 819                                                         &max_sclk_vddc);
 820         btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 821                                                         &max_mclk_vddci);
 822         btc_get_max_clock_from_voltage_dependency_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 823                                                         &max_mclk_vddc);
 824
 825         for (i = 0; i < ps->performance_level_count; i++) {
 826                 if (max_sclk_vddc) {
 827                         if (ps->performance_levels[i].sclk > max_sclk_vddc)
 828                                 ps->performance_levels[i].sclk = max_sclk_vddc;
 829                 }
 830                 if (max_mclk_vddci) {
 831                         if (ps->performance_levels[i].mclk > max_mclk_vddci)
 832                                 ps->performance_levels[i].mclk = max_mclk_vddci;
 833                 }
 834                 if (max_mclk_vddc) {
 835                         if (ps->performance_levels[i].mclk > max_mclk_vddc)
 836                                 ps->performance_levels[i].mclk = max_mclk_vddc;
 837                 }
 838         }
 839
 840         /* XXX validate the min clocks required for display */
 841
 842         if (disable_mclk_switching) {
 843                 mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
 844                 sclk = ps->performance_levels[0].sclk;
 845                 vddc = ps->performance_levels[0].vddc;
 846                 vddci = ps->performance_levels[ps->performance_level_count - 1].vddci;
 847         } else {
 848                 sclk = ps->performance_levels[0].sclk;
 849                 mclk = ps->performance_levels[0].mclk;
 850                 vddc = ps->performance_levels[0].vddc;
 851                 vddci = ps->performance_levels[0].vddci;
 852         }
 853
 854         /* adjusted low state */
 855         ps->performance_levels[0].sclk = sclk;
 856         ps->performance_levels[0].mclk = mclk;
 857         ps->performance_levels[0].vddc = vddc;
 858         ps->performance_levels[0].vddci = vddci;
 859
 860         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 861                                   &ps->performance_levels[0].sclk,
 862                                   &ps->performance_levels[0].mclk);
 863
 864         for (i = 1; i < ps->performance_level_count; i++) {
 865                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 866                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 867                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 868                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 869         }
 870
 871         if (disable_mclk_switching) {
 872                 mclk = ps->performance_levels[0].mclk;
 873                 for (i = 1; i < ps->performance_level_count; i++) {
 874                         if (mclk < ps->performance_levels[i].mclk)
 875                                 mclk = ps->performance_levels[i].mclk;
 876                 }
 877                 for (i = 0; i < ps->performance_level_count; i++) {
 878                         ps->performance_levels[i].mclk = mclk;
 879                         ps->performance_levels[i].vddci = vddci;
 880                 }
 881         } else {
 882                 for (i = 1; i < ps->performance_level_count; i++) {
 883                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 884                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 885                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 886                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 887                 }
 888         }
 889
 890         for (i = 1; i < ps->performance_level_count; i++)
 891                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 892                                           &ps->performance_levels[i].sclk,
 893                                           &ps->performance_levels[i].mclk);
 894
 895         for (i = 0; i < ps->performance_level_count; i++)
 896                 btc_adjust_clock_combinations(rdev, max_limits,
 897                                               &ps->performance_levels[i]);
 898
 899         for (i = 0; i < ps->performance_level_count; i++) {
 900                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 901                                                    ps->performance_levels[i].sclk,
 902                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 903                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 904                                                    ps->performance_levels[i].mclk,
 905                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 906                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 907                                                    ps->performance_levels[i].mclk,
 908                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 909                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 910                                                    rdev->clock.current_dispclk,
 911                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 912         }
 913
 914         for (i = 0; i < ps->performance_level_count; i++) {
 915                 btc_apply_voltage_delta_rules(rdev,
 916                                               max_limits->vddc, max_limits->vddci,
 917                                               &ps->performance_levels[i].vddc,
 918                                               &ps->performance_levels[i].vddci);
 919         }
 920
 921         ps->dc_compatible = true;
 922         for (i = 0; i < ps->performance_level_count; i++) {
 923                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 924                         ps->dc_compatible = false;
 925
 926                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 927                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 928         }
 929 }
 930
 931 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 932 {
 933         u32 count;
 934         const u32 *ps = NULL;
 935
 936         ps = (const u32 *)&cayman_cgcg_cgls_default;
 937         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 938
 939         btc_program_mgcg_hw_sequence(rdev, ps, count);
 940 }
 941
 942 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 943                                       bool enable)
 944 {
 945         u32 count;
 946         const u32 *ps = NULL;
 947
 948         if (enable) {
 949                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 950                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 951         } else {
 952                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 953                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 954         }
 955
 956         btc_program_mgcg_hw_sequence(rdev, ps, count);
 957 }
 958
 959 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 960 {
 961         u32 count;
 962         const u32 *ps = NULL;
 963
 964         ps = (const u32 *)&cayman_mgcg_default;
 965         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 966
 967         btc_program_mgcg_hw_sequence(rdev, ps, count);
 968 }
 969
 970 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 971                                      bool enable)
 972 {
 973         u32 count;
 974         const u32 *ps = NULL;
 975
 976         if (enable) {
 977                 ps = (const u32 *)&cayman_mgcg_enable;
 978                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 979         } else {
 980                 ps = (const u32 *)&cayman_mgcg_disable;
 981                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 982         }
 983
 984         btc_program_mgcg_hw_sequence(rdev, ps, count);
 985 }
 986
 987 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 988 {
 989         u32 count;
 990         const u32 *ps = NULL;
 991
 992         ps = (const u32 *)&cayman_sysls_default;
 993         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 994
 995         btc_program_mgcg_hw_sequence(rdev, ps, count);
 996 }
 997
 998 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 999                                      bool enable)
1000 {
1001         u32 count;
1002         const u32 *ps = NULL;
1003
1004         if (enable) {
1005                 ps = (const u32 *)&cayman_sysls_enable;
1006                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
1007         } else {
1008                 ps = (const u32 *)&cayman_sysls_disable;
1009                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
1010         }
1011
1012         btc_program_mgcg_hw_sequence(rdev, ps, count);
1013
1014 }
1015
1016 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
1017                                                              struct radeon_clock_voltage_dependency_table *table)
1018 {
1019         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1020         u32 i;
1021
1022         if (table) {
1023                 for (i = 0; i < table->count; i++) {
1024                         if (0xff01 == table->entries[i].v) {
1025                                 if (pi->max_vddc == 0)
1026                                         return -EINVAL;
1027                                 table->entries[i].v = pi->max_vddc;
1028                         }
1029                 }
1030         }
1031         return 0;
1032 }
1033
1034 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1035 {
1036         int ret = 0;
1037
1038         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1039                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1040
1041         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1042                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1043         return ret;
1044 }
1045
1046 static void ni_stop_dpm(struct radeon_device *rdev)
1047 {
1048         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1049 }
1050
1051 #if 0
1052 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1053                                         bool ac_power)
1054 {
1055         if (ac_power)
1056                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1057                         0 : -EINVAL;
1058
1059         return 0;
1060 }
1061 #endif
1062
1063 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1064                                                       PPSMC_Msg msg, u32 parameter)
1065 {
1066         WREG32(SMC_SCRATCH0, parameter);
1067         return rv770_send_msg_to_smc(rdev, msg);
1068 }
1069
1070 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1071 {
1072         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1073                 return -EINVAL;
1074
1075         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1076                 0 : -EINVAL;
1077 }
1078
1079 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1080                                    enum radeon_dpm_forced_level level)
1081 {
1082         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1083                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1084                         return -EINVAL;
1085
1086                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1087                         return -EINVAL;
1088         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1089                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1090                         return -EINVAL;
1091
1092                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1093                         return -EINVAL;
1094         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1095                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1096                         return -EINVAL;
1097
1098                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1099                         return -EINVAL;
1100         }
1101
1102         rdev->pm.dpm.forced_level = level;
1103
1104         return 0;
1105 }
1106
1107 static void ni_stop_smc(struct radeon_device *rdev)
1108 {
1109         u32 tmp;
1110         int i;
1111
1112         for (i = 0; i < rdev->usec_timeout; i++) {
1113                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1114                 if (tmp != 1)
1115                         break;
1116                 udelay(1);
1117         }
1118
1119         udelay(100);
1120
1121         r7xx_stop_smc(rdev);
1122 }
1123
1124 static int ni_process_firmware_header(struct radeon_device *rdev)
1125 {
1126         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1127         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1128         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1129         u32 tmp;
1130         int ret;
1131
1132         ret = rv770_read_smc_sram_dword(rdev,
1133                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1134                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1135                                         &tmp, pi->sram_end);
1136
1137         if (ret)
1138                 return ret;
1139
1140         pi->state_table_start = (u16)tmp;
1141
1142         ret = rv770_read_smc_sram_dword(rdev,
1143                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1144                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1145                                         &tmp, pi->sram_end);
1146
1147         if (ret)
1148                 return ret;
1149
1150         pi->soft_regs_start = (u16)tmp;
1151
1152         ret = rv770_read_smc_sram_dword(rdev,
1153                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1154                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1155                                         &tmp, pi->sram_end);
1156
1157         if (ret)
1158                 return ret;
1159
1160         eg_pi->mc_reg_table_start = (u16)tmp;
1161
1162         ret = rv770_read_smc_sram_dword(rdev,
1163                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1164                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1165                                         &tmp, pi->sram_end);
1166
1167         if (ret)
1168                 return ret;
1169
1170         ni_pi->fan_table_start = (u16)tmp;
1171
1172         ret = rv770_read_smc_sram_dword(rdev,
1173                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1174                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1175                                         &tmp, pi->sram_end);
1176
1177         if (ret)
1178                 return ret;
1179
1180         ni_pi->arb_table_start = (u16)tmp;
1181
1182         ret = rv770_read_smc_sram_dword(rdev,
1183                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1184                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1185                                         &tmp, pi->sram_end);
1186
1187         if (ret)
1188                 return ret;
1189
1190         ni_pi->cac_table_start = (u16)tmp;
1191
1192         ret = rv770_read_smc_sram_dword(rdev,
1193                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1194                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1195                                         &tmp, pi->sram_end);
1196
1197         if (ret)
1198                 return ret;
1199
1200         ni_pi->spll_table_start = (u16)tmp;
1201
1202
1203         return ret;
1204 }
1205
1206 static void ni_read_clock_registers(struct radeon_device *rdev)
1207 {
1208         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1209
1210         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1211         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1212         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1213         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1214         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1215         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1216         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1217         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1218         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1219         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1220         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1221         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1222         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1223         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1224 }
1225
1226 #if 0
1227 static int ni_enter_ulp_state(struct radeon_device *rdev)
1228 {
1229         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1230
1231         if (pi->gfx_clock_gating) {
1232                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1233                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1234                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1235                 RREG32(GB_ADDR_CONFIG);
1236         }
1237
1238         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1239                  ~HOST_SMC_MSG_MASK);
1240
1241         udelay(25000);
1242
1243         return 0;
1244 }
1245 #endif
1246
1247 static void ni_program_response_times(struct radeon_device *rdev)
1248 {
1249         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1250         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1251         u32 reference_clock;
1252
1253         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1254
1255         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1256         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1257
1258         if (voltage_response_time == 0)
1259                 voltage_response_time = 1000;
1260
1261         if (backbias_response_time == 0)
1262                 backbias_response_time = 1000;
1263
1264         acpi_delay_time = 15000;
1265         vbi_time_out = 100000;
1266
1267         reference_clock = radeon_get_xclk(rdev);
1268
1269         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1270         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1271         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1272         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1273
1274         mclk_switch_limit = (460 * reference_clock) / 100;
1275
1276         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1277         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1278         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1279         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1280         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1281         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1282 }
1283
1284 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1285                                           struct atom_voltage_table *voltage_table,
1286                                           NISLANDS_SMC_STATETABLE *table)
1287 {
1288         unsigned int i;
1289
1290         for (i = 0; i < voltage_table->count; i++) {
1291                 table->highSMIO[i] = 0;
1292                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1293         }
1294 }
1295
1296 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1297                                            NISLANDS_SMC_STATETABLE *table)
1298 {
1299         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1300         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1301         unsigned char i;
1302
1303         if (eg_pi->vddc_voltage_table.count) {
1304                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1305                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1306                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1307                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1308
1309                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1310                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1311                                 table->maxVDDCIndexInPPTable = i;
1312                                 break;
1313                         }
1314                 }
1315         }
1316
1317         if (eg_pi->vddci_voltage_table.count) {
1318                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1319
1320                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1321                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1322                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1323         }
1324 }
1325
1326 static int ni_populate_voltage_value(struct radeon_device *rdev,
1327                                      struct atom_voltage_table *table,
1328                                      u16 value,
1329                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1330 {
1331         unsigned int i;
1332
1333         for (i = 0; i < table->count; i++) {
1334                 if (value <= table->entries[i].value) {
1335                         voltage->index = (u8)i;
1336                         voltage->value = cpu_to_be16(table->entries[i].value);
1337                         break;
1338                 }
1339         }
1340
1341         if (i >= table->count)
1342                 return -EINVAL;
1343
1344         return 0;
1345 }
1346
1347 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1348                                    u32 mclk,
1349                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1350 {
1351         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1352         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1353
1354         if (!pi->mvdd_control) {
1355                 voltage->index = eg_pi->mvdd_high_index;
1356                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1357                 return;
1358         }
1359
1360         if (mclk <= pi->mvdd_split_frequency) {
1361                 voltage->index = eg_pi->mvdd_low_index;
1362                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1363         } else {
1364                 voltage->index = eg_pi->mvdd_high_index;
1365                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1366         }
1367 }
1368
1369 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1370                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1371                                     u16 *std_voltage)
1372 {
1373         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1374             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1375                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1376         else
1377                 *std_voltage = be16_to_cpu(voltage->value);
1378
1379         return 0;
1380 }
1381
1382 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1383                                           u16 value, u8 index,
1384                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1385 {
1386         voltage->index = index;
1387         voltage->value = cpu_to_be16(value);
1388 }
1389
1390 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1391 {
1392         u32 xclk_period;
1393         u32 xclk = radeon_get_xclk(rdev);
1394         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1395
1396         xclk_period = (1000000000UL / xclk);
1397         xclk_period /= 10000UL;
1398
1399         return tmp * xclk_period;
1400 }
1401
1402 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1403 {
1404         return (power_in_watts * scaling_factor) << 2;
1405 }
1406
1407 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1408                                           struct radeon_ps *radeon_state,
1409                                           u32 near_tdp_limit)
1410 {
1411         struct ni_ps *state = ni_get_ps(radeon_state);
1412         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1413         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1414         u32 power_boost_limit = 0;
1415         int ret;
1416
1417         if (ni_pi->enable_power_containment &&
1418             ni_pi->use_power_boost_limit) {
1419                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1420                 u16 std_vddc_med;
1421                 u16 std_vddc_high;
1422                 u64 tmp, n, d;
1423
1424                 if (state->performance_level_count < 3)
1425                         return 0;
1426
1427                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1428                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1429                                                 &vddc);
1430                 if (ret)
1431                         return 0;
1432
1433                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1434                 if (ret)
1435                         return 0;
1436
1437                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1438                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1439                                                 &vddc);
1440                 if (ret)
1441                         return 0;
1442
1443                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1444                 if (ret)
1445                         return 0;
1446
1447                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1448                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1449                 tmp = div64_u64(n, d);
1450
1451                 if (tmp >> 32)
1452                         return 0;
1453                 power_boost_limit = (u32)tmp;
1454         }
1455
1456         return power_boost_limit;
1457 }
1458
1459 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1460                                             bool adjust_polarity,
1461                                             u32 tdp_adjustment,
1462                                             u32 *tdp_limit,
1463                                             u32 *near_tdp_limit)
1464 {
1465         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1466                 return -EINVAL;
1467
1468         if (adjust_polarity) {
1469                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1470                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1471         } else {
1472                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1473                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1474         }
1475
1476         return 0;
1477 }
1478
1479 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1480                                       struct radeon_ps *radeon_state)
1481 {
1482         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1483         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1484
1485         if (ni_pi->enable_power_containment) {
1486                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1487                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1488                 u32 tdp_limit;
1489                 u32 near_tdp_limit;
1490                 u32 power_boost_limit;
1491                 int ret;
1492
1493                 if (scaling_factor == 0)
1494                         return -EINVAL;
1495
1496                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1497
1498                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1499                                                        false, /* ??? */
1500                                                        rdev->pm.dpm.tdp_adjustment,
1501                                                        &tdp_limit,
1502                                                        &near_tdp_limit);
1503                 if (ret)
1504                         return ret;
1505
1506                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1507                                                                    near_tdp_limit);
1508
1509                 smc_table->dpm2Params.TDPLimit =
1510                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1511                 smc_table->dpm2Params.NearTDPLimit =
1512                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1513                 smc_table->dpm2Params.SafePowerLimit =
1514                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1515                                                            scaling_factor));
1516                 smc_table->dpm2Params.PowerBoostLimit =
1517                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1518
1519                 ret = rv770_copy_bytes_to_smc(rdev,
1520                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1521                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1522                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1523                                               sizeof(u32) * 4, pi->sram_end);
1524                 if (ret)
1525                         return ret;
1526         }
1527
1528         return 0;
1529 }
1530
1531 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1532                                 u32 arb_freq_src, u32 arb_freq_dest)
1533 {
1534         u32 mc_arb_dram_timing;
1535         u32 mc_arb_dram_timing2;
1536         u32 burst_time;
1537         u32 mc_cg_config;
1538
1539         switch (arb_freq_src) {
1540         case MC_CG_ARB_FREQ_F0:
1541                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1542                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1543                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1544                 break;
1545         case MC_CG_ARB_FREQ_F1:
1546                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1547                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1548                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1549                 break;
1550         case MC_CG_ARB_FREQ_F2:
1551                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1552                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1553                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1554                 break;
1555         case MC_CG_ARB_FREQ_F3:
1556                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1557                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1558                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1559                 break;
1560         default:
1561                 return -EINVAL;
1562         }
1563
1564         switch (arb_freq_dest) {
1565         case MC_CG_ARB_FREQ_F0:
1566                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1567                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1568                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1569                 break;
1570         case MC_CG_ARB_FREQ_F1:
1571                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1572                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1573                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1574                 break;
1575         case MC_CG_ARB_FREQ_F2:
1576                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1577                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1578                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1579                 break;
1580         case MC_CG_ARB_FREQ_F3:
1581                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1582                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1583                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1584                 break;
1585         default:
1586                 return -EINVAL;
1587         }
1588
1589         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1590         WREG32(MC_CG_CONFIG, mc_cg_config);
1591         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1592
1593         return 0;
1594 }
1595
1596 static int ni_init_arb_table_index(struct radeon_device *rdev)
1597 {
1598         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1599         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1600         u32 tmp;
1601         int ret;
1602
1603         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1604                                         &tmp, pi->sram_end);
1605         if (ret)
1606                 return ret;
1607
1608         tmp &= 0x00FFFFFF;
1609         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1610
1611         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1612                                           tmp, pi->sram_end);
1613 }
1614
1615 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1616 {
1617         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1618 }
1619
1620 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1621 {
1622         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1623         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1624         u32 tmp;
1625         int ret;
1626
1627         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1628                                         &tmp, pi->sram_end);
1629         if (ret)
1630                 return ret;
1631
1632         tmp = (tmp >> 24) & 0xff;
1633
1634         if (tmp == MC_CG_ARB_FREQ_F0)
1635                 return 0;
1636
1637         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1638 }
1639
1640 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1641                                                 struct rv7xx_pl *pl,
1642                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1643 {
1644         u32 dram_timing;
1645         u32 dram_timing2;
1646
1647         arb_regs->mc_arb_rfsh_rate =
1648                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1649
1650
1651         radeon_atom_set_engine_dram_timings(rdev,
1652                                             pl->sclk,
1653                                             pl->mclk);
1654
1655         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1656         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1657
1658         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1659         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1660
1661         return 0;
1662 }
1663
1664 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1665                                                   struct radeon_ps *radeon_state,
1666                                                   unsigned int first_arb_set)
1667 {
1668         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1669         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1670         struct ni_ps *state = ni_get_ps(radeon_state);
1671         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1672         int i, ret = 0;
1673
1674         for (i = 0; i < state->performance_level_count; i++) {
1675                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1676                 if (ret)
1677                         break;
1678
1679                 ret = rv770_copy_bytes_to_smc(rdev,
1680                                               (u16)(ni_pi->arb_table_start +
1681                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1682                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1683                                               (u8 *)&arb_regs,
1684                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1685                                               pi->sram_end);
1686                 if (ret)
1687                         break;
1688         }
1689         return ret;
1690 }
1691
1692 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1693                                                struct radeon_ps *radeon_new_state)
1694 {
1695         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1696                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1697 }
1698
1699 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1700                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1701 {
1702         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1703
1704         voltage->index = eg_pi->mvdd_high_index;
1705         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1706 }
1707
1708 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1709                                          struct radeon_ps *radeon_initial_state,
1710                                          NISLANDS_SMC_STATETABLE *table)
1711 {
1712         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1713         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1714         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1715         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1716         u32 reg;
1717         int ret;
1718
1719         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1720                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1721         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1722                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1723         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1724                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1725         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1726                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1727         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1728                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1729         table->initialState.levels[0].mclk.vDLL_CNTL =
1730                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1731         table->initialState.levels[0].mclk.vMPLL_SS =
1732                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1733         table->initialState.levels[0].mclk.vMPLL_SS2 =
1734                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1735         table->initialState.levels[0].mclk.mclk_value =
1736                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1737
1738         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1739                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1740         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1741                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1742         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1743                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1744         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1745                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1746         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1747                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1748         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1749                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1750         table->initialState.levels[0].sclk.sclk_value =
1751                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1752         table->initialState.levels[0].arbRefreshState =
1753                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1754
1755         table->initialState.levels[0].ACIndex = 0;
1756
1757         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1758                                         initial_state->performance_levels[0].vddc,
1759                                         &table->initialState.levels[0].vddc);
1760         if (!ret) {
1761                 u16 std_vddc;
1762
1763                 ret = ni_get_std_voltage_value(rdev,
1764                                                &table->initialState.levels[0].vddc,
1765                                                &std_vddc);
1766                 if (!ret)
1767                         ni_populate_std_voltage_value(rdev, std_vddc,
1768                                                       table->initialState.levels[0].vddc.index,
1769                                                       &table->initialState.levels[0].std_vddc);
1770         }
1771
1772         if (eg_pi->vddci_control)
1773                 ni_populate_voltage_value(rdev,
1774                                           &eg_pi->vddci_voltage_table,
1775                                           initial_state->performance_levels[0].vddci,
1776                                           &table->initialState.levels[0].vddci);
1777
1778         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1779
1780         reg = CG_R(0xffff) | CG_L(0);
1781         table->initialState.levels[0].aT = cpu_to_be32(reg);
1782
1783         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1784
1785         if (pi->boot_in_gen2)
1786                 table->initialState.levels[0].gen2PCIE = 1;
1787         else
1788                 table->initialState.levels[0].gen2PCIE = 0;
1789
1790         if (pi->mem_gddr5) {
1791                 table->initialState.levels[0].strobeMode =
1792                         cypress_get_strobe_mode_settings(rdev,
1793                                                          initial_state->performance_levels[0].mclk);
1794
1795                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1796                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1797                 else
1798                         table->initialState.levels[0].mcFlags =  0;
1799         }
1800
1801         table->initialState.levelCount = 1;
1802
1803         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1804
1805         table->initialState.levels[0].dpm2.MaxPS = 0;
1806         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1807         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1808         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1809
1810         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1811         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1812
1813         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1814         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1815
1816         return 0;
1817 }
1818
1819 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1820                                       NISLANDS_SMC_STATETABLE *table)
1821 {
1822         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1823         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1824         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1825         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1826         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1827         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1828         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1829         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1830         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1831         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1832         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1833         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1834         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1835         u32 reg;
1836         int ret;
1837
1838         table->ACPIState = table->initialState;
1839
1840         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1841
1842         if (pi->acpi_vddc) {
1843                 ret = ni_populate_voltage_value(rdev,
1844                                                 &eg_pi->vddc_voltage_table,
1845                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1846                 if (!ret) {
1847                         u16 std_vddc;
1848
1849                         ret = ni_get_std_voltage_value(rdev,
1850                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1851                         if (!ret)
1852                                 ni_populate_std_voltage_value(rdev, std_vddc,
1853                                                               table->ACPIState.levels[0].vddc.index,
1854                                                               &table->ACPIState.levels[0].std_vddc);
1855                 }
1856
1857                 if (pi->pcie_gen2) {
1858                         if (pi->acpi_pcie_gen2)
1859                                 table->ACPIState.levels[0].gen2PCIE = 1;
1860                         else
1861                                 table->ACPIState.levels[0].gen2PCIE = 0;
1862                 } else {
1863                         table->ACPIState.levels[0].gen2PCIE = 0;
1864                 }
1865         } else {
1866                 ret = ni_populate_voltage_value(rdev,
1867                                                 &eg_pi->vddc_voltage_table,
1868                                                 pi->min_vddc_in_table,
1869                                                 &table->ACPIState.levels[0].vddc);
1870                 if (!ret) {
1871                         u16 std_vddc;
1872
1873                         ret = ni_get_std_voltage_value(rdev,
1874                                                        &table->ACPIState.levels[0].vddc,
1875                                                        &std_vddc);
1876                         if (!ret)
1877                                 ni_populate_std_voltage_value(rdev, std_vddc,
1878                                                               table->ACPIState.levels[0].vddc.index,
1879                                                               &table->ACPIState.levels[0].std_vddc);
1880                 }
1881                 table->ACPIState.levels[0].gen2PCIE = 0;
1882         }
1883
1884         if (eg_pi->acpi_vddci) {
1885                 if (eg_pi->vddci_control)
1886                         ni_populate_voltage_value(rdev,
1887                                                   &eg_pi->vddci_voltage_table,
1888                                                   eg_pi->acpi_vddci,
1889                                                   &table->ACPIState.levels[0].vddci);
1890         }
1891
1892
1893         mpll_ad_func_cntl &= ~PDNB;
1894
1895         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1896
1897         if (pi->mem_gddr5)
1898                 mpll_dq_func_cntl &= ~PDNB;
1899         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1900
1901
1902         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1903                              MRDCKA1_RESET |
1904                              MRDCKB0_RESET |
1905                              MRDCKB1_RESET |
1906                              MRDCKC0_RESET |
1907                              MRDCKC1_RESET |
1908                              MRDCKD0_RESET |
1909                              MRDCKD1_RESET);
1910
1911         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1912                               MRDCKA1_PDNB |
1913                               MRDCKB0_PDNB |
1914                               MRDCKB1_PDNB |
1915                               MRDCKC0_PDNB |
1916                               MRDCKC1_PDNB |
1917                               MRDCKD0_PDNB |
1918                               MRDCKD1_PDNB);
1919
1920         dll_cntl |= (MRDCKA0_BYPASS |
1921                      MRDCKA1_BYPASS |
1922                      MRDCKB0_BYPASS |
1923                      MRDCKB1_BYPASS |
1924                      MRDCKC0_BYPASS |
1925                      MRDCKC1_BYPASS |
1926                      MRDCKD0_BYPASS |
1927                      MRDCKD1_BYPASS);
1928
1929         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1930         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1931
1932         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1933         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1934         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1935         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1936         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1937         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1938
1939         table->ACPIState.levels[0].mclk.mclk_value = 0;
1940
1941         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1942         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1943         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1944         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1945
1946         table->ACPIState.levels[0].sclk.sclk_value = 0;
1947
1948         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1949
1950         if (eg_pi->dynamic_ac_timing)
1951                 table->ACPIState.levels[0].ACIndex = 1;
1952
1953         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1954         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1955         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1956         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1957
1958         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1959         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1960
1961         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1962         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1963
1964         return 0;
1965 }
1966
1967 static int ni_init_smc_table(struct radeon_device *rdev)
1968 {
1969         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1970         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1971         int ret;
1972         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1973         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1974
1975         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1976
1977         ni_populate_smc_voltage_tables(rdev, table);
1978
1979         switch (rdev->pm.int_thermal_type) {
1980         case THERMAL_TYPE_NI:
1981         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1982                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1983                 break;
1984         case THERMAL_TYPE_NONE:
1985                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1986                 break;
1987         default:
1988                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1989                 break;
1990         }
1991
1992         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1993                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1994
1995         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1996                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1997
1998         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1999                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
2000
2001         if (pi->mem_gddr5)
2002                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
2003
2004         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
2005         if (ret)
2006                 return ret;
2007
2008         ret = ni_populate_smc_acpi_state(rdev, table);
2009         if (ret)
2010                 return ret;
2011
2012         table->driverState = table->initialState;
2013
2014         table->ULVState = table->initialState;
2015
2016         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
2017                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
2018         if (ret)
2019                 return ret;
2020
2021         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
2022                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
2023 }
2024
2025 static int ni_calculate_sclk_params(struct radeon_device *rdev,
2026                                     u32 engine_clock,
2027                                     NISLANDS_SMC_SCLK_VALUE *sclk)
2028 {
2029         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2030         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2031         struct atom_clock_dividers dividers;
2032         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2033         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2034         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2035         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2036         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2037         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2038         u64 tmp;
2039         u32 reference_clock = rdev->clock.spll.reference_freq;
2040         u32 reference_divider;
2041         u32 fbdiv;
2042         int ret;
2043
2044         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2045                                              engine_clock, false, &dividers);
2046         if (ret)
2047                 return ret;
2048
2049         reference_divider = 1 + dividers.ref_div;
2050
2051
2052         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2053         do_div(tmp, reference_clock);
2054         fbdiv = (u32) tmp;
2055
2056         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2057         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2058         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2059
2060         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2061         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2062
2063         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2064         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2065         spll_func_cntl_3 |= SPLL_DITHEN;
2066
2067         if (pi->sclk_ss) {
2068                 struct radeon_atom_ss ss;
2069                 u32 vco_freq = engine_clock * dividers.post_div;
2070
2071                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2072                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2073                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2074                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2075
2076                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2077                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2078                         cg_spll_spread_spectrum |= SSEN;
2079
2080                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2081                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2082                 }
2083         }
2084
2085         sclk->sclk_value = engine_clock;
2086         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2087         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2088         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2089         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2090         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2091         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2092
2093         return 0;
2094 }
2095
2096 static int ni_populate_sclk_value(struct radeon_device *rdev,
2097                                   u32 engine_clock,
2098                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2099 {
2100         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2101         int ret;
2102
2103         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2104         if (!ret) {
2105                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2106                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2107                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2108                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2109                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2110                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2111                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2112         }
2113
2114         return ret;
2115 }
2116
2117 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2118 {
2119         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2120         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2121         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2122         NISLANDS_SMC_SCLK_VALUE sclk_params;
2123         u32 fb_div;
2124         u32 p_div;
2125         u32 clk_s;
2126         u32 clk_v;
2127         u32 sclk = 0;
2128         int i, ret;
2129         u32 tmp;
2130
2131         if (ni_pi->spll_table_start == 0)
2132                 return -EINVAL;
2133
2134         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2135         if (spll_table == NULL)
2136                 return -ENOMEM;
2137
2138         for (i = 0; i < 256; i++) {
2139                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2140                 if (ret)
2141                         break;
2142
2143                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2144                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2145                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2146                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2147
2148                 fb_div &= ~0x00001FFF;
2149                 fb_div >>= 1;
2150                 clk_v >>= 6;
2151
2152                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2153                         ret = -EINVAL;
2154
2155                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2156                         ret = -EINVAL;
2157
2158                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2159                         ret = -EINVAL;
2160
2161                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2162                         ret = -EINVAL;
2163
2164                 if (ret)
2165                         break;
2166
2167                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2168                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2169                 spll_table->freq[i] = cpu_to_be32(tmp);
2170
2171                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2172                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2173                 spll_table->ss[i] = cpu_to_be32(tmp);
2174
2175                 sclk += 512;
2176         }
2177
2178         if (!ret)
2179                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2180                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2181
2182         kfree(spll_table);
2183
2184         return ret;
2185 }
2186
2187 static int ni_populate_mclk_value(struct radeon_device *rdev,
2188                                   u32 engine_clock,
2189                                   u32 memory_clock,
2190                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2191                                   bool strobe_mode,
2192                                   bool dll_state_on)
2193 {
2194         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2195         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2196         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2197         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2198         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2199         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2200         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2201         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2202         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2203         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2204         struct atom_clock_dividers dividers;
2205         u32 ibias;
2206         u32 dll_speed;
2207         int ret;
2208         u32 mc_seq_misc7;
2209
2210         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2211                                              memory_clock, strobe_mode, &dividers);
2212         if (ret)
2213                 return ret;
2214
2215         if (!strobe_mode) {
2216                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2217
2218                 if (mc_seq_misc7 & 0x8000000)
2219                         dividers.post_div = 1;
2220         }
2221
2222         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2223
2224         mpll_ad_func_cntl &= ~(CLKR_MASK |
2225                                YCLK_POST_DIV_MASK |
2226                                CLKF_MASK |
2227                                CLKFRAC_MASK |
2228                                IBIAS_MASK);
2229         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2230         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2231         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2232         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2233         mpll_ad_func_cntl |= IBIAS(ibias);
2234
2235         if (dividers.vco_mode)
2236                 mpll_ad_func_cntl_2 |= VCO_MODE;
2237         else
2238                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2239
2240         if (pi->mem_gddr5) {
2241                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2242                                        YCLK_POST_DIV_MASK |
2243                                        CLKF_MASK |
2244                                        CLKFRAC_MASK |
2245                                        IBIAS_MASK);
2246                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2247                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2248                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2249                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2250                 mpll_dq_func_cntl |= IBIAS(ibias);
2251
2252                 if (strobe_mode)
2253                         mpll_dq_func_cntl &= ~PDNB;
2254                 else
2255                         mpll_dq_func_cntl |= PDNB;
2256
2257                 if (dividers.vco_mode)
2258                         mpll_dq_func_cntl_2 |= VCO_MODE;
2259                 else
2260                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2261         }
2262
2263         if (pi->mclk_ss) {
2264                 struct radeon_atom_ss ss;
2265                 u32 vco_freq = memory_clock * dividers.post_div;
2266
2267                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2268                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2269                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2270                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2271                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2272                         u32 clk_v = ss.percentage *
2273                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2274
2275                         mpll_ss1 &= ~CLKV_MASK;
2276                         mpll_ss1 |= CLKV(clk_v);
2277
2278                         mpll_ss2 &= ~CLKS_MASK;
2279                         mpll_ss2 |= CLKS(clk_s);
2280                 }
2281         }
2282
2283         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2284                                         memory_clock);
2285
2286         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2287         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2288         if (dll_state_on)
2289                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2290                                      MRDCKA1_PDNB |
2291                                      MRDCKB0_PDNB |
2292                                      MRDCKB1_PDNB |
2293                                      MRDCKC0_PDNB |
2294                                      MRDCKC1_PDNB |
2295                                      MRDCKD0_PDNB |
2296                                      MRDCKD1_PDNB);
2297         else
2298                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2299                                       MRDCKA1_PDNB |
2300                                       MRDCKB0_PDNB |
2301                                       MRDCKB1_PDNB |
2302                                       MRDCKC0_PDNB |
2303                                       MRDCKC1_PDNB |
2304                                       MRDCKD0_PDNB |
2305                                       MRDCKD1_PDNB);
2306
2307
2308         mclk->mclk_value = cpu_to_be32(memory_clock);
2309         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2310         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2311         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2312         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2313         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2314         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2315         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2316         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2317
2318         return 0;
2319 }
2320
2321 static void ni_populate_smc_sp(struct radeon_device *rdev,
2322                                struct radeon_ps *radeon_state,
2323                                NISLANDS_SMC_SWSTATE *smc_state)
2324 {
2325         struct ni_ps *ps = ni_get_ps(radeon_state);
2326         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2327         int i;
2328
2329         for (i = 0; i < ps->performance_level_count - 1; i++)
2330                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2331
2332         smc_state->levels[ps->performance_level_count - 1].bSP =
2333                 cpu_to_be32(pi->psp);
2334 }
2335
2336 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2337                                          struct rv7xx_pl *pl,
2338                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2339 {
2340         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2341         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2342         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2343         int ret;
2344         bool dll_state_on;
2345         u16 std_vddc;
2346         u32 tmp = RREG32(DC_STUTTER_CNTL);
2347
2348         level->gen2PCIE = pi->pcie_gen2 ?
2349                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2350
2351         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2352         if (ret)
2353                 return ret;
2354
2355         level->mcFlags =  0;
2356         if (pi->mclk_stutter_mode_threshold &&
2357             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2358             !eg_pi->uvd_enabled &&
2359             (tmp & DC_STUTTER_ENABLE_A) &&
2360             (tmp & DC_STUTTER_ENABLE_B))
2361                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2362
2363         if (pi->mem_gddr5) {
2364                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2365                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2366                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2367                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2368
2369                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2370
2371                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2372                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2373                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2374                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2375                         else
2376                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2377                 } else {
2378                         dll_state_on = false;
2379                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2380                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2381                 }
2382
2383                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2384                                              &level->mclk,
2385                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2386                                              dll_state_on);
2387         } else
2388                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2389
2390         if (ret)
2391                 return ret;
2392
2393         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2394                                         pl->vddc, &level->vddc);
2395         if (ret)
2396                 return ret;
2397
2398         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2399         if (ret)
2400                 return ret;
2401
2402         ni_populate_std_voltage_value(rdev, std_vddc,
2403                                       level->vddc.index, &level->std_vddc);
2404
2405         if (eg_pi->vddci_control) {
2406                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2407                                                 pl->vddci, &level->vddci);
2408                 if (ret)
2409                         return ret;
2410         }
2411
2412         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2413
2414         return ret;
2415 }
2416
2417 static int ni_populate_smc_t(struct radeon_device *rdev,
2418                              struct radeon_ps *radeon_state,
2419                              NISLANDS_SMC_SWSTATE *smc_state)
2420 {
2421         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2422         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2423         struct ni_ps *state = ni_get_ps(radeon_state);
2424         u32 a_t;
2425         u32 t_l, t_h;
2426         u32 high_bsp;
2427         int i, ret;
2428
2429         if (state->performance_level_count >= 9)
2430                 return -EINVAL;
2431
2432         if (state->performance_level_count < 2) {
2433                 a_t = CG_R(0xffff) | CG_L(0);
2434                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2435                 return 0;
2436         }
2437
2438         smc_state->levels[0].aT = cpu_to_be32(0);
2439
2440         for (i = 0; i <= state->performance_level_count - 2; i++) {
2441                 if (eg_pi->uvd_enabled)
2442                         ret = r600_calculate_at(
2443                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2444                                 100 * R600_AH_DFLT,
2445                                 state->performance_levels[i + 1].sclk,
2446                                 state->performance_levels[i].sclk,
2447                                 &t_l,
2448                                 &t_h);
2449                 else
2450                         ret = r600_calculate_at(
2451                                 1000 * (i + 1),
2452                                 100 * R600_AH_DFLT,
2453                                 state->performance_levels[i + 1].sclk,
2454                                 state->performance_levels[i].sclk,
2455                                 &t_l,
2456                                 &t_h);
2457
2458                 if (ret) {
2459                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2460                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2461                 }
2462
2463                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2464                 a_t |= CG_R(t_l * pi->bsp / 20000);
2465                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2466
2467                 high_bsp = (i == state->performance_level_count - 2) ?
2468                         pi->pbsp : pi->bsp;
2469
2470                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2471                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2472         }
2473
2474         return 0;
2475 }
2476
2477 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2478                                                 struct radeon_ps *radeon_state,
2479                                                 NISLANDS_SMC_SWSTATE *smc_state)
2480 {
2481         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2482         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2483         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2484         struct ni_ps *state = ni_get_ps(radeon_state);
2485         u32 prev_sclk;
2486         u32 max_sclk;
2487         u32 min_sclk;
2488         int i, ret;
2489         u32 tdp_limit;
2490         u32 near_tdp_limit;
2491         u32 power_boost_limit;
2492         u8 max_ps_percent;
2493
2494         if (ni_pi->enable_power_containment == false)
2495                 return 0;
2496
2497         if (state->performance_level_count == 0)
2498                 return -EINVAL;
2499
2500         if (smc_state->levelCount != state->performance_level_count)
2501                 return -EINVAL;
2502
2503         ret = ni_calculate_adjusted_tdp_limits(rdev,
2504                                                false, /* ??? */
2505                                                rdev->pm.dpm.tdp_adjustment,
2506                                                &tdp_limit,
2507                                                &near_tdp_limit);
2508         if (ret)
2509                 return ret;
2510
2511         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2512
2513         ret = rv770_write_smc_sram_dword(rdev,
2514                                          pi->state_table_start +
2515                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2516                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2517                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2518                                          pi->sram_end);
2519         if (ret)
2520                 power_boost_limit = 0;
2521
2522         smc_state->levels[0].dpm2.MaxPS = 0;
2523         smc_state->levels[0].dpm2.NearTDPDec = 0;
2524         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2525         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2526         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2527
2528         for (i = 1; i < state->performance_level_count; i++) {
2529                 prev_sclk = state->performance_levels[i-1].sclk;
2530                 max_sclk  = state->performance_levels[i].sclk;
2531                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2532                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2533
2534                 if (max_sclk < prev_sclk)
2535                         return -EINVAL;
2536
2537                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2538                         min_sclk = max_sclk;
2539                 else if (1 == i)
2540                         min_sclk = prev_sclk;
2541                 else
2542                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2543
2544                 if (min_sclk < state->performance_levels[0].sclk)
2545                         min_sclk = state->performance_levels[0].sclk;
2546
2547                 if (min_sclk == 0)
2548                         return -EINVAL;
2549
2550                 smc_state->levels[i].dpm2.MaxPS =
2551                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2552                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2553                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2554                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2555                 smc_state->levels[i].stateFlags |=
2556                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2557                         PPSMC_STATEFLAG_POWERBOOST : 0;
2558         }
2559
2560         return 0;
2561 }
2562
2563 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2564                                          struct radeon_ps *radeon_state,
2565                                          NISLANDS_SMC_SWSTATE *smc_state)
2566 {
2567         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2568         struct ni_ps *state = ni_get_ps(radeon_state);
2569         u32 sq_power_throttle;
2570         u32 sq_power_throttle2;
2571         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2572         int i;
2573
2574         if (state->performance_level_count == 0)
2575                 return -EINVAL;
2576
2577         if (smc_state->levelCount != state->performance_level_count)
2578                 return -EINVAL;
2579
2580         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2581                 return -EINVAL;
2582
2583         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2584                 enable_sq_ramping = false;
2585
2586         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2587                 enable_sq_ramping = false;
2588
2589         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2590                 enable_sq_ramping = false;
2591
2592         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2593                 enable_sq_ramping = false;
2594
2595         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO <= (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2596                 enable_sq_ramping = false;
2597
2598         for (i = 0; i < state->performance_level_count; i++) {
2599                 sq_power_throttle  = 0;
2600                 sq_power_throttle2 = 0;
2601
2602                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2603                     enable_sq_ramping) {
2604                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2605                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2606                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2607                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2608                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2609                 } else {
2610                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2611                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2612                 }
2613
2614                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2615                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2616         }
2617
2618         return 0;
2619 }
2620
2621 static int ni_enable_power_containment(struct radeon_device *rdev,
2622                                        struct radeon_ps *radeon_new_state,
2623                                        bool enable)
2624 {
2625         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2626         PPSMC_Result smc_result;
2627         int ret = 0;
2628
2629         if (ni_pi->enable_power_containment) {
2630                 if (enable) {
2631                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2632                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2633                                 if (smc_result != PPSMC_Result_OK) {
2634                                         ret = -EINVAL;
2635                                         ni_pi->pc_enabled = false;
2636                                 } else {
2637                                         ni_pi->pc_enabled = true;
2638                                 }
2639                         }
2640                 } else {
2641                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2642                         if (smc_result != PPSMC_Result_OK)
2643                                 ret = -EINVAL;
2644                         ni_pi->pc_enabled = false;
2645                 }
2646         }
2647
2648         return ret;
2649 }
2650
2651 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2652                                          struct radeon_ps *radeon_state,
2653                                          NISLANDS_SMC_SWSTATE *smc_state)
2654 {
2655         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2656         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2657         struct ni_ps *state = ni_get_ps(radeon_state);
2658         int i, ret;
2659         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2660
2661         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2662                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2663
2664         smc_state->levelCount = 0;
2665
2666         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2667                 return -EINVAL;
2668
2669         for (i = 0; i < state->performance_level_count; i++) {
2670                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2671                                                     &smc_state->levels[i]);
2672                 smc_state->levels[i].arbRefreshState =
2673                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2674
2675                 if (ret)
2676                         return ret;
2677
2678                 if (ni_pi->enable_power_containment)
2679                         smc_state->levels[i].displayWatermark =
2680                                 (state->performance_levels[i].sclk < threshold) ?
2681                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2682                 else
2683                         smc_state->levels[i].displayWatermark = (i < 2) ?
2684                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2685
2686                 if (eg_pi->dynamic_ac_timing)
2687                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2688                 else
2689                         smc_state->levels[i].ACIndex = 0;
2690
2691                 smc_state->levelCount++;
2692         }
2693
2694         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2695                                       cpu_to_be32(threshold / 512));
2696
2697         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2698
2699         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2700         if (ret)
2701                 ni_pi->enable_power_containment = false;
2702
2703         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2704         if (ret)
2705                 ni_pi->enable_sq_ramping = false;
2706
2707         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2708 }
2709
2710 static int ni_upload_sw_state(struct radeon_device *rdev,
2711                               struct radeon_ps *radeon_new_state)
2712 {
2713         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2714         u16 address = pi->state_table_start +
2715                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2716         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2717                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2718         int ret;
2719         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2720
2721         if (smc_state == NULL)
2722                 return -ENOMEM;
2723
2724         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2725         if (ret)
2726                 goto done;
2727
2728         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2729
2730 done:
2731         kfree(smc_state);
2732
2733         return ret;
2734 }
2735
2736 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2737                                        struct ni_mc_reg_table *table)
2738 {
2739         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2740         u8 i, j, k;
2741         u32 temp_reg;
2742
2743         for (i = 0, j = table->last; i < table->last; i++) {
2744                 switch (table->mc_reg_address[i].s1) {
2745                 case MC_SEQ_MISC1 >> 2:
2746                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2747                                 return -EINVAL;
2748                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2749                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2750                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2751                         for (k = 0; k < table->num_entries; k++)
2752                                 table->mc_reg_table_entry[k].mc_data[j] =
2753                                         ((temp_reg & 0xffff0000)) |
2754                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2755                         j++;
2756                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2757                                 return -EINVAL;
2758
2759                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2760                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2761                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2762                         for(k = 0; k < table->num_entries; k++) {
2763                                 table->mc_reg_table_entry[k].mc_data[j] =
2764                                         (temp_reg & 0xffff0000) |
2765                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2766                                 if (!pi->mem_gddr5)
2767                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2768                         }
2769                         j++;
2770                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2771                                 return -EINVAL;
2772                         break;
2773                 case MC_SEQ_RESERVE_M >> 2:
2774                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2775                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2776                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2777                         for (k = 0; k < table->num_entries; k++)
2778                                 table->mc_reg_table_entry[k].mc_data[j] =
2779                                         (temp_reg & 0xffff0000) |
2780                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2781                         j++;
2782                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2783                                 return -EINVAL;
2784                         break;
2785                 default:
2786                         break;
2787                 }
2788         }
2789
2790         table->last = j;
2791
2792         return 0;
2793 }
2794
2795 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2796 {
2797         bool result = true;
2798
2799         switch (in_reg) {
2800         case  MC_SEQ_RAS_TIMING >> 2:
2801                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2802                 break;
2803         case MC_SEQ_CAS_TIMING >> 2:
2804                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2805                 break;
2806         case MC_SEQ_MISC_TIMING >> 2:
2807                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2808                 break;
2809         case MC_SEQ_MISC_TIMING2 >> 2:
2810                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2811                 break;
2812         case MC_SEQ_RD_CTL_D0 >> 2:
2813                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2814                 break;
2815         case MC_SEQ_RD_CTL_D1 >> 2:
2816                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2817                 break;
2818         case MC_SEQ_WR_CTL_D0 >> 2:
2819                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2820                 break;
2821         case MC_SEQ_WR_CTL_D1 >> 2:
2822                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2823                 break;
2824         case MC_PMG_CMD_EMRS >> 2:
2825                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2826                 break;
2827         case MC_PMG_CMD_MRS >> 2:
2828                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2829                 break;
2830         case MC_PMG_CMD_MRS1 >> 2:
2831                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2832                 break;
2833         case MC_SEQ_PMG_TIMING >> 2:
2834                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2835                 break;
2836         case MC_PMG_CMD_MRS2 >> 2:
2837                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2838                 break;
2839         default:
2840                 result = false;
2841                 break;
2842         }
2843
2844         return result;
2845 }
2846
2847 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2848 {
2849         u8 i, j;
2850
2851         for (i = 0; i < table->last; i++) {
2852                 for (j = 1; j < table->num_entries; j++) {
2853                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2854                                 table->valid_flag |= 1 << i;
2855                                 break;
2856                         }
2857                 }
2858         }
2859 }
2860
2861 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2862 {
2863         u32 i;
2864         u16 address;
2865
2866         for (i = 0; i < table->last; i++)
2867                 table->mc_reg_address[i].s0 =
2868                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2869                         address : table->mc_reg_address[i].s1;
2870 }
2871
2872 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2873                                       struct ni_mc_reg_table *ni_table)
2874 {
2875         u8 i, j;
2876
2877         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2878                 return -EINVAL;
2879         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2880                 return -EINVAL;
2881
2882         for (i = 0; i < table->last; i++)
2883                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2884         ni_table->last = table->last;
2885
2886         for (i = 0; i < table->num_entries; i++) {
2887                 ni_table->mc_reg_table_entry[i].mclk_max =
2888                         table->mc_reg_table_entry[i].mclk_max;
2889                 for (j = 0; j < table->last; j++)
2890                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2891                                 table->mc_reg_table_entry[i].mc_data[j];
2892         }
2893         ni_table->num_entries = table->num_entries;
2894
2895         return 0;
2896 }
2897
2898 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2899 {
2900         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2901         int ret;
2902         struct atom_mc_reg_table *table;
2903         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2904         u8 module_index = rv770_get_memory_module_index(rdev);
2905
2906         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2907         if (!table)
2908                 return -ENOMEM;
2909
2910         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2911         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2912         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2913         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2914         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2915         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2916         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2917         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2918         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2919         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2920         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2921         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2922         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2923
2924         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2925
2926         if (ret)
2927                 goto init_mc_done;
2928
2929         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2930
2931         if (ret)
2932                 goto init_mc_done;
2933
2934         ni_set_s0_mc_reg_index(ni_table);
2935
2936         ret = ni_set_mc_special_registers(rdev, ni_table);
2937
2938         if (ret)
2939                 goto init_mc_done;
2940
2941         ni_set_valid_flag(ni_table);
2942
2943 init_mc_done:
2944         kfree(table);
2945
2946         return ret;
2947 }
2948
2949 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2950                                          SMC_NIslands_MCRegisters *mc_reg_table)
2951 {
2952         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2953         u32 i, j;
2954
2955         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2956                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2957                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2958                                 break;
2959                         mc_reg_table->address[i].s0 =
2960                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2961                         mc_reg_table->address[i].s1 =
2962                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2963                         i++;
2964                 }
2965         }
2966         mc_reg_table->last = (u8)i;
2967 }
2968
2969
2970 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2971                                     SMC_NIslands_MCRegisterSet *data,
2972                                     u32 num_entries, u32 valid_flag)
2973 {
2974         u32 i, j;
2975
2976         for (i = 0, j = 0; j < num_entries; j++) {
2977                 if (valid_flag & (1 << j)) {
2978                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2979                         i++;
2980                 }
2981         }
2982 }
2983
2984 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2985                                                  struct rv7xx_pl *pl,
2986                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2987 {
2988         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2989         u32 i = 0;
2990
2991         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2992                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2993                         break;
2994         }
2995
2996         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2997                 --i;
2998
2999         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
3000                                 mc_reg_table_data,
3001                                 ni_pi->mc_reg_table.last,
3002                                 ni_pi->mc_reg_table.valid_flag);
3003 }
3004
3005 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
3006                                            struct radeon_ps *radeon_state,
3007                                            SMC_NIslands_MCRegisters *mc_reg_table)
3008 {
3009         struct ni_ps *state = ni_get_ps(radeon_state);
3010         int i;
3011
3012         for (i = 0; i < state->performance_level_count; i++) {
3013                 ni_convert_mc_reg_table_entry_to_smc(rdev,
3014                                                      &state->performance_levels[i],
3015                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
3016         }
3017 }
3018
3019 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
3020                                     struct radeon_ps *radeon_boot_state)
3021 {
3022         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3023         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3024         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3025         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
3026         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3027
3028         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3029
3030         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3031
3032         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3033
3034         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3035                                              &mc_reg_table->data[0]);
3036
3037         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3038                                 &mc_reg_table->data[1],
3039                                 ni_pi->mc_reg_table.last,
3040                                 ni_pi->mc_reg_table.valid_flag);
3041
3042         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3043
3044         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3045                                        (u8 *)mc_reg_table,
3046                                        sizeof(SMC_NIslands_MCRegisters),
3047                                        pi->sram_end);
3048 }
3049
3050 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3051                                   struct radeon_ps *radeon_new_state)
3052 {
3053         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3054         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3055         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3056         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3057         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3058         u16 address;
3059
3060         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3061
3062         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3063
3064         address = eg_pi->mc_reg_table_start +
3065                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3066
3067         return rv770_copy_bytes_to_smc(rdev, address,
3068                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3069                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3070                                        pi->sram_end);
3071 }
3072
3073 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3074                                                    PP_NIslands_CACTABLES *cac_tables)
3075 {
3076         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3077         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3078         u32 leakage = 0;
3079         unsigned int i, j, table_size;
3080         s32 t;
3081         u32 smc_leakage, max_leakage = 0;
3082         u32 scaling_factor;
3083
3084         table_size = eg_pi->vddc_voltage_table.count;
3085
3086         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3087                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3088
3089         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3090
3091         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3092                 for (j = 0; j < table_size; j++) {
3093                         t = (1000 * ((i + 1) * 8));
3094
3095                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3096                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3097
3098                         ni_calculate_leakage_for_v_and_t(rdev,
3099                                                          &ni_pi->cac_data.leakage_coefficients,
3100                                                          eg_pi->vddc_voltage_table.entries[j].value,
3101                                                          t,
3102                                                          ni_pi->cac_data.i_leakage,
3103                                                          &leakage);
3104
3105                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3106                         if (smc_leakage > max_leakage)
3107                                 max_leakage = smc_leakage;
3108
3109                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3110                 }
3111         }
3112
3113         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3114                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3115                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3116         }
3117         return 0;
3118 }
3119
3120 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3121                                             PP_NIslands_CACTABLES *cac_tables)
3122 {
3123         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3124         struct radeon_cac_leakage_table *leakage_table =
3125                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3126         u32 i, j, table_size;
3127         u32 smc_leakage, max_leakage = 0;
3128         u32 scaling_factor;
3129
3130         if (!leakage_table)
3131                 return -EINVAL;
3132
3133         table_size = leakage_table->count;
3134
3135         if (eg_pi->vddc_voltage_table.count != table_size)
3136                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3137                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3138
3139         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3140                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3141
3142         if (table_size == 0)
3143                 return -EINVAL;
3144
3145         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3146
3147         for (j = 0; j < table_size; j++) {
3148                 smc_leakage = leakage_table->entries[j].leakage;
3149
3150                 if (smc_leakage > max_leakage)
3151                         max_leakage = smc_leakage;
3152
3153                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3154                         cac_tables->cac_lkge_lut[i][j] =
3155                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3156         }
3157
3158         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3159                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3160                         cac_tables->cac_lkge_lut[i][j] =
3161                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3162         }
3163         return 0;
3164 }
3165
3166 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3167 {
3168         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3169         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3170         PP_NIslands_CACTABLES *cac_tables = NULL;
3171         int i, ret;
3172         u32 reg;
3173
3174         if (ni_pi->enable_cac == false)
3175                 return 0;
3176
3177         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3178         if (!cac_tables)
3179                 return -ENOMEM;
3180
3181         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3182         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3183                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3184         WREG32(CG_CAC_CTRL, reg);
3185
3186         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3187                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3188
3189         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3190                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3191
3192         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3193         ni_pi->cac_data.pwr_const = 0;
3194         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3195         ni_pi->cac_data.bif_cac_value = 0;
3196         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3197         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3198         ni_pi->cac_data.allow_ovrflw = 0;
3199         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3200         ni_pi->cac_data.num_win_tdp = 0;
3201         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3202
3203         if (ni_pi->driver_calculate_cac_leakage)
3204                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3205         else
3206                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3207
3208         if (ret)
3209                 goto done_free;
3210
3211         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3212         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3213         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3214         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3215         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3216         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3217         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3218         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3219         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3220
3221         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3222                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3223
3224 done_free:
3225         if (ret) {
3226                 ni_pi->enable_cac = false;
3227                 ni_pi->enable_power_containment = false;
3228         }
3229
3230         kfree(cac_tables);
3231
3232         return 0;
3233 }
3234
3235 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3236 {
3237         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3238         u32 reg;
3239
3240         if (!ni_pi->enable_cac ||
3241             !ni_pi->cac_configuration_required)
3242                 return 0;
3243
3244         if (ni_pi->cac_weights == NULL)
3245                 return -EINVAL;
3246
3247         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3248                                                       WEIGHT_TCP_SIG1_MASK |
3249                                                       WEIGHT_TA_SIG_MASK);
3250         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3251                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3252                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3253         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3254
3255         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3256                                                       WEIGHT_TCC_EN1_MASK |
3257                                                       WEIGHT_TCC_EN2_MASK);
3258         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3259                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3260                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3261         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3262
3263         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3264                                                       WEIGHT_CB_EN1_MASK |
3265                                                       WEIGHT_CB_EN2_MASK |
3266                                                       WEIGHT_CB_EN3_MASK);
3267         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3268                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3269                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3270                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3271         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3272
3273         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3274                                                       WEIGHT_DB_SIG1_MASK |
3275                                                       WEIGHT_DB_SIG2_MASK |
3276                                                       WEIGHT_DB_SIG3_MASK);
3277         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3278                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3279                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3280                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3281         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3282
3283         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3284                                                       WEIGHT_SXM_SIG1_MASK |
3285                                                       WEIGHT_SXM_SIG2_MASK |
3286                                                       WEIGHT_SXS_SIG0_MASK |
3287                                                       WEIGHT_SXS_SIG1_MASK);
3288         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3289                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3290                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3291                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3292                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3293         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3294
3295         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3296                                                       WEIGHT_XBR_1_MASK |
3297                                                       WEIGHT_XBR_2_MASK |
3298                                                       WEIGHT_SPI_SIG0_MASK);
3299         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3300                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3301                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3302                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3303         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3304
3305         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3306                                                       WEIGHT_SPI_SIG2_MASK |
3307                                                       WEIGHT_SPI_SIG3_MASK |
3308                                                       WEIGHT_SPI_SIG4_MASK |
3309                                                       WEIGHT_SPI_SIG5_MASK);
3310         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3311                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3312                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3313                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3314                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3315         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3316
3317         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3318                                                       WEIGHT_LDS_SIG1_MASK |
3319                                                       WEIGHT_SC_MASK);
3320         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3321                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3322                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3323         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3324
3325         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3326                                                       WEIGHT_CP_MASK |
3327                                                       WEIGHT_PA_SIG0_MASK |
3328                                                       WEIGHT_PA_SIG1_MASK |
3329                                                       WEIGHT_VGT_SIG0_MASK);
3330         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3331                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3332                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3333                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3334                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3335         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3336
3337         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3338                                                       WEIGHT_VGT_SIG2_MASK |
3339                                                       WEIGHT_DC_SIG0_MASK |
3340                                                       WEIGHT_DC_SIG1_MASK |
3341                                                       WEIGHT_DC_SIG2_MASK);
3342         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3343                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3344                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3345                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3346                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3347         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3348
3349         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3350                                                       WEIGHT_UVD_SIG0_MASK |
3351                                                       WEIGHT_UVD_SIG1_MASK |
3352                                                       WEIGHT_SPARE0_MASK |
3353                                                       WEIGHT_SPARE1_MASK);
3354         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3355                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3356                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3357                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3358                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3359         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3360
3361         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3362                                                       WEIGHT_SQ_VSP0_MASK);
3363         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3364                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3365         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3366
3367         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3368         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3369         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3370
3371         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3372                                                         OVR_VAL_SPARE_0_MASK |
3373                                                         OVR_MODE_SPARE_1_MASK |
3374                                                         OVR_VAL_SPARE_1_MASK);
3375         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3376                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3377                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3378                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3379         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3380
3381         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3382                                            VSP0_MASK |
3383                                            GPR_MASK);
3384         reg |= (VSP(ni_pi->cac_weights->vsp) |
3385                 VSP0(ni_pi->cac_weights->vsp0) |
3386                 GPR(ni_pi->cac_weights->gpr));
3387         WREG32(SQ_CAC_THRESHOLD, reg);
3388
3389         reg = (MCDW_WR_ENABLE |
3390                MCDX_WR_ENABLE |
3391                MCDY_WR_ENABLE |
3392                MCDZ_WR_ENABLE |
3393                INDEX(0x09D4));
3394         WREG32(MC_CG_CONFIG, reg);
3395
3396         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3397                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3398                ALLOW_OVERFLOW);
3399         WREG32(MC_CG_DATAPORT, reg);
3400
3401         return 0;
3402 }
3403
3404 static int ni_enable_smc_cac(struct radeon_device *rdev,
3405                              struct radeon_ps *radeon_new_state,
3406                              bool enable)
3407 {
3408         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3409         int ret = 0;
3410         PPSMC_Result smc_result;
3411
3412         if (ni_pi->enable_cac) {
3413                 if (enable) {
3414                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3415                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3416
3417                                 if (ni_pi->support_cac_long_term_average) {
3418                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3419                                         if (PPSMC_Result_OK != smc_result)
3420                                                 ni_pi->support_cac_long_term_average = false;
3421                                 }
3422
3423                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3424                                 if (PPSMC_Result_OK != smc_result)
3425                                         ret = -EINVAL;
3426
3427                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3428                         }
3429                 } else if (ni_pi->cac_enabled) {
3430                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3431
3432                         ni_pi->cac_enabled = false;
3433
3434                         if (ni_pi->support_cac_long_term_average) {
3435                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3436                                 if (PPSMC_Result_OK != smc_result)
3437                                         ni_pi->support_cac_long_term_average = false;
3438                         }
3439                 }
3440         }
3441
3442         return ret;
3443 }
3444
3445 static int ni_pcie_performance_request(struct radeon_device *rdev,
3446                                        u8 perf_req, bool advertise)
3447 {
3448 #if defined(CONFIG_ACPI)
3449         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3450
3451         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3452             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3453                 if (eg_pi->pcie_performance_request_registered == false)
3454                         radeon_acpi_pcie_notify_device_ready(rdev);
3455                 eg_pi->pcie_performance_request_registered = true;
3456                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3457         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3458                    eg_pi->pcie_performance_request_registered) {
3459                 eg_pi->pcie_performance_request_registered = false;
3460                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3461         }
3462 #endif
3463         return 0;
3464 }
3465
3466 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3467 {
3468         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3469         u32 tmp;
3470
3471         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3472
3473         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3474             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3475                 pi->pcie_gen2 = true;
3476         else
3477                 pi->pcie_gen2 = false;
3478
3479         if (!pi->pcie_gen2)
3480                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3481
3482         return 0;
3483 }
3484
3485 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3486                                             bool enable)
3487 {
3488         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3489         u32 tmp, bif;
3490
3491         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3492
3493         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3494             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3495                 if (enable) {
3496                         if (!pi->boot_in_gen2) {
3497                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3498                                 bif |= CG_CLIENT_REQ(0xd);
3499                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3500                         }
3501                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3502                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3503                         tmp |= LC_GEN2_EN_STRAP;
3504
3505                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3506                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3507                         udelay(10);
3508                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3509                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3510                 } else {
3511                         if (!pi->boot_in_gen2) {
3512                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3513                                 bif |= CG_CLIENT_REQ(0xd);
3514                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3515
3516                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3517                                 tmp &= ~LC_GEN2_EN_STRAP;
3518                         }
3519                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3520                 }
3521         }
3522 }
3523
3524 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3525                                         bool enable)
3526 {
3527         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3528
3529         if (enable)
3530                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3531         else
3532                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3533 }
3534
3535 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3536                                            struct radeon_ps *new_ps,
3537                                            struct radeon_ps *old_ps)
3538 {
3539         struct ni_ps *new_state = ni_get_ps(new_ps);
3540         struct ni_ps *current_state = ni_get_ps(old_ps);
3541
3542         if ((new_ps->vclk == old_ps->vclk) &&
3543             (new_ps->dclk == old_ps->dclk))
3544                 return;
3545
3546         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3547             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3548                 return;
3549
3550         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3551 }
3552
3553 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3554                                           struct radeon_ps *new_ps,
3555                                           struct radeon_ps *old_ps)
3556 {
3557         struct ni_ps *new_state = ni_get_ps(new_ps);
3558         struct ni_ps *current_state = ni_get_ps(old_ps);
3559
3560         if ((new_ps->vclk == old_ps->vclk) &&
3561             (new_ps->dclk == old_ps->dclk))
3562                 return;
3563
3564         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3565             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3566                 return;
3567
3568         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3569 }
3570
3571 void ni_dpm_setup_asic(struct radeon_device *rdev)
3572 {
3573         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3574
3575         ni_read_clock_registers(rdev);
3576         btc_read_arb_registers(rdev);
3577         rv770_get_memory_type(rdev);
3578         if (eg_pi->pcie_performance_request)
3579                 ni_advertise_gen2_capability(rdev);
3580         rv770_get_pcie_gen2_status(rdev);
3581         rv770_enable_acpi_pm(rdev);
3582 }
3583
3584 void ni_update_current_ps(struct radeon_device *rdev,
3585                           struct radeon_ps *rps)
3586 {
3587         struct ni_ps *new_ps = ni_get_ps(rps);
3588         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3589         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3590
3591         eg_pi->current_rps = *rps;
3592         ni_pi->current_ps = *new_ps;
3593         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3594 }
3595
3596 void ni_update_requested_ps(struct radeon_device *rdev,
3597                             struct radeon_ps *rps)
3598 {
3599         struct ni_ps *new_ps = ni_get_ps(rps);
3600         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3601         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3602
3603         eg_pi->requested_rps = *rps;
3604         ni_pi->requested_ps = *new_ps;
3605         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3606 }
3607
3608 int ni_dpm_enable(struct radeon_device *rdev)
3609 {
3610         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3611         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3612         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3613         int ret;
3614
3615         if (pi->gfx_clock_gating)
3616                 ni_cg_clockgating_default(rdev);
3617         if (btc_dpm_enabled(rdev))
3618                 return -EINVAL;
3619         if (pi->mg_clock_gating)
3620                 ni_mg_clockgating_default(rdev);
3621         if (eg_pi->ls_clock_gating)
3622                 ni_ls_clockgating_default(rdev);
3623         if (pi->voltage_control) {
3624                 rv770_enable_voltage_control(rdev, true);
3625                 ret = cypress_construct_voltage_tables(rdev);
3626                 if (ret) {
3627                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3628                         return ret;
3629                 }
3630         }
3631         if (eg_pi->dynamic_ac_timing) {
3632                 ret = ni_initialize_mc_reg_table(rdev);
3633                 if (ret)
3634                         eg_pi->dynamic_ac_timing = false;
3635         }
3636         if (pi->dynamic_ss)
3637                 cypress_enable_spread_spectrum(rdev, true);
3638         if (pi->thermal_protection)
3639                 rv770_enable_thermal_protection(rdev, true);
3640         rv770_setup_bsp(rdev);
3641         rv770_program_git(rdev);
3642         rv770_program_tp(rdev);
3643         rv770_program_tpp(rdev);
3644         rv770_program_sstp(rdev);
3645         cypress_enable_display_gap(rdev);
3646         rv770_program_vc(rdev);
3647         if (pi->dynamic_pcie_gen2)
3648                 ni_enable_dynamic_pcie_gen2(rdev, true);
3649         ret = rv770_upload_firmware(rdev);
3650         if (ret) {
3651                 DRM_ERROR("rv770_upload_firmware failed\n");
3652                 return ret;
3653         }
3654         ret = ni_process_firmware_header(rdev);
3655         if (ret) {
3656                 DRM_ERROR("ni_process_firmware_header failed\n");
3657                 return ret;
3658         }
3659         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3660         if (ret) {
3661                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3662                 return ret;
3663         }
3664         ret = ni_init_smc_table(rdev);
3665         if (ret) {
3666                 DRM_ERROR("ni_init_smc_table failed\n");
3667                 return ret;
3668         }
3669         ret = ni_init_smc_spll_table(rdev);
3670         if (ret) {
3671                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3672                 return ret;
3673         }
3674         ret = ni_init_arb_table_index(rdev);
3675         if (ret) {
3676                 DRM_ERROR("ni_init_arb_table_index failed\n");
3677                 return ret;
3678         }
3679         if (eg_pi->dynamic_ac_timing) {
3680                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3681                 if (ret) {
3682                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3683                         return ret;
3684                 }
3685         }
3686         ret = ni_initialize_smc_cac_tables(rdev);
3687         if (ret) {
3688                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3689                 return ret;
3690         }
3691         ret = ni_initialize_hardware_cac_manager(rdev);
3692         if (ret) {
3693                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3694                 return ret;
3695         }
3696         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3697         if (ret) {
3698                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3699                 return ret;
3700         }
3701         ni_program_response_times(rdev);
3702         r7xx_start_smc(rdev);
3703         ret = cypress_notify_smc_display_change(rdev, false);
3704         if (ret) {
3705                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3706                 return ret;
3707         }
3708         cypress_enable_sclk_control(rdev, true);
3709         if (eg_pi->memory_transition)
3710                 cypress_enable_mclk_control(rdev, true);
3711         cypress_start_dpm(rdev);
3712         if (pi->gfx_clock_gating)
3713                 ni_gfx_clockgating_enable(rdev, true);
3714         if (pi->mg_clock_gating)
3715                 ni_mg_clockgating_enable(rdev, true);
3716         if (eg_pi->ls_clock_gating)
3717                 ni_ls_clockgating_enable(rdev, true);
3718
3719         if (rdev->irq.installed &&
3720             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3721                 PPSMC_Result result;
3722
3723                 ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, 0xff * 1000);
3724                 if (ret)
3725                         return ret;
3726                 rdev->irq.dpm_thermal = true;
3727                 radeon_irq_set(rdev);
3728                 result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
3729
3730                 if (result != PPSMC_Result_OK)
3731                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
3732         }
3733
3734         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3735
3736         ni_update_current_ps(rdev, boot_ps);
3737
3738         return 0;
3739 }
3740
3741 void ni_dpm_disable(struct radeon_device *rdev)
3742 {
3743         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3744         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3745         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3746
3747         if (!btc_dpm_enabled(rdev))
3748                 return;
3749         rv770_clear_vc(rdev);
3750         if (pi->thermal_protection)
3751                 rv770_enable_thermal_protection(rdev, false);
3752         ni_enable_power_containment(rdev, boot_ps, false);
3753         ni_enable_smc_cac(rdev, boot_ps, false);
3754         cypress_enable_spread_spectrum(rdev, false);
3755         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3756         if (pi->dynamic_pcie_gen2)
3757                 ni_enable_dynamic_pcie_gen2(rdev, false);
3758
3759         if (rdev->irq.installed &&
3760             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3761                 rdev->irq.dpm_thermal = false;
3762                 radeon_irq_set(rdev);
3763         }
3764
3765         if (pi->gfx_clock_gating)
3766                 ni_gfx_clockgating_enable(rdev, false);
3767         if (pi->mg_clock_gating)
3768                 ni_mg_clockgating_enable(rdev, false);
3769         if (eg_pi->ls_clock_gating)
3770                 ni_ls_clockgating_enable(rdev, false);
3771         ni_stop_dpm(rdev);
3772         btc_reset_to_default(rdev);
3773         ni_stop_smc(rdev);
3774         ni_force_switch_to_arb_f0(rdev);
3775
3776         ni_update_current_ps(rdev, boot_ps);
3777 }
3778
3779 static int ni_power_control_set_level(struct radeon_device *rdev)
3780 {
3781         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3782         int ret;
3783
3784         ret = ni_restrict_performance_levels_before_switch(rdev);
3785         if (ret)
3786                 return ret;
3787         ret = rv770_halt_smc(rdev);
3788         if (ret)
3789                 return ret;
3790         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3791         if (ret)
3792                 return ret;
3793         ret = rv770_resume_smc(rdev);
3794         if (ret)
3795                 return ret;
3796         ret = rv770_set_sw_state(rdev);
3797         if (ret)
3798                 return ret;
3799
3800         return 0;
3801 }
3802
3803 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3804 {
3805         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3806         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3807         struct radeon_ps *new_ps = &requested_ps;
3808
3809         ni_update_requested_ps(rdev, new_ps);
3810
3811         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3812
3813         return 0;
3814 }
3815
3816 int ni_dpm_set_power_state(struct radeon_device *rdev)
3817 {
3818         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3819         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3820         struct radeon_ps *old_ps = &eg_pi->current_rps;
3821         int ret;
3822
3823         ret = ni_restrict_performance_levels_before_switch(rdev);
3824         if (ret) {
3825                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3826                 return ret;
3827         }
3828         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3829         ret = ni_enable_power_containment(rdev, new_ps, false);
3830         if (ret) {
3831                 DRM_ERROR("ni_enable_power_containment failed\n");
3832                 return ret;
3833         }
3834         ret = ni_enable_smc_cac(rdev, new_ps, false);
3835         if (ret) {
3836                 DRM_ERROR("ni_enable_smc_cac failed\n");
3837                 return ret;
3838         }
3839         ret = rv770_halt_smc(rdev);
3840         if (ret) {
3841                 DRM_ERROR("rv770_halt_smc failed\n");
3842                 return ret;
3843         }
3844         if (eg_pi->smu_uvd_hs)
3845                 btc_notify_uvd_to_smc(rdev, new_ps);
3846         ret = ni_upload_sw_state(rdev, new_ps);
3847         if (ret) {
3848                 DRM_ERROR("ni_upload_sw_state failed\n");
3849                 return ret;
3850         }
3851         if (eg_pi->dynamic_ac_timing) {
3852                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3853                 if (ret) {
3854                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3855                         return ret;
3856                 }
3857         }
3858         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3859         if (ret) {
3860                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3861                 return ret;
3862         }
3863         ret = rv770_resume_smc(rdev);
3864         if (ret) {
3865                 DRM_ERROR("rv770_resume_smc failed\n");
3866                 return ret;
3867         }
3868         ret = rv770_set_sw_state(rdev);
3869         if (ret) {
3870                 DRM_ERROR("rv770_set_sw_state failed\n");
3871                 return ret;
3872         }
3873         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3874         ret = ni_enable_smc_cac(rdev, new_ps, true);
3875         if (ret) {
3876                 DRM_ERROR("ni_enable_smc_cac failed\n");
3877                 return ret;
3878         }
3879         ret = ni_enable_power_containment(rdev, new_ps, true);
3880         if (ret) {
3881                 DRM_ERROR("ni_enable_power_containment failed\n");
3882                 return ret;
3883         }
3884
3885         /* update tdp */
3886         ret = ni_power_control_set_level(rdev);
3887         if (ret) {
3888                 DRM_ERROR("ni_power_control_set_level failed\n");
3889                 return ret;
3890         }
3891
3892         return 0;
3893 }
3894
3895 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3896 {
3897         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3898         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3899
3900         ni_update_current_ps(rdev, new_ps);
3901 }
3902
3903 void ni_dpm_reset_asic(struct radeon_device *rdev)
3904 {
3905         ni_restrict_performance_levels_before_switch(rdev);
3906         rv770_set_boot_state(rdev);
3907 }
3908
3909 union power_info {
3910         struct _ATOM_POWERPLAY_INFO info;
3911         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3912         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3913         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3914         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3915         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3916 };
3917
3918 union pplib_clock_info {
3919         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3920         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3921         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3922         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3923 };
3924
3925 union pplib_power_state {
3926         struct _ATOM_PPLIB_STATE v1;
3927         struct _ATOM_PPLIB_STATE_V2 v2;
3928 };
3929
3930 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3931                                           struct radeon_ps *rps,
3932                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3933                                           u8 table_rev)
3934 {
3935         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3936         rps->class = le16_to_cpu(non_clock_info->usClassification);
3937         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3938
3939         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3940                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3941                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3942         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3943                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3944                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3945         } else {
3946                 rps->vclk = 0;
3947                 rps->dclk = 0;
3948         }
3949
3950         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3951                 rdev->pm.dpm.boot_ps = rps;
3952         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3953                 rdev->pm.dpm.uvd_ps = rps;
3954 }
3955
3956 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3957                                       struct radeon_ps *rps, int index,
3958                                       union pplib_clock_info *clock_info)
3959 {
3960         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3961         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3962         struct ni_ps *ps = ni_get_ps(rps);
3963         u16 vddc;
3964         struct rv7xx_pl *pl = &ps->performance_levels[index];
3965
3966         ps->performance_level_count = index + 1;
3967
3968         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3969         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3970         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3971         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3972
3973         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3974         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3975         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3976
3977         /* patch up vddc if necessary */
3978         if (pl->vddc == 0xff01) {
3979                 if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
3980                         pl->vddc = vddc;
3981         }
3982
3983         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3984                 pi->acpi_vddc = pl->vddc;
3985                 eg_pi->acpi_vddci = pl->vddci;
3986                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3987                         pi->acpi_pcie_gen2 = true;
3988                 else
3989                         pi->acpi_pcie_gen2 = false;
3990         }
3991
3992         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3993                 eg_pi->ulv.supported = true;
3994                 eg_pi->ulv.pl = pl;
3995         }
3996
3997         if (pi->min_vddc_in_table > pl->vddc)
3998                 pi->min_vddc_in_table = pl->vddc;
3999
4000         if (pi->max_vddc_in_table < pl->vddc)
4001                 pi->max_vddc_in_table = pl->vddc;
4002
4003         /* patch up boot state */
4004         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
4005                 u16 vddc, vddci, mvdd;
4006                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
4007                 pl->mclk = rdev->clock.default_mclk;
4008                 pl->sclk = rdev->clock.default_sclk;
4009                 pl->vddc = vddc;
4010                 pl->vddci = vddci;
4011         }
4012
4013         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
4014             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
4015                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
4016                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
4017                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
4018                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
4019         }
4020 }
4021
4022 static int ni_parse_power_table(struct radeon_device *rdev)
4023 {
4024         struct radeon_mode_info *mode_info = &rdev->mode_info;
4025         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
4026         union pplib_power_state *power_state;
4027         int i, j;
4028         union pplib_clock_info *clock_info;
4029         union power_info *power_info;
4030         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
4031         u16 data_offset;
4032         u8 frev, crev;
4033         struct ni_ps *ps;
4034
4035         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
4036                                    &frev, &crev, &data_offset))
4037                 return -EINVAL;
4038         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4039
4040         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
4041                                   power_info->pplib.ucNumStates, GFP_KERNEL);
4042         if (!rdev->pm.dpm.ps)
4043                 return -ENOMEM;
4044         rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
4045         rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
4046         rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
4047
4048         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4049                 power_state = (union pplib_power_state *)
4050                         (mode_info->atom_context->bios + data_offset +
4051                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4052                          i * power_info->pplib.ucStateEntrySize);
4053                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4054                         (mode_info->atom_context->bios + data_offset +
4055                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4056                          (power_state->v1.ucNonClockStateIndex *
4057                           power_info->pplib.ucNonClockSize));
4058                 if (power_info->pplib.ucStateEntrySize - 1) {
4059                         u8 *idx;
4060                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4061                         if (ps == NULL) {
4062                                 kfree(rdev->pm.dpm.ps);
4063                                 return -ENOMEM;
4064                         }
4065                         rdev->pm.dpm.ps[i].ps_priv = ps;
4066                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4067                                                          non_clock_info,
4068                                                          power_info->pplib.ucNonClockSize);
4069                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4070                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4071                                 clock_info = (union pplib_clock_info *)
4072                                         (mode_info->atom_context->bios + data_offset +
4073                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4074                                          (idx[j] * power_info->pplib.ucClockInfoSize));
4075                                 ni_parse_pplib_clock_info(rdev,
4076                                                           &rdev->pm.dpm.ps[i], j,
4077                                                           clock_info);
4078                         }
4079                 }
4080         }
4081         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4082         return 0;
4083 }
4084
4085 int ni_dpm_init(struct radeon_device *rdev)
4086 {
4087         struct rv7xx_power_info *pi;
4088         struct evergreen_power_info *eg_pi;
4089         struct ni_power_info *ni_pi;
4090         struct atom_clock_dividers dividers;
4091         int ret;
4092
4093         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4094         if (ni_pi == NULL)
4095                 return -ENOMEM;
4096         rdev->pm.dpm.priv = ni_pi;
4097         eg_pi = &ni_pi->eg;
4098         pi = &eg_pi->rv7xx;
4099
4100         rv770_get_max_vddc(rdev);
4101
4102         eg_pi->ulv.supported = false;
4103         pi->acpi_vddc = 0;
4104         eg_pi->acpi_vddci = 0;
4105         pi->min_vddc_in_table = 0;
4106         pi->max_vddc_in_table = 0;
4107
4108         ret = ni_parse_power_table(rdev);
4109         if (ret)
4110                 return ret;
4111         ret = r600_parse_extended_power_table(rdev);
4112         if (ret)
4113                 return ret;
4114
4115         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4116                 kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
4117         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4118                 r600_free_extended_power_table(rdev);
4119                 return -ENOMEM;
4120         }
4121         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4122         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4123         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4124         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4125         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4126         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4127         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4128         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4129         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4130
4131         ni_patch_dependency_tables_based_on_leakage(rdev);
4132
4133         if (rdev->pm.dpm.voltage_response_time == 0)
4134                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4135         if (rdev->pm.dpm.backbias_response_time == 0)
4136                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4137
4138         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4139                                              0, false, &dividers);
4140         if (ret)
4141                 pi->ref_div = dividers.ref_div + 1;
4142         else
4143                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4144
4145         pi->rlp = RV770_RLP_DFLT;
4146         pi->rmp = RV770_RMP_DFLT;
4147         pi->lhp = RV770_LHP_DFLT;
4148         pi->lmp = RV770_LMP_DFLT;
4149
4150         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4151         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4152         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4153         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4154
4155         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4156         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4157         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4158         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4159
4160         eg_pi->smu_uvd_hs = true;
4161
4162         if (rdev->pdev->device == 0x6707) {
4163                 pi->mclk_strobe_mode_threshold = 55000;
4164                 pi->mclk_edc_enable_threshold = 55000;
4165                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4166         } else {
4167                 pi->mclk_strobe_mode_threshold = 40000;
4168                 pi->mclk_edc_enable_threshold = 40000;
4169                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4170         }
4171         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4172
4173         pi->voltage_control =
4174                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4175
4176         pi->mvdd_control =
4177                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4178
4179         eg_pi->vddci_control =
4180                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4181
4182         rv770_get_engine_memory_ss(rdev);
4183
4184         pi->asi = RV770_ASI_DFLT;
4185         pi->pasi = CYPRESS_HASI_DFLT;
4186         pi->vrc = CYPRESS_VRC_DFLT;
4187
4188         pi->power_gating = false;
4189
4190         pi->gfx_clock_gating = true;
4191
4192         pi->mg_clock_gating = true;
4193         pi->mgcgtssm = true;
4194         eg_pi->ls_clock_gating = false;
4195         eg_pi->sclk_deep_sleep = false;
4196
4197         pi->dynamic_pcie_gen2 = true;
4198
4199         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4200                 pi->thermal_protection = true;
4201         else
4202                 pi->thermal_protection = false;
4203
4204         pi->display_gap = true;
4205
4206         pi->dcodt = true;
4207
4208         pi->ulps = true;
4209
4210         eg_pi->dynamic_ac_timing = true;
4211         eg_pi->abm = true;
4212         eg_pi->mcls = true;
4213         eg_pi->light_sleep = true;
4214         eg_pi->memory_transition = true;
4215 #if defined(CONFIG_ACPI)
4216         eg_pi->pcie_performance_request =
4217                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4218 #else
4219         eg_pi->pcie_performance_request = false;
4220 #endif
4221
4222         eg_pi->dll_default_on = false;
4223
4224         eg_pi->sclk_deep_sleep = false;
4225
4226         pi->mclk_stutter_mode_threshold = 0;
4227
4228         pi->sram_end = SMC_RAM_END;
4229
4230         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4231         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4232         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4233         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4234         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4235         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4236         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4237         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4238
4239         ni_pi->cac_data.leakage_coefficients.at = 516;
4240         ni_pi->cac_data.leakage_coefficients.bt = 18;
4241         ni_pi->cac_data.leakage_coefficients.av = 51;
4242         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4243
4244         switch (rdev->pdev->device) {
4245         case 0x6700:
4246         case 0x6701:
4247         case 0x6702:
4248         case 0x6703:
4249         case 0x6718:
4250                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4251                 break;
4252         case 0x6705:
4253         case 0x6719:
4254         case 0x671D:
4255         case 0x671C:
4256         default:
4257                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4258                 break;
4259         case 0x6704:
4260         case 0x6706:
4261         case 0x6707:
4262         case 0x6708:
4263         case 0x6709:
4264                 ni_pi->cac_weights = &cac_weights_cayman_le;
4265                 break;
4266         }
4267
4268         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4269                 ni_pi->enable_power_containment = true;
4270                 ni_pi->enable_cac = true;
4271                 ni_pi->enable_sq_ramping = true;
4272         } else {
4273                 ni_pi->enable_power_containment = false;
4274                 ni_pi->enable_cac = false;
4275                 ni_pi->enable_sq_ramping = false;
4276         }
4277
4278         ni_pi->driver_calculate_cac_leakage = false;
4279         ni_pi->cac_configuration_required = true;
4280
4281         if (ni_pi->cac_configuration_required) {
4282                 ni_pi->support_cac_long_term_average = true;
4283                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4284                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4285         } else {
4286                 ni_pi->support_cac_long_term_average = false;
4287                 ni_pi->lta_window_size = 0;
4288                 ni_pi->lts_truncate = 0;
4289         }
4290
4291         ni_pi->use_power_boost_limit = true;
4292
4293         /* make sure dc limits are valid */
4294         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4295             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4296                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4297                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4298
4299         return 0;
4300 }
4301
4302 void ni_dpm_fini(struct radeon_device *rdev)
4303 {
4304         int i;
4305
4306         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4307                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4308         }
4309         kfree(rdev->pm.dpm.ps);
4310         kfree(rdev->pm.dpm.priv);
4311         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4312         r600_free_extended_power_table(rdev);
4313 }
4314
4315 void ni_dpm_print_power_state(struct radeon_device *rdev,
4316                               struct radeon_ps *rps)
4317 {
4318         struct ni_ps *ps = ni_get_ps(rps);
4319         struct rv7xx_pl *pl;
4320         int i;
4321
4322         r600_dpm_print_class_info(rps->class, rps->class2);
4323         r600_dpm_print_cap_info(rps->caps);
4324         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4325         for (i = 0; i < ps->performance_level_count; i++) {
4326                 pl = &ps->performance_levels[i];
4327                 if (rdev->family >= CHIP_TAHITI)
4328                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4329                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4330                 else
4331                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4332                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4333         }
4334         r600_dpm_print_ps_status(rdev, rps);
4335 }
4336
4337 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4338                                                     struct seq_file *m)
4339 {
4340         struct radeon_ps *rps = rdev->pm.dpm.current_ps;
4341         struct ni_ps *ps = ni_get_ps(rps);
4342         struct rv7xx_pl *pl;
4343         u32 current_index =
4344                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4345                 CURRENT_STATE_INDEX_SHIFT;
4346
4347         if (current_index >= ps->performance_level_count) {
4348                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4349         } else {
4350                 pl = &ps->performance_levels[current_index];
4351                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4352                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4353                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4354         }
4355 }
4356
4357 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4358 {
4359         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4360         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4361
4362         if (low)
4363                 return requested_state->performance_levels[0].sclk;
4364         else
4365                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4366 }
4367
4368 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4369 {
4370         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4371         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4372
4373         if (low)
4374                 return requested_state->performance_levels[0].mclk;
4375         else
4376                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4377 }
4378