drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <linux/circ_buf.h>
  34 #include <drm/drmP.h>
  35 #include <drm/i915_drm.h>
  36 #include "i915_drv.h"
  37 #include "i915_trace.h"
  38 #include "intel_drv.h"
  39
  40 /**
  41  * DOC: interrupt handling
  42  *
  43  * These functions provide the basic support for enabling and disabling the
  44  * interrupt handling support. There's a lot more functionality in i915_irq.c
  45  * and related files, but that will be described in separate chapters.
  46  */
  47
  48 static const u32 hpd_ilk[HPD_NUM_PINS] = {
  49         [HPD_PORT_A] = DE_DP_A_HOTPLUG,
  50 };
  51
  52 static const u32 hpd_ivb[HPD_NUM_PINS] = {
  53         [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
  54 };
  55
  56 static const u32 hpd_bdw[HPD_NUM_PINS] = {
  57         [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
  58 };
  59
  60 static const u32 hpd_ibx[HPD_NUM_PINS] = {
  61         [HPD_CRT] = SDE_CRT_HOTPLUG,
  62         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  63         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  64         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  65         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  66 };
  67
  68 static const u32 hpd_cpt[HPD_NUM_PINS] = {
  69         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  70         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  71         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  72         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  73         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  74 };
  75
  76 static const u32 hpd_spt[HPD_NUM_PINS] = {
  77         [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
  78         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  79         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  80         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
  81         [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
  82 };
  83
  84 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
  85         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  86         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  87         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  88         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  89         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  90         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  91 };
  92
  93 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
  94         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  95         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  96         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  97         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  98         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  99         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
 100 };
 101
 102 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
 103         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
 104         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
 105         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
 106         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
 107         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
 108         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
 109 };
 110
 111 /* BXT hpd list */
 112 static const u32 hpd_bxt[HPD_NUM_PINS] = {
 113         [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
 114         [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
 115         [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
 116 };
 117
 118 /* IIR can theoretically queue up two events. Be paranoid. */
 119 #define GEN8_IRQ_RESET_NDX(type, which) do { \
 120         I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
 121         POSTING_READ(GEN8_##type##_IMR(which)); \
 122         I915_WRITE(GEN8_##type##_IER(which), 0); \
 123         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 124         POSTING_READ(GEN8_##type##_IIR(which)); \
 125         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 126         POSTING_READ(GEN8_##type##_IIR(which)); \
 127 } while (0)
 128
 129 #define GEN5_IRQ_RESET(type) do { \
 130         I915_WRITE(type##IMR, 0xffffffff); \
 131         POSTING_READ(type##IMR); \
 132         I915_WRITE(type##IER, 0); \
 133         I915_WRITE(type##IIR, 0xffffffff); \
 134         POSTING_READ(type##IIR); \
 135         I915_WRITE(type##IIR, 0xffffffff); \
 136         POSTING_READ(type##IIR); \
 137 } while (0)
 138
 139 /*
 140  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 141  */
 142 static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
 143                                     i915_reg_t reg)
 144 {
 145         u32 val = I915_READ(reg);
 146
 147         if (val == 0)
 148                 return;
 149
 150         WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
 151              i915_mmio_reg_offset(reg), val);
 152         I915_WRITE(reg, 0xffffffff);
 153         POSTING_READ(reg);
 154         I915_WRITE(reg, 0xffffffff);
 155         POSTING_READ(reg);
 156 }
 157
 158 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
 159         gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
 160         I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
 161         I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
 162         POSTING_READ(GEN8_##type##_IMR(which)); \
 163 } while (0)
 164
 165 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
 166         gen5_assert_iir_is_zero(dev_priv, type##IIR); \
 167         I915_WRITE(type##IER, (ier_val)); \
 168         I915_WRITE(type##IMR, (imr_val)); \
 169         POSTING_READ(type##IMR); \
 170 } while (0)
 171
 172 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
 173
 174 /* For display hotplug interrupt */
 175 static inline void
 176 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
 177                                      uint32_t mask,
 178                                      uint32_t bits)
 179 {
 180         uint32_t val;
 181
 182         assert_spin_locked(&dev_priv->irq_lock);
 183         WARN_ON(bits & ~mask);
 184
 185         val = I915_READ(PORT_HOTPLUG_EN);
 186         val &= ~mask;
 187         val |= bits;
 188         I915_WRITE(PORT_HOTPLUG_EN, val);
 189 }
 190
 191 /**
 192  * i915_hotplug_interrupt_update - update hotplug interrupt enable
 193  * @dev_priv: driver private
 194  * @mask: bits to update
 195  * @bits: bits to enable
 196  * NOTE: the HPD enable bits are modified both inside and outside
 197  * of an interrupt context. To avoid that read-modify-write cycles
 198  * interfer, these bits are protected by a spinlock. Since this
 199  * function is usually not called from a context where the lock is
 200  * held already, this function acquires the lock itself. A non-locking
 201  * version is also available.
 202  */
 203 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
 204                                    uint32_t mask,
 205                                    uint32_t bits)
 206 {
 207         spin_lock_irq(&dev_priv->irq_lock);
 208         i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
 209         spin_unlock_irq(&dev_priv->irq_lock);
 210 }
 211
 212 /**
 213  * ilk_update_display_irq - update DEIMR
 214  * @dev_priv: driver private
 215  * @interrupt_mask: mask of interrupt bits to update
 216  * @enabled_irq_mask: mask of interrupt bits to enable
 217  */
 218 static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
 219                                    uint32_t interrupt_mask,
 220                                    uint32_t enabled_irq_mask)
 221 {
 222         uint32_t new_val;
 223
 224         assert_spin_locked(&dev_priv->irq_lock);
 225
 226         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 227
 228         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 229                 return;
 230
 231         new_val = dev_priv->irq_mask;
 232         new_val &= ~interrupt_mask;
 233         new_val |= (~enabled_irq_mask & interrupt_mask);
 234
 235         if (new_val != dev_priv->irq_mask) {
 236                 dev_priv->irq_mask = new_val;
 237                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 238                 POSTING_READ(DEIMR);
 239         }
 240 }
 241
 242 void
 243 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 244 {
 245         ilk_update_display_irq(dev_priv, mask, mask);
 246 }
 247
 248 void
 249 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 250 {
 251         ilk_update_display_irq(dev_priv, mask, 0);
 252 }
 253
 254 /**
 255  * ilk_update_gt_irq - update GTIMR
 256  * @dev_priv: driver private
 257  * @interrupt_mask: mask of interrupt bits to update
 258  * @enabled_irq_mask: mask of interrupt bits to enable
 259  */
 260 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 261                               uint32_t interrupt_mask,
 262                               uint32_t enabled_irq_mask)
 263 {
 264         assert_spin_locked(&dev_priv->irq_lock);
 265
 266         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 267
 268         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 269                 return;
 270
 271         dev_priv->gt_irq_mask &= ~interrupt_mask;
 272         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 273         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 274         POSTING_READ(GTIMR);
 275 }
 276
 277 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 278 {
 279         ilk_update_gt_irq(dev_priv, mask, mask);
 280 }
 281
 282 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 283 {
 284         ilk_update_gt_irq(dev_priv, mask, 0);
 285 }
 286
 287 static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
 288 {
 289         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 290 }
 291
 292 static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
 293 {
 294         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
 295 }
 296
 297 static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
 298 {
 299         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
 300 }
 301
 302 /**
 303   * snb_update_pm_irq - update GEN6_PMIMR
 304   * @dev_priv: driver private
 305   * @interrupt_mask: mask of interrupt bits to update
 306   * @enabled_irq_mask: mask of interrupt bits to enable
 307   */
 308 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 309                               uint32_t interrupt_mask,
 310                               uint32_t enabled_irq_mask)
 311 {
 312         uint32_t new_val;
 313
 314         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 315
 316         assert_spin_locked(&dev_priv->irq_lock);
 317
 318         new_val = dev_priv->pm_irq_mask;
 319         new_val &= ~interrupt_mask;
 320         new_val |= (~enabled_irq_mask & interrupt_mask);
 321
 322         if (new_val != dev_priv->pm_irq_mask) {
 323                 dev_priv->pm_irq_mask = new_val;
 324                 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
 325                 POSTING_READ(gen6_pm_imr(dev_priv));
 326         }
 327 }
 328
 329 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 330 {
 331         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 332                 return;
 333
 334         snb_update_pm_irq(dev_priv, mask, mask);
 335 }
 336
 337 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
 338                                   uint32_t mask)
 339 {
 340         snb_update_pm_irq(dev_priv, mask, 0);
 341 }
 342
 343 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 344 {
 345         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 346                 return;
 347
 348         __gen6_disable_pm_irq(dev_priv, mask);
 349 }
 350
 351 void gen6_reset_rps_interrupts(struct drm_device *dev)
 352 {
 353         struct drm_i915_private *dev_priv = dev->dev_private;
 354         i915_reg_t reg = gen6_pm_iir(dev_priv);
 355
 356         spin_lock_irq(&dev_priv->irq_lock);
 357         I915_WRITE(reg, dev_priv->pm_rps_events);
 358         I915_WRITE(reg, dev_priv->pm_rps_events);
 359         POSTING_READ(reg);
 360         dev_priv->rps.pm_iir = 0;
 361         spin_unlock_irq(&dev_priv->irq_lock);
 362 }
 363
 364 void gen6_enable_rps_interrupts(struct drm_device *dev)
 365 {
 366         struct drm_i915_private *dev_priv = dev->dev_private;
 367
 368         spin_lock_irq(&dev_priv->irq_lock);
 369
 370         WARN_ON(dev_priv->rps.pm_iir);
 371         WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
 372         dev_priv->rps.interrupts_enabled = true;
 373         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
 374                                 dev_priv->pm_rps_events);
 375         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 376
 377         spin_unlock_irq(&dev_priv->irq_lock);
 378 }
 379
 380 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 381 {
 382         /*
 383          * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
 384          * if GEN6_PM_UP_EI_EXPIRED is masked.
 385          *
 386          * TODO: verify if this can be reproduced on VLV,CHV.
 387          */
 388         if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
 389                 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
 390
 391         if (INTEL_INFO(dev_priv)->gen >= 8)
 392                 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
 393
 394         return mask;
 395 }
 396
 397 void gen6_disable_rps_interrupts(struct drm_device *dev)
 398 {
 399         struct drm_i915_private *dev_priv = dev->dev_private;
 400
 401         spin_lock_irq(&dev_priv->irq_lock);
 402         dev_priv->rps.interrupts_enabled = false;
 403         spin_unlock_irq(&dev_priv->irq_lock);
 404
 405         cancel_work_sync(&dev_priv->rps.work);
 406
 407         spin_lock_irq(&dev_priv->irq_lock);
 408
 409         I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 410
 411         __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 412         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
 413                                 ~dev_priv->pm_rps_events);
 414
 415         spin_unlock_irq(&dev_priv->irq_lock);
 416
 417         synchronize_irq(dev->irq);
 418 }
 419
 420 /**
 421   * bdw_update_port_irq - update DE port interrupt
 422   * @dev_priv: driver private
 423   * @interrupt_mask: mask of interrupt bits to update
 424   * @enabled_irq_mask: mask of interrupt bits to enable
 425   */
 426 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
 427                                 uint32_t interrupt_mask,
 428                                 uint32_t enabled_irq_mask)
 429 {
 430         uint32_t new_val;
 431         uint32_t old_val;
 432
 433         assert_spin_locked(&dev_priv->irq_lock);
 434
 435         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 436
 437         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 438                 return;
 439
 440         old_val = I915_READ(GEN8_DE_PORT_IMR);
 441
 442         new_val = old_val;
 443         new_val &= ~interrupt_mask;
 444         new_val |= (~enabled_irq_mask & interrupt_mask);
 445
 446         if (new_val != old_val) {
 447                 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
 448                 POSTING_READ(GEN8_DE_PORT_IMR);
 449         }
 450 }
 451
 452 /**
 453  * ibx_display_interrupt_update - update SDEIMR
 454  * @dev_priv: driver private
 455  * @interrupt_mask: mask of interrupt bits to update
 456  * @enabled_irq_mask: mask of interrupt bits to enable
 457  */
 458 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 459                                   uint32_t interrupt_mask,
 460                                   uint32_t enabled_irq_mask)
 461 {
 462         uint32_t sdeimr = I915_READ(SDEIMR);
 463         sdeimr &= ~interrupt_mask;
 464         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 465
 466         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 467
 468         assert_spin_locked(&dev_priv->irq_lock);
 469
 470         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 471                 return;
 472
 473         I915_WRITE(SDEIMR, sdeimr);
 474         POSTING_READ(SDEIMR);
 475 }
 476
 477 static void
 478 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 479                        u32 enable_mask, u32 status_mask)
 480 {
 481         i915_reg_t reg = PIPESTAT(pipe);
 482         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 483
 484         assert_spin_locked(&dev_priv->irq_lock);
 485         WARN_ON(!intel_irqs_enabled(dev_priv));
 486
 487         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 488                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 489                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 490                       pipe_name(pipe), enable_mask, status_mask))
 491                 return;
 492
 493         if ((pipestat & enable_mask) == enable_mask)
 494                 return;
 495
 496         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
 497
 498         /* Enable the interrupt, clear any pending status */
 499         pipestat |= enable_mask | status_mask;
 500         I915_WRITE(reg, pipestat);
 501         POSTING_READ(reg);
 502 }
 503
 504 static void
 505 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 506                         u32 enable_mask, u32 status_mask)
 507 {
 508         i915_reg_t reg = PIPESTAT(pipe);
 509         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 510
 511         assert_spin_locked(&dev_priv->irq_lock);
 512         WARN_ON(!intel_irqs_enabled(dev_priv));
 513
 514         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 515                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 516                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 517                       pipe_name(pipe), enable_mask, status_mask))
 518                 return;
 519
 520         if ((pipestat & enable_mask) == 0)
 521                 return;
 522
 523         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
 524
 525         pipestat &= ~enable_mask;
 526         I915_WRITE(reg, pipestat);
 527         POSTING_READ(reg);
 528 }
 529
 530 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
 531 {
 532         u32 enable_mask = status_mask << 16;
 533
 534         /*
 535          * On pipe A we don't support the PSR interrupt yet,
 536          * on pipe B and C the same bit MBZ.
 537          */
 538         if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
 539                 return 0;
 540         /*
 541          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 542          * A the same bit is for perf counters which we don't use either.
 543          */
 544         if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
 545                 return 0;
 546
 547         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 548                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 549                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 550         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 551                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 552         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 553                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 554
 555         return enable_mask;
 556 }
 557
 558 void
 559 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 560                      u32 status_mask)
 561 {
 562         u32 enable_mask;
 563
 564         if (IS_VALLEYVIEW(dev_priv->dev))
 565                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 566                                                            status_mask);
 567         else
 568                 enable_mask = status_mask << 16;
 569         __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 570 }
 571
 572 void
 573 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 574                       u32 status_mask)
 575 {
 576         u32 enable_mask;
 577
 578         if (IS_VALLEYVIEW(dev_priv->dev))
 579                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 580                                                            status_mask);
 581         else
 582                 enable_mask = status_mask << 16;
 583         __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 584 }
 585
 586 /**
 587  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 588  * @dev: drm device
 589  */
 590 static void i915_enable_asle_pipestat(struct drm_device *dev)
 591 {
 592         struct drm_i915_private *dev_priv = dev->dev_private;
 593
 594         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 595                 return;
 596
 597         spin_lock_irq(&dev_priv->irq_lock);
 598
 599         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 600         if (INTEL_INFO(dev)->gen >= 4)
 601                 i915_enable_pipestat(dev_priv, PIPE_A,
 602                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 603
 604         spin_unlock_irq(&dev_priv->irq_lock);
 605 }
 606
 607 /*
 608  * This timing diagram depicts the video signal in and
 609  * around the vertical blanking period.
 610  *
 611  * Assumptions about the fictitious mode used in this example:
 612  *  vblank_start >= 3
 613  *  vsync_start = vblank_start + 1
 614  *  vsync_end = vblank_start + 2
 615  *  vtotal = vblank_start + 3
 616  *
 617  *           start of vblank:
 618  *           latch double buffered registers
 619  *           increment frame counter (ctg+)
 620  *           generate start of vblank interrupt (gen4+)
 621  *           |
 622  *           |          frame start:
 623  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 624  *           |          may be shifted forward 1-3 extra lines via PIPECONF
 625  *           |          |
 626  *           |          |  start of vsync:
 627  *           |          |  generate vsync interrupt
 628  *           |          |  |
 629  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 630  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 631  * ----va---> <-----------------vb--------------------> <--------va-------------
 632  *       |          |       <----vs----->                     |
 633  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 634  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 635  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 636  *       |          |                                         |
 637  *       last visible pixel                                   first visible pixel
 638  *                  |                                         increment frame counter (gen3/4)
 639  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 640  *
 641  * x  = horizontal active
 642  * _  = horizontal blanking
 643  * hs = horizontal sync
 644  * va = vertical active
 645  * vb = vertical blanking
 646  * vs = vertical sync
 647  * vbs = vblank_start (number)
 648  *
 649  * Summary:
 650  * - most events happen at the start of horizontal sync
 651  * - frame start happens at the start of horizontal blank, 1-4 lines
 652  *   (depending on PIPECONF settings) after the start of vblank
 653  * - gen3/4 pixel and frame counter are synchronized with the start
 654  *   of horizontal active on the first line of vertical active
 655  */
 656
 657 static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 658 {
 659         /* Gen2 doesn't have a hardware frame counter */
 660         return 0;
 661 }
 662
 663 /* Called from drm generic code, passed a 'crtc', which
 664  * we use as a pipe index
 665  */
 666 static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 667 {
 668         struct drm_i915_private *dev_priv = dev->dev_private;
 669         i915_reg_t high_frame, low_frame;
 670         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 671         struct intel_crtc *intel_crtc =
 672                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 673         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 674
 675         htotal = mode->crtc_htotal;
 676         hsync_start = mode->crtc_hsync_start;
 677         vbl_start = mode->crtc_vblank_start;
 678         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 679                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 680
 681         /* Convert to pixel count */
 682         vbl_start *= htotal;
 683
 684         /* Start of vblank event occurs at start of hsync */
 685         vbl_start -= htotal - hsync_start;
 686
 687         high_frame = PIPEFRAME(pipe);
 688         low_frame = PIPEFRAMEPIXEL(pipe);
 689
 690         /*
 691          * High & low register fields aren't synchronized, so make sure
 692          * we get a low value that's stable across two reads of the high
 693          * register.
 694          */
 695         do {
 696                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 697                 low   = I915_READ(low_frame);
 698                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 699         } while (high1 != high2);
 700
 701         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 702         pixel = low & PIPE_PIXEL_MASK;
 703         low >>= PIPE_FRAME_LOW_SHIFT;
 704
 705         /*
 706          * The frame counter increments at beginning of active.
 707          * Cook up a vblank counter by also checking the pixel
 708          * counter against vblank start.
 709          */
 710         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
 711 }
 712
 713 static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 714 {
 715         struct drm_i915_private *dev_priv = dev->dev_private;
 716
 717         return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
 718 }
 719
 720 /* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
 721 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 722 {
 723         struct drm_device *dev = crtc->base.dev;
 724         struct drm_i915_private *dev_priv = dev->dev_private;
 725         const struct drm_display_mode *mode = &crtc->base.hwmode;
 726         enum pipe pipe = crtc->pipe;
 727         int position, vtotal;
 728
 729         vtotal = mode->crtc_vtotal;
 730         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 731                 vtotal /= 2;
 732
 733         if (IS_GEN2(dev))
 734                 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 735         else
 736                 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 737
 738         /*
 739          * On HSW, the DSL reg (0x70000) appears to return 0 if we
 740          * read it just before the start of vblank.  So try it again
 741          * so we don't accidentally end up spanning a vblank frame
 742          * increment, causing the pipe_update_end() code to squak at us.
 743          *
 744          * The nature of this problem means we can't simply check the ISR
 745          * bit and return the vblank start value; nor can we use the scanline
 746          * debug register in the transcoder as it appears to have the same
 747          * problem.  We may need to extend this to include other platforms,
 748          * but so far testing only shows the problem on HSW.
 749          */
 750         if (HAS_DDI(dev) && !position) {
 751                 int i, temp;
 752
 753                 for (i = 0; i < 100; i++) {
 754                         udelay(1);
 755                         temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
 756                                 DSL_LINEMASK_GEN3;
 757                         if (temp != position) {
 758                                 position = temp;
 759                                 break;
 760                         }
 761                 }
 762         }
 763
 764         /*
 765          * See update_scanline_offset() for the details on the
 766          * scanline_offset adjustment.
 767          */
 768         return (position + crtc->scanline_offset) % vtotal;
 769 }
 770
 771 static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 772                                     unsigned int flags, int *vpos, int *hpos,
 773                                     ktime_t *stime, ktime_t *etime,
 774                                     const struct drm_display_mode *mode)
 775 {
 776         struct drm_i915_private *dev_priv = dev->dev_private;
 777         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 778         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 779         int position;
 780         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 781         bool in_vbl = true;
 782         int ret = 0;
 783         unsigned long irqflags;
 784
 785         if (WARN_ON(!mode->crtc_clock)) {
 786                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 787                                  "pipe %c\n", pipe_name(pipe));
 788                 return 0;
 789         }
 790
 791         htotal = mode->crtc_htotal;
 792         hsync_start = mode->crtc_hsync_start;
 793         vtotal = mode->crtc_vtotal;
 794         vbl_start = mode->crtc_vblank_start;
 795         vbl_end = mode->crtc_vblank_end;
 796
 797         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 798                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 799                 vbl_end /= 2;
 800                 vtotal /= 2;
 801         }
 802
 803         ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 804
 805         /*
 806          * Lock uncore.lock, as we will do multiple timing critical raw
 807          * register reads, potentially with preemption disabled, so the
 808          * following code must not block on uncore.lock.
 809          */
 810         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 811
 812         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 813
 814         /* Get optional system timestamp before query. */
 815         if (stime)
 816                 *stime = ktime_get();
 817
 818         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 819                 /* No obvious pixelcount register. Only query vertical
 820                  * scanout position from Display scan line register.
 821                  */
 822                 position = __intel_get_crtc_scanline(intel_crtc);
 823         } else {
 824                 /* Have access to pixelcount since start of frame.
 825                  * We can split this into vertical and horizontal
 826                  * scanout position.
 827                  */
 828                 position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 829
 830                 /* convert to pixel counts */
 831                 vbl_start *= htotal;
 832                 vbl_end *= htotal;
 833                 vtotal *= htotal;
 834
 835                 /*
 836                  * In interlaced modes, the pixel counter counts all pixels,
 837                  * so one field will have htotal more pixels. In order to avoid
 838                  * the reported position from jumping backwards when the pixel
 839                  * counter is beyond the length of the shorter field, just
 840                  * clamp the position the length of the shorter field. This
 841                  * matches how the scanline counter based position works since
 842                  * the scanline counter doesn't count the two half lines.
 843                  */
 844                 if (position >= vtotal)
 845                         position = vtotal - 1;
 846
 847                 /*
 848                  * Start of vblank interrupt is triggered at start of hsync,
 849                  * just prior to the first active line of vblank. However we
 850                  * consider lines to start at the leading edge of horizontal
 851                  * active. So, should we get here before we've crossed into
 852                  * the horizontal active of the first line in vblank, we would
 853                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
 854                  * always add htotal-hsync_start to the current pixel position.
 855                  */
 856                 position = (position + htotal - hsync_start) % vtotal;
 857         }
 858
 859         /* Get optional system timestamp after query. */
 860         if (etime)
 861                 *etime = ktime_get();
 862
 863         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 864
 865         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 866
 867         in_vbl = position >= vbl_start && position < vbl_end;
 868
 869         /*
 870          * While in vblank, position will be negative
 871          * counting up towards 0 at vbl_end. And outside
 872          * vblank, position will be positive counting
 873          * up since vbl_end.
 874          */
 875         if (position >= vbl_start)
 876                 position -= vbl_end;
 877         else
 878                 position += vtotal - vbl_end;
 879
 880         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 881                 *vpos = position;
 882                 *hpos = 0;
 883         } else {
 884                 *vpos = position / htotal;
 885                 *hpos = position - (*vpos * htotal);
 886         }
 887
 888         /* In vblank? */
 889         if (in_vbl)
 890                 ret |= DRM_SCANOUTPOS_IN_VBLANK;
 891
 892         return ret;
 893 }
 894
 895 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 896 {
 897         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 898         unsigned long irqflags;
 899         int position;
 900
 901         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 902         position = __intel_get_crtc_scanline(crtc);
 903         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 904
 905         return position;
 906 }
 907
 908 static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
 909                               int *max_error,
 910                               struct timeval *vblank_time,
 911                               unsigned flags)
 912 {
 913         struct drm_crtc *crtc;
 914
 915         if (pipe >= INTEL_INFO(dev)->num_pipes) {
 916                 DRM_ERROR("Invalid crtc %u\n", pipe);
 917                 return -EINVAL;
 918         }
 919
 920         /* Get drm_crtc to timestamp: */
 921         crtc = intel_get_crtc_for_pipe(dev, pipe);
 922         if (crtc == NULL) {
 923                 DRM_ERROR("Invalid crtc %u\n", pipe);
 924                 return -EINVAL;
 925         }
 926
 927         if (!crtc->hwmode.crtc_clock) {
 928                 DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
 929                 return -EBUSY;
 930         }
 931
 932         /* Helper routine in DRM core does all the work: */
 933         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 934                                                      vblank_time, flags,
 935                                                      &crtc->hwmode);
 936 }
 937
 938 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 939 {
 940         struct drm_i915_private *dev_priv = dev->dev_private;
 941         u32 busy_up, busy_down, max_avg, min_avg;
 942         u8 new_delay;
 943
 944         spin_lock(&mchdev_lock);
 945
 946         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 947
 948         new_delay = dev_priv->ips.cur_delay;
 949
 950         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 951         busy_up = I915_READ(RCPREVBSYTUPAVG);
 952         busy_down = I915_READ(RCPREVBSYTDNAVG);
 953         max_avg = I915_READ(RCBMAXAVG);
 954         min_avg = I915_READ(RCBMINAVG);
 955
 956         /* Handle RCS change request from hw */
 957         if (busy_up > max_avg) {
 958                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 959                         new_delay = dev_priv->ips.cur_delay - 1;
 960                 if (new_delay < dev_priv->ips.max_delay)
 961                         new_delay = dev_priv->ips.max_delay;
 962         } else if (busy_down < min_avg) {
 963                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 964                         new_delay = dev_priv->ips.cur_delay + 1;
 965                 if (new_delay > dev_priv->ips.min_delay)
 966                         new_delay = dev_priv->ips.min_delay;
 967         }
 968
 969         if (ironlake_set_drps(dev, new_delay))
 970                 dev_priv->ips.cur_delay = new_delay;
 971
 972         spin_unlock(&mchdev_lock);
 973
 974         return;
 975 }
 976
 977 static void notify_ring(struct intel_engine_cs *ring)
 978 {
 979         if (!intel_ring_initialized(ring))
 980                 return;
 981
 982         trace_i915_gem_request_notify(ring);
 983
 984         wake_up_all(&ring->irq_queue);
 985 }
 986
 987 static void vlv_c0_read(struct drm_i915_private *dev_priv,
 988                         struct intel_rps_ei *ei)
 989 {
 990         ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
 991         ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
 992         ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
 993 }
 994
 995 static bool vlv_c0_above(struct drm_i915_private *dev_priv,
 996                          const struct intel_rps_ei *old,
 997                          const struct intel_rps_ei *now,
 998                          int threshold)
 999 {
1000         u64 time, c0;
1001         unsigned int mul = 100;
1002
1003         if (old->cz_clock == 0)
1004                 return false;
1005
1006         if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
1007                 mul <<= 8;
1008
1009         time = now->cz_clock - old->cz_clock;
1010         time *= threshold * dev_priv->czclk_freq;
1011
1012         /* Workload can be split between render + media, e.g. SwapBuffers
1013          * being blitted in X after being rendered in mesa. To account for
1014          * this we need to combine both engines into our activity counter.
1015          */
1016         c0 = now->render_c0 - old->render_c0;
1017         c0 += now->media_c0 - old->media_c0;
1018         c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1019
1020         return c0 >= time;
1021 }
1022
1023 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1024 {
1025         vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
1026         dev_priv->rps.up_ei = dev_priv->rps.down_ei;
1027 }
1028
1029 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
1030 {
1031         struct intel_rps_ei now;
1032         u32 events = 0;
1033
1034         if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1035                 return 0;
1036
1037         vlv_c0_read(dev_priv, &now);
1038         if (now.cz_clock == 0)
1039                 return 0;
1040
1041         if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
1042                 if (!vlv_c0_above(dev_priv,
1043                                   &dev_priv->rps.down_ei, &now,
1044                                   dev_priv->rps.down_threshold))
1045                         events |= GEN6_PM_RP_DOWN_THRESHOLD;
1046                 dev_priv->rps.down_ei = now;
1047         }
1048
1049         if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
1050                 if (vlv_c0_above(dev_priv,
1051                                  &dev_priv->rps.up_ei, &now,
1052                                  dev_priv->rps.up_threshold))
1053                         events |= GEN6_PM_RP_UP_THRESHOLD;
1054                 dev_priv->rps.up_ei = now;
1055         }
1056
1057         return events;
1058 }
1059
1060 static bool any_waiters(struct drm_i915_private *dev_priv)
1061 {
1062         struct intel_engine_cs *ring;
1063         int i;
1064
1065         for_each_ring(ring, dev_priv, i)
1066                 if (ring->irq_refcount)
1067                         return true;
1068
1069         return false;
1070 }
1071
1072 static void gen6_pm_rps_work(struct work_struct *work)
1073 {
1074         struct drm_i915_private *dev_priv =
1075                 container_of(work, struct drm_i915_private, rps.work);
1076         bool client_boost;
1077         int new_delay, adj, min, max;
1078         u32 pm_iir;
1079
1080         spin_lock_irq(&dev_priv->irq_lock);
1081         /* Speed up work cancelation during disabling rps interrupts. */
1082         if (!dev_priv->rps.interrupts_enabled) {
1083                 spin_unlock_irq(&dev_priv->irq_lock);
1084                 return;
1085         }
1086         pm_iir = dev_priv->rps.pm_iir;
1087         dev_priv->rps.pm_iir = 0;
1088         /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1089         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1090         client_boost = dev_priv->rps.client_boost;
1091         dev_priv->rps.client_boost = false;
1092         spin_unlock_irq(&dev_priv->irq_lock);
1093
1094         /* Make sure we didn't queue anything we're not going to process. */
1095         WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1096
1097         if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1098                 return;
1099
1100         mutex_lock(&dev_priv->rps.hw_lock);
1101
1102         pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
1103
1104         adj = dev_priv->rps.last_adj;
1105         new_delay = dev_priv->rps.cur_freq;
1106         min = dev_priv->rps.min_freq_softlimit;
1107         max = dev_priv->rps.max_freq_softlimit;
1108
1109         if (client_boost) {
1110                 new_delay = dev_priv->rps.max_freq_softlimit;
1111                 adj = 0;
1112         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1113                 if (adj > 0)
1114                         adj *= 2;
1115                 else /* CHV needs even encode values */
1116                         adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1117                 /*
1118                  * For better performance, jump directly
1119                  * to RPe if we're below it.
1120                  */
1121                 if (new_delay < dev_priv->rps.efficient_freq - adj) {
1122                         new_delay = dev_priv->rps.efficient_freq;
1123                         adj = 0;
1124                 }
1125         } else if (any_waiters(dev_priv)) {
1126                 adj = 0;
1127         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1128                 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1129                         new_delay = dev_priv->rps.efficient_freq;
1130                 else
1131                         new_delay = dev_priv->rps.min_freq_softlimit;
1132                 adj = 0;
1133         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1134                 if (adj < 0)
1135                         adj *= 2;
1136                 else /* CHV needs even encode values */
1137                         adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1138         } else { /* unknown event */
1139                 adj = 0;
1140         }
1141
1142         dev_priv->rps.last_adj = adj;
1143
1144         /* sysfs frequency interfaces may have snuck in while servicing the
1145          * interrupt
1146          */
1147         new_delay += adj;
1148         new_delay = clamp_t(int, new_delay, min, max);
1149
1150         intel_set_rps(dev_priv->dev, new_delay);
1151
1152         mutex_unlock(&dev_priv->rps.hw_lock);
1153 }
1154
1155
1156 /**
1157  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1158  * occurred.
1159  * @work: workqueue struct
1160  *
1161  * Doesn't actually do anything except notify userspace. As a consequence of
1162  * this event, userspace should try to remap the bad rows since statistically
1163  * it is likely the same row is more likely to go bad again.
1164  */
1165 static void ivybridge_parity_work(struct work_struct *work)
1166 {
1167         struct drm_i915_private *dev_priv =
1168                 container_of(work, struct drm_i915_private, l3_parity.error_work);
1169         u32 error_status, row, bank, subbank;
1170         char *parity_event[6];
1171         uint32_t misccpctl;
1172         uint8_t slice = 0;
1173
1174         /* We must turn off DOP level clock gating to access the L3 registers.
1175          * In order to prevent a get/put style interface, acquire struct mutex
1176          * any time we access those registers.
1177          */
1178         mutex_lock(&dev_priv->dev->struct_mutex);
1179
1180         /* If we've screwed up tracking, just let the interrupt fire again */
1181         if (WARN_ON(!dev_priv->l3_parity.which_slice))
1182                 goto out;
1183
1184         misccpctl = I915_READ(GEN7_MISCCPCTL);
1185         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1186         POSTING_READ(GEN7_MISCCPCTL);
1187
1188         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1189                 i915_reg_t reg;
1190
1191                 slice--;
1192                 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1193                         break;
1194
1195                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1196
1197                 reg = GEN7_L3CDERRST1(slice);
1198
1199                 error_status = I915_READ(reg);
1200                 row = GEN7_PARITY_ERROR_ROW(error_status);
1201                 bank = GEN7_PARITY_ERROR_BANK(error_status);
1202                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1203
1204                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1205                 POSTING_READ(reg);
1206
1207                 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1208                 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1209                 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1210                 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1211                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1212                 parity_event[5] = NULL;
1213
1214                 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1215                                    KOBJ_CHANGE, parity_event);
1216
1217                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1218                           slice, row, bank, subbank);
1219
1220                 kfree(parity_event[4]);
1221                 kfree(parity_event[3]);
1222                 kfree(parity_event[2]);
1223                 kfree(parity_event[1]);
1224         }
1225
1226         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1227
1228 out:
1229         WARN_ON(dev_priv->l3_parity.which_slice);
1230         spin_lock_irq(&dev_priv->irq_lock);
1231         gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1232         spin_unlock_irq(&dev_priv->irq_lock);
1233
1234         mutex_unlock(&dev_priv->dev->struct_mutex);
1235 }
1236
1237 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1238 {
1239         struct drm_i915_private *dev_priv = dev->dev_private;
1240
1241         if (!HAS_L3_DPF(dev))
1242                 return;
1243
1244         spin_lock(&dev_priv->irq_lock);
1245         gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1246         spin_unlock(&dev_priv->irq_lock);
1247
1248         iir &= GT_PARITY_ERROR(dev);
1249         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1250                 dev_priv->l3_parity.which_slice |= 1 << 1;
1251
1252         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1253                 dev_priv->l3_parity.which_slice |= 1 << 0;
1254
1255         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1256 }
1257
1258 static void ilk_gt_irq_handler(struct drm_device *dev,
1259                                struct drm_i915_private *dev_priv,
1260                                u32 gt_iir)
1261 {
1262         if (gt_iir &
1263             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1264                 notify_ring(&dev_priv->ring[RCS]);
1265         if (gt_iir & ILK_BSD_USER_INTERRUPT)
1266                 notify_ring(&dev_priv->ring[VCS]);
1267 }
1268
1269 static void snb_gt_irq_handler(struct drm_device *dev,
1270                                struct drm_i915_private *dev_priv,
1271                                u32 gt_iir)
1272 {
1273
1274         if (gt_iir &
1275             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1276                 notify_ring(&dev_priv->ring[RCS]);
1277         if (gt_iir & GT_BSD_USER_INTERRUPT)
1278                 notify_ring(&dev_priv->ring[VCS]);
1279         if (gt_iir & GT_BLT_USER_INTERRUPT)
1280                 notify_ring(&dev_priv->ring[BCS]);
1281
1282         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1283                       GT_BSD_CS_ERROR_INTERRUPT |
1284                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1285                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1286
1287         if (gt_iir & GT_PARITY_ERROR(dev))
1288                 ivybridge_parity_error_irq_handler(dev, gt_iir);
1289 }
1290
1291 static __always_inline void
1292 gen8_cs_irq_handler(struct intel_engine_cs *ring, u32 iir, int test_shift)
1293 {
1294         if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
1295                 notify_ring(ring);
1296         if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift))
1297                 intel_lrc_irq_handler(ring);
1298 }
1299
1300 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1301                                        u32 master_ctl)
1302 {
1303         irqreturn_t ret = IRQ_NONE;
1304
1305         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1306                 u32 iir = I915_READ_FW(GEN8_GT_IIR(0));
1307                 if (iir) {
1308                         I915_WRITE_FW(GEN8_GT_IIR(0), iir);
1309                         ret = IRQ_HANDLED;
1310
1311                         gen8_cs_irq_handler(&dev_priv->ring[RCS],
1312                                         iir, GEN8_RCS_IRQ_SHIFT);
1313
1314                         gen8_cs_irq_handler(&dev_priv->ring[BCS],
1315                                         iir, GEN8_BCS_IRQ_SHIFT);
1316                 } else
1317                         DRM_ERROR("The master control interrupt lied (GT0)!\n");
1318         }
1319
1320         if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1321                 u32 iir = I915_READ_FW(GEN8_GT_IIR(1));
1322                 if (iir) {
1323                         I915_WRITE_FW(GEN8_GT_IIR(1), iir);
1324                         ret = IRQ_HANDLED;
1325
1326                         gen8_cs_irq_handler(&dev_priv->ring[VCS],
1327                                         iir, GEN8_VCS1_IRQ_SHIFT);
1328
1329                         gen8_cs_irq_handler(&dev_priv->ring[VCS2],
1330                                         iir, GEN8_VCS2_IRQ_SHIFT);
1331                 } else
1332                         DRM_ERROR("The master control interrupt lied (GT1)!\n");
1333         }
1334
1335         if (master_ctl & GEN8_GT_VECS_IRQ) {
1336                 u32 iir = I915_READ_FW(GEN8_GT_IIR(3));
1337                 if (iir) {
1338                         I915_WRITE_FW(GEN8_GT_IIR(3), iir);
1339                         ret = IRQ_HANDLED;
1340
1341                         gen8_cs_irq_handler(&dev_priv->ring[VECS],
1342                                         iir, GEN8_VECS_IRQ_SHIFT);
1343                 } else
1344                         DRM_ERROR("The master control interrupt lied (GT3)!\n");
1345         }
1346
1347         if (master_ctl & GEN8_GT_PM_IRQ) {
1348                 u32 iir = I915_READ_FW(GEN8_GT_IIR(2));
1349                 if (iir & dev_priv->pm_rps_events) {
1350                         I915_WRITE_FW(GEN8_GT_IIR(2),
1351                                       iir & dev_priv->pm_rps_events);
1352                         ret = IRQ_HANDLED;
1353                         gen6_rps_irq_handler(dev_priv, iir);
1354                 } else
1355                         DRM_ERROR("The master control interrupt lied (PM)!\n");
1356         }
1357
1358         return ret;
1359 }
1360
1361 static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
1362 {
1363         switch (port) {
1364         case PORT_A:
1365                 return val & PORTA_HOTPLUG_LONG_DETECT;
1366         case PORT_B:
1367                 return val & PORTB_HOTPLUG_LONG_DETECT;
1368         case PORT_C:
1369                 return val & PORTC_HOTPLUG_LONG_DETECT;
1370         default:
1371                 return false;
1372         }
1373 }
1374
1375 static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
1376 {
1377         switch (port) {
1378         case PORT_E:
1379                 return val & PORTE_HOTPLUG_LONG_DETECT;
1380         default:
1381                 return false;
1382         }
1383 }
1384
1385 static bool spt_port_hotplug_long_detect(enum port port, u32 val)
1386 {
1387         switch (port) {
1388         case PORT_A:
1389                 return val & PORTA_HOTPLUG_LONG_DETECT;
1390         case PORT_B:
1391                 return val & PORTB_HOTPLUG_LONG_DETECT;
1392         case PORT_C:
1393                 return val & PORTC_HOTPLUG_LONG_DETECT;
1394         case PORT_D:
1395                 return val & PORTD_HOTPLUG_LONG_DETECT;
1396         default:
1397                 return false;
1398         }
1399 }
1400
1401 static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
1402 {
1403         switch (port) {
1404         case PORT_A:
1405                 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1406         default:
1407                 return false;
1408         }
1409 }
1410
1411 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1412 {
1413         switch (port) {
1414         case PORT_B:
1415                 return val & PORTB_HOTPLUG_LONG_DETECT;
1416         case PORT_C:
1417                 return val & PORTC_HOTPLUG_LONG_DETECT;
1418         case PORT_D:
1419                 return val & PORTD_HOTPLUG_LONG_DETECT;
1420         default:
1421                 return false;
1422         }
1423 }
1424
1425 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1426 {
1427         switch (port) {
1428         case PORT_B:
1429                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1430         case PORT_C:
1431                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1432         case PORT_D:
1433                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1434         default:
1435                 return false;
1436         }
1437 }
1438
1439 /*
1440  * Get a bit mask of pins that have triggered, and which ones may be long.
1441  * This can be called multiple times with the same masks to accumulate
1442  * hotplug detection results from several registers.
1443  *
1444  * Note that the caller is expected to zero out the masks initially.
1445  */
1446 static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1447                              u32 hotplug_trigger, u32 dig_hotplug_reg,
1448                              const u32 hpd[HPD_NUM_PINS],
1449                              bool long_pulse_detect(enum port port, u32 val))
1450 {
1451         enum port port;
1452         int i;
1453
1454         for_each_hpd_pin(i) {
1455                 if ((hpd[i] & hotplug_trigger) == 0)
1456                         continue;
1457
1458                 *pin_mask |= BIT(i);
1459
1460                 if (!intel_hpd_pin_to_port(i, &port))
1461                         continue;
1462
1463                 if (long_pulse_detect(port, dig_hotplug_reg))
1464                         *long_mask |= BIT(i);
1465         }
1466
1467         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1468                          hotplug_trigger, dig_hotplug_reg, *pin_mask);
1469
1470 }
1471
1472 static void gmbus_irq_handler(struct drm_device *dev)
1473 {
1474         struct drm_i915_private *dev_priv = dev->dev_private;
1475
1476         wake_up_all(&dev_priv->gmbus_wait_queue);
1477 }
1478
1479 static void dp_aux_irq_handler(struct drm_device *dev)
1480 {
1481         struct drm_i915_private *dev_priv = dev->dev_private;
1482
1483         wake_up_all(&dev_priv->gmbus_wait_queue);
1484 }
1485
1486 #if defined(CONFIG_DEBUG_FS)
1487 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1488                                          uint32_t crc0, uint32_t crc1,
1489                                          uint32_t crc2, uint32_t crc3,
1490                                          uint32_t crc4)
1491 {
1492         struct drm_i915_private *dev_priv = dev->dev_private;
1493         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1494         struct intel_pipe_crc_entry *entry;
1495         int head, tail;
1496
1497         spin_lock(&pipe_crc->lock);
1498
1499         if (!pipe_crc->entries) {
1500                 spin_unlock(&pipe_crc->lock);
1501                 DRM_DEBUG_KMS("spurious interrupt\n");
1502                 return;
1503         }
1504
1505         head = pipe_crc->head;
1506         tail = pipe_crc->tail;
1507
1508         if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1509                 spin_unlock(&pipe_crc->lock);
1510                 DRM_ERROR("CRC buffer overflowing\n");
1511                 return;
1512         }
1513
1514         entry = &pipe_crc->entries[head];
1515
1516         entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1517         entry->crc[0] = crc0;
1518         entry->crc[1] = crc1;
1519         entry->crc[2] = crc2;
1520         entry->crc[3] = crc3;
1521         entry->crc[4] = crc4;
1522
1523         head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1524         pipe_crc->head = head;
1525
1526         spin_unlock(&pipe_crc->lock);
1527
1528         wake_up_interruptible(&pipe_crc->wq);
1529 }
1530 #else
1531 static inline void
1532 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1533                              uint32_t crc0, uint32_t crc1,
1534                              uint32_t crc2, uint32_t crc3,
1535                              uint32_t crc4) {}
1536 #endif
1537
1538
1539 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1540 {
1541         struct drm_i915_private *dev_priv = dev->dev_private;
1542
1543         display_pipe_crc_irq_handler(dev, pipe,
1544                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1545                                      0, 0, 0, 0);
1546 }
1547
1548 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1549 {
1550         struct drm_i915_private *dev_priv = dev->dev_private;
1551
1552         display_pipe_crc_irq_handler(dev, pipe,
1553                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1554                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1555                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1556                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1557                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1558 }
1559
1560 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1561 {
1562         struct drm_i915_private *dev_priv = dev->dev_private;
1563         uint32_t res1, res2;
1564
1565         if (INTEL_INFO(dev)->gen >= 3)
1566                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1567         else
1568                 res1 = 0;
1569
1570         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1571                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1572         else
1573                 res2 = 0;
1574
1575         display_pipe_crc_irq_handler(dev, pipe,
1576                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
1577                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1578                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1579                                      res1, res2);
1580 }
1581
1582 /* The RPS events need forcewake, so we add them to a work queue and mask their
1583  * IMR bits until the work is done. Other interrupts can be processed without
1584  * the work queue. */
1585 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1586 {
1587         if (pm_iir & dev_priv->pm_rps_events) {
1588                 spin_lock(&dev_priv->irq_lock);
1589                 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1590                 if (dev_priv->rps.interrupts_enabled) {
1591                         dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1592                         queue_work(dev_priv->wq, &dev_priv->rps.work);
1593                 }
1594                 spin_unlock(&dev_priv->irq_lock);
1595         }
1596
1597         if (INTEL_INFO(dev_priv)->gen >= 8)
1598                 return;
1599
1600         if (HAS_VEBOX(dev_priv->dev)) {
1601                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1602                         notify_ring(&dev_priv->ring[VECS]);
1603
1604                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1605                         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1606         }
1607 }
1608
1609 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1610 {
1611         if (!drm_handle_vblank(dev, pipe))
1612                 return false;
1613
1614         return true;
1615 }
1616
1617 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1618 {
1619         struct drm_i915_private *dev_priv = dev->dev_private;
1620         u32 pipe_stats[I915_MAX_PIPES] = { };
1621         int pipe;
1622
1623         spin_lock(&dev_priv->irq_lock);
1624         for_each_pipe(dev_priv, pipe) {
1625                 i915_reg_t reg;
1626                 u32 mask, iir_bit = 0;
1627
1628                 /*
1629                  * PIPESTAT bits get signalled even when the interrupt is
1630                  * disabled with the mask bits, and some of the status bits do
1631                  * not generate interrupts at all (like the underrun bit). Hence
1632                  * we need to be careful that we only handle what we want to
1633                  * handle.
1634                  */
1635
1636                 /* fifo underruns are filterered in the underrun handler. */
1637                 mask = PIPE_FIFO_UNDERRUN_STATUS;
1638
1639                 switch (pipe) {
1640                 case PIPE_A:
1641                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1642                         break;
1643                 case PIPE_B:
1644                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1645                         break;
1646                 case PIPE_C:
1647                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1648                         break;
1649                 }
1650                 if (iir & iir_bit)
1651                         mask |= dev_priv->pipestat_irq_mask[pipe];
1652
1653                 if (!mask)
1654                         continue;
1655
1656                 reg = PIPESTAT(pipe);
1657                 mask |= PIPESTAT_INT_ENABLE_MASK;
1658                 pipe_stats[pipe] = I915_READ(reg) & mask;
1659
1660                 /*
1661                  * Clear the PIPE*STAT regs before the IIR
1662                  */
1663                 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1664                                         PIPESTAT_INT_STATUS_MASK))
1665                         I915_WRITE(reg, pipe_stats[pipe]);
1666         }
1667         spin_unlock(&dev_priv->irq_lock);
1668
1669         for_each_pipe(dev_priv, pipe) {
1670                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1671                     intel_pipe_handle_vblank(dev, pipe))
1672                         intel_check_page_flip(dev, pipe);
1673
1674                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1675                         intel_prepare_page_flip(dev, pipe);
1676                         intel_finish_page_flip(dev, pipe);
1677                 }
1678
1679                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1680                         i9xx_pipe_crc_irq_handler(dev, pipe);
1681
1682                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1683                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1684         }
1685
1686         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1687                 gmbus_irq_handler(dev);
1688 }
1689
1690 static void i9xx_hpd_irq_handler(struct drm_device *dev)
1691 {
1692         struct drm_i915_private *dev_priv = dev->dev_private;
1693         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1694         u32 pin_mask = 0, long_mask = 0;
1695
1696         if (!hotplug_status)
1697                 return;
1698
1699         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1700         /*
1701          * Make sure hotplug status is cleared before we clear IIR, or else we
1702          * may miss hotplug events.
1703          */
1704         POSTING_READ(PORT_HOTPLUG_STAT);
1705
1706         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1707                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1708
1709                 if (hotplug_trigger) {
1710                         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1711                                            hotplug_trigger, hpd_status_g4x,
1712                                            i9xx_port_hotplug_long_detect);
1713
1714                         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1715                 }
1716
1717                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1718                         dp_aux_irq_handler(dev);
1719         } else {
1720                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1721
1722                 if (hotplug_trigger) {
1723                         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1724                                            hotplug_trigger, hpd_status_i915,
1725                                            i9xx_port_hotplug_long_detect);
1726                         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1727                 }
1728         }
1729 }
1730
1731 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1732 {
1733         struct drm_device *dev = arg;
1734         struct drm_i915_private *dev_priv = dev->dev_private;
1735         u32 iir, gt_iir, pm_iir;
1736         irqreturn_t ret = IRQ_NONE;
1737
1738         if (!intel_irqs_enabled(dev_priv))
1739                 return IRQ_NONE;
1740
1741         while (true) {
1742                 /* Find, clear, then process each source of interrupt */
1743
1744                 gt_iir = I915_READ(GTIIR);
1745                 if (gt_iir)
1746                         I915_WRITE(GTIIR, gt_iir);
1747
1748                 pm_iir = I915_READ(GEN6_PMIIR);
1749                 if (pm_iir)
1750                         I915_WRITE(GEN6_PMIIR, pm_iir);
1751
1752                 iir = I915_READ(VLV_IIR);
1753                 if (iir) {
1754                         /* Consume port before clearing IIR or we'll miss events */
1755                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1756                                 i9xx_hpd_irq_handler(dev);
1757                         I915_WRITE(VLV_IIR, iir);
1758                 }
1759
1760                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1761                         goto out;
1762
1763                 ret = IRQ_HANDLED;
1764
1765                 if (gt_iir)
1766                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1767                 if (pm_iir)
1768                         gen6_rps_irq_handler(dev_priv, pm_iir);
1769                 /* Call regardless, as some status bits might not be
1770                  * signalled in iir */
1771                 valleyview_pipestat_irq_handler(dev, iir);
1772         }
1773
1774 out:
1775         return ret;
1776 }
1777
1778 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1779 {
1780         struct drm_device *dev = arg;
1781         struct drm_i915_private *dev_priv = dev->dev_private;
1782         u32 master_ctl, iir;
1783         irqreturn_t ret = IRQ_NONE;
1784
1785         if (!intel_irqs_enabled(dev_priv))
1786                 return IRQ_NONE;
1787
1788         for (;;) {
1789                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1790                 iir = I915_READ(VLV_IIR);
1791
1792                 if (master_ctl == 0 && iir == 0)
1793                         break;
1794
1795                 ret = IRQ_HANDLED;
1796
1797                 I915_WRITE(GEN8_MASTER_IRQ, 0);
1798
1799                 /* Find, clear, then process each source of interrupt */
1800
1801                 if (iir) {
1802                         /* Consume port before clearing IIR or we'll miss events */
1803                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1804                                 i9xx_hpd_irq_handler(dev);
1805                         I915_WRITE(VLV_IIR, iir);
1806                 }
1807
1808                 gen8_gt_irq_handler(dev_priv, master_ctl);
1809
1810                 /* Call regardless, as some status bits might not be
1811                  * signalled in iir */
1812                 valleyview_pipestat_irq_handler(dev, iir);
1813
1814                 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1815                 POSTING_READ(GEN8_MASTER_IRQ);
1816         }
1817
1818         return ret;
1819 }
1820
1821 static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
1822                                 const u32 hpd[HPD_NUM_PINS])
1823 {
1824         struct drm_i915_private *dev_priv = to_i915(dev);
1825         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1826
1827         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1828         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1829
1830         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1831                            dig_hotplug_reg, hpd,
1832                            pch_port_hotplug_long_detect);
1833
1834         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1835 }
1836
1837 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1838 {
1839         struct drm_i915_private *dev_priv = dev->dev_private;
1840         int pipe;
1841         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1842
1843         if (hotplug_trigger)
1844                 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1845
1846         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1847                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1848                                SDE_AUDIO_POWER_SHIFT);
1849                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1850                                  port_name(port));
1851         }
1852
1853         if (pch_iir & SDE_AUX_MASK)
1854                 dp_aux_irq_handler(dev);
1855
1856         if (pch_iir & SDE_GMBUS)
1857                 gmbus_irq_handler(dev);
1858
1859         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1860                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1861
1862         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1863                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1864
1865         if (pch_iir & SDE_POISON)
1866                 DRM_ERROR("PCH poison interrupt\n");
1867
1868         if (pch_iir & SDE_FDI_MASK)
1869                 for_each_pipe(dev_priv, pipe)
1870                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1871                                          pipe_name(pipe),
1872                                          I915_READ(FDI_RX_IIR(pipe)));
1873
1874         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1875                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1876
1877         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1878                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1879
1880         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1881                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1882
1883         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1884                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1885 }
1886
1887 static void ivb_err_int_handler(struct drm_device *dev)
1888 {
1889         struct drm_i915_private *dev_priv = dev->dev_private;
1890         u32 err_int = I915_READ(GEN7_ERR_INT);
1891         enum pipe pipe;
1892
1893         if (err_int & ERR_INT_POISON)
1894                 DRM_ERROR("Poison interrupt\n");
1895
1896         for_each_pipe(dev_priv, pipe) {
1897                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1898                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1899
1900                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1901                         if (IS_IVYBRIDGE(dev))
1902                                 ivb_pipe_crc_irq_handler(dev, pipe);
1903                         else
1904                                 hsw_pipe_crc_irq_handler(dev, pipe);
1905                 }
1906         }
1907
1908         I915_WRITE(GEN7_ERR_INT, err_int);
1909 }
1910
1911 static void cpt_serr_int_handler(struct drm_device *dev)
1912 {
1913         struct drm_i915_private *dev_priv = dev->dev_private;
1914         u32 serr_int = I915_READ(SERR_INT);
1915
1916         if (serr_int & SERR_INT_POISON)
1917                 DRM_ERROR("PCH poison interrupt\n");
1918
1919         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1920                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1921
1922         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1923                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1924
1925         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1926                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1927
1928         I915_WRITE(SERR_INT, serr_int);
1929 }
1930
1931 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1932 {
1933         struct drm_i915_private *dev_priv = dev->dev_private;
1934         int pipe;
1935         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1936
1937         if (hotplug_trigger)
1938                 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1939
1940         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1941                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1942                                SDE_AUDIO_POWER_SHIFT_CPT);
1943                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1944                                  port_name(port));
1945         }
1946
1947         if (pch_iir & SDE_AUX_MASK_CPT)
1948                 dp_aux_irq_handler(dev);
1949
1950         if (pch_iir & SDE_GMBUS_CPT)
1951                 gmbus_irq_handler(dev);
1952
1953         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1954                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1955
1956         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1957                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1958
1959         if (pch_iir & SDE_FDI_MASK_CPT)
1960                 for_each_pipe(dev_priv, pipe)
1961                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1962                                          pipe_name(pipe),
1963                                          I915_READ(FDI_RX_IIR(pipe)));
1964
1965         if (pch_iir & SDE_ERROR_CPT)
1966                 cpt_serr_int_handler(dev);
1967 }
1968
1969 static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
1970 {
1971         struct drm_i915_private *dev_priv = dev->dev_private;
1972         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1973                 ~SDE_PORTE_HOTPLUG_SPT;
1974         u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1975         u32 pin_mask = 0, long_mask = 0;
1976
1977         if (hotplug_trigger) {
1978                 u32 dig_hotplug_reg;
1979
1980                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1981                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1982
1983                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1984                                    dig_hotplug_reg, hpd_spt,
1985                                    spt_port_hotplug_long_detect);
1986         }
1987
1988         if (hotplug2_trigger) {
1989                 u32 dig_hotplug_reg;
1990
1991                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1992                 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1993
1994                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
1995                                    dig_hotplug_reg, hpd_spt,
1996                                    spt_port_hotplug2_long_detect);
1997         }
1998
1999         if (pin_mask)
2000                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
2001
2002         if (pch_iir & SDE_GMBUS_CPT)
2003                 gmbus_irq_handler(dev);
2004 }
2005
2006 static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
2007                                 const u32 hpd[HPD_NUM_PINS])
2008 {
2009         struct drm_i915_private *dev_priv = to_i915(dev);
2010         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2011
2012         dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2013         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2014
2015         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2016                            dig_hotplug_reg, hpd,
2017                            ilk_port_hotplug_long_detect);
2018
2019         intel_hpd_irq_handler(dev, pin_mask, long_mask);
2020 }
2021
2022 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
2023 {
2024         struct drm_i915_private *dev_priv = dev->dev_private;
2025         enum pipe pipe;
2026         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2027
2028         if (hotplug_trigger)
2029                 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
2030
2031         if (de_iir & DE_AUX_CHANNEL_A)
2032                 dp_aux_irq_handler(dev);
2033
2034         if (de_iir & DE_GSE)
2035                 intel_opregion_asle_intr(dev);
2036
2037         if (de_iir & DE_POISON)
2038                 DRM_ERROR("Poison interrupt\n");
2039
2040         for_each_pipe(dev_priv, pipe) {
2041                 if (de_iir & DE_PIPE_VBLANK(pipe) &&
2042                     intel_pipe_handle_vblank(dev, pipe))
2043                         intel_check_page_flip(dev, pipe);
2044
2045                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2046                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2047
2048                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
2049                         i9xx_pipe_crc_irq_handler(dev, pipe);
2050
2051                 /* plane/pipes map 1:1 on ilk+ */
2052                 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
2053                         intel_prepare_page_flip(dev, pipe);
2054                         intel_finish_page_flip_plane(dev, pipe);
2055                 }
2056         }
2057
2058         /* check event from PCH */
2059         if (de_iir & DE_PCH_EVENT) {
2060                 u32 pch_iir = I915_READ(SDEIIR);
2061
2062                 if (HAS_PCH_CPT(dev))
2063                         cpt_irq_handler(dev, pch_iir);
2064                 else
2065                         ibx_irq_handler(dev, pch_iir);
2066
2067                 /* should clear PCH hotplug event before clear CPU irq */
2068                 I915_WRITE(SDEIIR, pch_iir);
2069         }
2070
2071         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
2072                 ironlake_rps_change_irq_handler(dev);
2073 }
2074
2075 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
2076 {
2077         struct drm_i915_private *dev_priv = dev->dev_private;
2078         enum pipe pipe;
2079         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2080
2081         if (hotplug_trigger)
2082                 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2083
2084         if (de_iir & DE_ERR_INT_IVB)
2085                 ivb_err_int_handler(dev);
2086
2087         if (de_iir & DE_AUX_CHANNEL_A_IVB)
2088                 dp_aux_irq_handler(dev);
2089
2090         if (de_iir & DE_GSE_IVB)
2091                 intel_opregion_asle_intr(dev);
2092
2093         for_each_pipe(dev_priv, pipe) {
2094                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
2095                     intel_pipe_handle_vblank(dev, pipe))
2096                         intel_check_page_flip(dev, pipe);
2097
2098                 /* plane/pipes map 1:1 on ilk+ */
2099                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
2100                         intel_prepare_page_flip(dev, pipe);
2101                         intel_finish_page_flip_plane(dev, pipe);
2102                 }
2103         }
2104
2105         /* check event from PCH */
2106         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
2107                 u32 pch_iir = I915_READ(SDEIIR);
2108
2109                 cpt_irq_handler(dev, pch_iir);
2110
2111                 /* clear PCH hotplug event before clear CPU irq */
2112                 I915_WRITE(SDEIIR, pch_iir);
2113         }
2114 }
2115
2116 /*
2117  * To handle irqs with the minimum potential races with fresh interrupts, we:
2118  * 1 - Disable Master Interrupt Control.
2119  * 2 - Find the source(s) of the interrupt.
2120  * 3 - Clear the Interrupt Identity bits (IIR).
2121  * 4 - Process the interrupt(s) that had bits set in the IIRs.
2122  * 5 - Re-enable Master Interrupt Control.
2123  */
2124 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2125 {
2126         struct drm_device *dev = arg;
2127         struct drm_i915_private *dev_priv = dev->dev_private;
2128         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2129         irqreturn_t ret = IRQ_NONE;
2130
2131         if (!intel_irqs_enabled(dev_priv))
2132                 return IRQ_NONE;
2133
2134         /* We get interrupts on unclaimed registers, so check for this before we
2135          * do any I915_{READ,WRITE}. */
2136         intel_uncore_check_errors(dev);
2137
2138         /* disable master interrupt before clearing iir  */
2139         de_ier = I915_READ(DEIER);
2140         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2141         POSTING_READ(DEIER);
2142
2143         /* Disable south interrupts. We'll only write to SDEIIR once, so further
2144          * interrupts will will be stored on its back queue, and then we'll be
2145          * able to process them after we restore SDEIER (as soon as we restore
2146          * it, we'll get an interrupt if SDEIIR still has something to process
2147          * due to its back queue). */
2148         if (!HAS_PCH_NOP(dev)) {
2149                 sde_ier = I915_READ(SDEIER);
2150                 I915_WRITE(SDEIER, 0);
2151                 POSTING_READ(SDEIER);
2152         }
2153
2154         /* Find, clear, then process each source of interrupt */
2155
2156         gt_iir = I915_READ(GTIIR);
2157         if (gt_iir) {
2158                 I915_WRITE(GTIIR, gt_iir);
2159                 ret = IRQ_HANDLED;
2160                 if (INTEL_INFO(dev)->gen >= 6)
2161                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
2162                 else
2163                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2164         }
2165
2166         de_iir = I915_READ(DEIIR);
2167         if (de_iir) {
2168                 I915_WRITE(DEIIR, de_iir);
2169                 ret = IRQ_HANDLED;
2170                 if (INTEL_INFO(dev)->gen >= 7)
2171                         ivb_display_irq_handler(dev, de_iir);
2172                 else
2173                         ilk_display_irq_handler(dev, de_iir);
2174         }
2175
2176         if (INTEL_INFO(dev)->gen >= 6) {
2177                 u32 pm_iir = I915_READ(GEN6_PMIIR);
2178                 if (pm_iir) {
2179                         I915_WRITE(GEN6_PMIIR, pm_iir);
2180                         ret = IRQ_HANDLED;
2181                         gen6_rps_irq_handler(dev_priv, pm_iir);
2182                 }
2183         }
2184
2185         I915_WRITE(DEIER, de_ier);
2186         POSTING_READ(DEIER);
2187         if (!HAS_PCH_NOP(dev)) {
2188                 I915_WRITE(SDEIER, sde_ier);
2189                 POSTING_READ(SDEIER);
2190         }
2191
2192         return ret;
2193 }
2194
2195 static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
2196                                 const u32 hpd[HPD_NUM_PINS])
2197 {
2198         struct drm_i915_private *dev_priv = to_i915(dev);
2199         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2200
2201         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2202         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2203
2204         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2205                            dig_hotplug_reg, hpd,
2206                            bxt_port_hotplug_long_detect);
2207
2208         intel_hpd_irq_handler(dev, pin_mask, long_mask);
2209 }
2210
2211 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2212 {
2213         struct drm_device *dev = arg;
2214         struct drm_i915_private *dev_priv = dev->dev_private;
2215         u32 master_ctl;
2216         irqreturn_t ret = IRQ_NONE;
2217         uint32_t tmp = 0;
2218         enum pipe pipe;
2219         u32 aux_mask = GEN8_AUX_CHANNEL_A;
2220
2221         if (!intel_irqs_enabled(dev_priv))
2222                 return IRQ_NONE;
2223
2224         if (INTEL_INFO(dev_priv)->gen >= 9)
2225                 aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2226                         GEN9_AUX_CHANNEL_D;
2227
2228         master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2229         master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2230         if (!master_ctl)
2231                 return IRQ_NONE;
2232
2233         I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2234
2235         /* Find, clear, then process each source of interrupt */
2236
2237         ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2238
2239         if (master_ctl & GEN8_DE_MISC_IRQ) {
2240                 tmp = I915_READ(GEN8_DE_MISC_IIR);
2241                 if (tmp) {
2242                         I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2243                         ret = IRQ_HANDLED;
2244                         if (tmp & GEN8_DE_MISC_GSE)
2245                                 intel_opregion_asle_intr(dev);
2246                         else
2247                                 DRM_ERROR("Unexpected DE Misc interrupt\n");
2248                 }
2249                 else
2250                         DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2251         }
2252
2253         if (master_ctl & GEN8_DE_PORT_IRQ) {
2254                 tmp = I915_READ(GEN8_DE_PORT_IIR);
2255                 if (tmp) {
2256                         bool found = false;
2257                         u32 hotplug_trigger = 0;
2258
2259                         if (IS_BROXTON(dev_priv))
2260                                 hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
2261                         else if (IS_BROADWELL(dev_priv))
2262                                 hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
2263
2264                         I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2265                         ret = IRQ_HANDLED;
2266
2267                         if (tmp & aux_mask) {
2268                                 dp_aux_irq_handler(dev);
2269                                 found = true;
2270                         }
2271
2272                         if (hotplug_trigger) {
2273                                 if (IS_BROXTON(dev))
2274                                         bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
2275                                 else
2276                                         ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
2277                                 found = true;
2278                         }
2279
2280                         if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2281                                 gmbus_irq_handler(dev);
2282                                 found = true;
2283                         }
2284
2285                         if (!found)
2286                                 DRM_ERROR("Unexpected DE Port interrupt\n");
2287                 }
2288                 else
2289                         DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2290         }
2291
2292         for_each_pipe(dev_priv, pipe) {
2293                 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2294
2295                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2296                         continue;
2297
2298                 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2299                 if (pipe_iir) {
2300                         ret = IRQ_HANDLED;
2301                         I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2302
2303                         if (pipe_iir & GEN8_PIPE_VBLANK &&
2304                             intel_pipe_handle_vblank(dev, pipe))
2305                                 intel_check_page_flip(dev, pipe);
2306
2307                         if (INTEL_INFO(dev_priv)->gen >= 9)
2308                                 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2309                         else
2310                                 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2311
2312                         if (flip_done) {
2313                                 intel_prepare_page_flip(dev, pipe);
2314                                 intel_finish_page_flip_plane(dev, pipe);
2315                         }
2316
2317                         if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2318                                 hsw_pipe_crc_irq_handler(dev, pipe);
2319
2320                         if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2321                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2322                                                                     pipe);
2323
2324
2325                         if (INTEL_INFO(dev_priv)->gen >= 9)
2326                                 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2327                         else
2328                                 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2329
2330                         if (fault_errors)
2331                                 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2332                                           pipe_name(pipe),
2333                                           pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2334                 } else
2335                         DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2336         }
2337
2338         if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2339             master_ctl & GEN8_DE_PCH_IRQ) {
2340                 /*
2341                  * FIXME(BDW): Assume for now that the new interrupt handling
2342                  * scheme also closed the SDE interrupt handling race we've seen
2343                  * on older pch-split platforms. But this needs testing.
2344                  */
2345                 u32 pch_iir = I915_READ(SDEIIR);
2346                 if (pch_iir) {
2347                         I915_WRITE(SDEIIR, pch_iir);
2348                         ret = IRQ_HANDLED;
2349
2350                         if (HAS_PCH_SPT(dev_priv))
2351                                 spt_irq_handler(dev, pch_iir);
2352                         else
2353                                 cpt_irq_handler(dev, pch_iir);
2354                 } else {
2355                         /*
2356                          * Like on previous PCH there seems to be something
2357                          * fishy going on with forwarding PCH interrupts.
2358                          */
2359                         DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
2360                 }
2361         }
2362
2363         I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2364         POSTING_READ_FW(GEN8_MASTER_IRQ);
2365
2366         return ret;
2367 }
2368
2369 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2370                                bool reset_completed)
2371 {
2372         struct intel_engine_cs *ring;
2373         int i;
2374
2375         /*
2376          * Notify all waiters for GPU completion events that reset state has
2377          * been changed, and that they need to restart their wait after
2378          * checking for potential errors (and bail out to drop locks if there is
2379          * a gpu reset pending so that i915_error_work_func can acquire them).
2380          */
2381
2382         /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2383         for_each_ring(ring, dev_priv, i)
2384                 wake_up_all(&ring->irq_queue);
2385
2386         /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2387         wake_up_all(&dev_priv->pending_flip_queue);
2388
2389         /*
2390          * Signal tasks blocked in i915_gem_wait_for_error that the pending
2391          * reset state is cleared.
2392          */
2393         if (reset_completed)
2394                 wake_up_all(&dev_priv->gpu_error.reset_queue);
2395 }
2396
2397 /**
2398  * i915_reset_and_wakeup - do process context error handling work
2399  * @dev: drm device
2400  *
2401  * Fire an error uevent so userspace can see that a hang or error
2402  * was detected.
2403  */
2404 static void i915_reset_and_wakeup(struct drm_device *dev)
2405 {
2406         struct drm_i915_private *dev_priv = to_i915(dev);
2407         struct i915_gpu_error *error = &dev_priv->gpu_error;
2408         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2409         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2410         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2411         int ret;
2412
2413         kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2414
2415         /*
2416          * Note that there's only one work item which does gpu resets, so we
2417          * need not worry about concurrent gpu resets potentially incrementing
2418          * error->reset_counter twice. We only need to take care of another
2419          * racing irq/hangcheck declaring the gpu dead for a second time. A
2420          * quick check for that is good enough: schedule_work ensures the
2421          * correct ordering between hang detection and this work item, and since
2422          * the reset in-progress bit is only ever set by code outside of this
2423          * work we don't need to worry about any other races.
2424          */
2425         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2426                 DRM_DEBUG_DRIVER("resetting chip\n");
2427                 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2428                                    reset_event);
2429
2430                 /*
2431                  * In most cases it's guaranteed that we get here with an RPM
2432                  * reference held, for example because there is a pending GPU
2433                  * request that won't finish until the reset is done. This
2434                  * isn't the case at least when we get here by doing a
2435                  * simulated reset via debugs, so get an RPM reference.
2436                  */
2437                 intel_runtime_pm_get(dev_priv);
2438
2439                 intel_prepare_reset(dev);
2440
2441                 /*
2442                  * All state reset _must_ be completed before we update the
2443                  * reset counter, for otherwise waiters might miss the reset
2444                  * pending state and not properly drop locks, resulting in
2445                  * deadlocks with the reset work.
2446                  */
2447                 ret = i915_reset(dev);
2448
2449                 intel_finish_reset(dev);
2450
2451                 intel_runtime_pm_put(dev_priv);
2452
2453                 if (ret == 0) {
2454                         /*
2455                          * After all the gem state is reset, increment the reset
2456                          * counter and wake up everyone waiting for the reset to
2457                          * complete.
2458                          *
2459                          * Since unlock operations are a one-sided barrier only,
2460                          * we need to insert a barrier here to order any seqno
2461                          * updates before
2462                          * the counter increment.
2463                          */
2464                         smp_mb__before_atomic();
2465                         atomic_inc(&dev_priv->gpu_error.reset_counter);
2466
2467                         kobject_uevent_env(&dev->primary->kdev->kobj,
2468                                            KOBJ_CHANGE, reset_done_event);
2469                 } else {
2470                         atomic_or(I915_WEDGED, &error->reset_counter);
2471                 }
2472
2473                 /*
2474                  * Note: The wake_up also serves as a memory barrier so that
2475                  * waiters see the update value of the reset counter atomic_t.
2476                  */
2477                 i915_error_wake_up(dev_priv, true);
2478         }
2479 }
2480
2481 static void i915_report_and_clear_eir(struct drm_device *dev)
2482 {
2483         struct drm_i915_private *dev_priv = dev->dev_private;
2484         uint32_t instdone[I915_NUM_INSTDONE_REG];
2485         u32 eir = I915_READ(EIR);
2486         int pipe, i;
2487
2488         if (!eir)
2489                 return;
2490
2491         pr_err("render error detected, EIR: 0x%08x\n", eir);
2492
2493         i915_get_extra_instdone(dev, instdone);
2494
2495         if (IS_G4X(dev)) {
2496                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2497                         u32 ipeir = I915_READ(IPEIR_I965);
2498
2499                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2500                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2501                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
2502                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2503                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2504                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2505                         I915_WRITE(IPEIR_I965, ipeir);
2506                         POSTING_READ(IPEIR_I965);
2507                 }
2508                 if (eir & GM45_ERROR_PAGE_TABLE) {
2509                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2510                         pr_err("page table error\n");
2511                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2512                         I915_WRITE(PGTBL_ER, pgtbl_err);
2513                         POSTING_READ(PGTBL_ER);
2514                 }
2515         }
2516
2517         if (!IS_GEN2(dev)) {
2518                 if (eir & I915_ERROR_PAGE_TABLE) {
2519                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2520                         pr_err("page table error\n");
2521                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2522                         I915_WRITE(PGTBL_ER, pgtbl_err);
2523                         POSTING_READ(PGTBL_ER);
2524                 }
2525         }
2526
2527         if (eir & I915_ERROR_MEMORY_REFRESH) {
2528                 pr_err("memory refresh error:\n");
2529                 for_each_pipe(dev_priv, pipe)
2530                         pr_err("pipe %c stat: 0x%08x\n",
2531                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2532                 /* pipestat has already been acked */
2533         }
2534         if (eir & I915_ERROR_INSTRUCTION) {
2535                 pr_err("instruction error\n");
2536                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2537                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2538                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2539                 if (INTEL_INFO(dev)->gen < 4) {
2540                         u32 ipeir = I915_READ(IPEIR);
2541
2542                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2543                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2544                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2545                         I915_WRITE(IPEIR, ipeir);
2546                         POSTING_READ(IPEIR);
2547                 } else {
2548                         u32 ipeir = I915_READ(IPEIR_I965);
2549
2550                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2551                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2552                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2553                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2554                         I915_WRITE(IPEIR_I965, ipeir);
2555                         POSTING_READ(IPEIR_I965);
2556                 }
2557         }
2558
2559         I915_WRITE(EIR, eir);
2560         POSTING_READ(EIR);
2561         eir = I915_READ(EIR);
2562         if (eir) {
2563                 /*
2564                  * some errors might have become stuck,
2565                  * mask them.
2566                  */
2567                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2568                 I915_WRITE(EMR, I915_READ(EMR) | eir);
2569                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2570         }
2571 }
2572
2573 /**
2574  * i915_handle_error - handle a gpu error
2575  * @dev: drm device
2576  *
2577  * Do some basic checking of register state at error time and
2578  * dump it to the syslog.  Also call i915_capture_error_state() to make
2579  * sure we get a record and make it available in debugfs.  Fire a uevent
2580  * so userspace knows something bad happened (should trigger collection
2581  * of a ring dump etc.).
2582  */
2583 void i915_handle_error(struct drm_device *dev, bool wedged,
2584                        const char *fmt, ...)
2585 {
2586         struct drm_i915_private *dev_priv = dev->dev_private;
2587         va_list args;
2588         char error_msg[80];
2589
2590         va_start(args, fmt);
2591         vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2592         va_end(args);
2593
2594         i915_capture_error_state(dev, wedged, error_msg);
2595         i915_report_and_clear_eir(dev);
2596
2597         if (wedged) {
2598                 atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2599                                 &dev_priv->gpu_error.reset_counter);
2600
2601                 /*
2602                  * Wakeup waiting processes so that the reset function
2603                  * i915_reset_and_wakeup doesn't deadlock trying to grab
2604                  * various locks. By bumping the reset counter first, the woken
2605                  * processes will see a reset in progress and back off,
2606                  * releasing their locks and then wait for the reset completion.
2607                  * We must do this for _all_ gpu waiters that might hold locks
2608                  * that the reset work needs to acquire.
2609                  *
2610                  * Note: The wake_up serves as the required memory barrier to
2611                  * ensure that the waiters see the updated value of the reset
2612                  * counter atomic_t.
2613                  */
2614                 i915_error_wake_up(dev_priv, false);
2615         }
2616
2617         i915_reset_and_wakeup(dev);
2618 }
2619
2620 /* Called from drm generic code, passed 'crtc' which
2621  * we use as a pipe index
2622  */
2623 static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
2624 {
2625         struct drm_i915_private *dev_priv = dev->dev_private;
2626         unsigned long irqflags;
2627
2628         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2629         if (INTEL_INFO(dev)->gen >= 4)
2630                 i915_enable_pipestat(dev_priv, pipe,
2631                                      PIPE_START_VBLANK_INTERRUPT_STATUS);
2632         else
2633                 i915_enable_pipestat(dev_priv, pipe,
2634                                      PIPE_VBLANK_INTERRUPT_STATUS);
2635         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2636
2637         return 0;
2638 }
2639
2640 static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
2641 {
2642         struct drm_i915_private *dev_priv = dev->dev_private;
2643         unsigned long irqflags;
2644         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2645                                                      DE_PIPE_VBLANK(pipe);
2646
2647         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2648         ironlake_enable_display_irq(dev_priv, bit);
2649         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2650
2651         return 0;
2652 }
2653
2654 static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
2655 {
2656         struct drm_i915_private *dev_priv = dev->dev_private;
2657         unsigned long irqflags;
2658
2659         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2660         i915_enable_pipestat(dev_priv, pipe,
2661                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2662         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2663
2664         return 0;
2665 }
2666
2667 static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2668 {
2669         struct drm_i915_private *dev_priv = dev->dev_private;
2670         unsigned long irqflags;
2671
2672         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2673         dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2674         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2675         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2676         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2677         return 0;
2678 }
2679
2680 /* Called from drm generic code, passed 'crtc' which
2681  * we use as a pipe index
2682  */
2683 static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
2684 {
2685         struct drm_i915_private *dev_priv = dev->dev_private;
2686         unsigned long irqflags;
2687
2688         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2689         i915_disable_pipestat(dev_priv, pipe,
2690                               PIPE_VBLANK_INTERRUPT_STATUS |
2691                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2692         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2693 }
2694
2695 static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
2696 {
2697         struct drm_i915_private *dev_priv = dev->dev_private;
2698         unsigned long irqflags;
2699         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2700                                                      DE_PIPE_VBLANK(pipe);
2701
2702         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2703         ironlake_disable_display_irq(dev_priv, bit);
2704         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2705 }
2706
2707 static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
2708 {
2709         struct drm_i915_private *dev_priv = dev->dev_private;
2710         unsigned long irqflags;
2711
2712         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2713         i915_disable_pipestat(dev_priv, pipe,
2714                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2715         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2716 }
2717
2718 static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2719 {
2720         struct drm_i915_private *dev_priv = dev->dev_private;
2721         unsigned long irqflags;
2722
2723         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2724         dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2725         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2726         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2727         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2728 }
2729
2730 static bool
2731 ring_idle(struct intel_engine_cs *ring, u32 seqno)
2732 {
2733         return (list_empty(&ring->request_list) ||
2734                 i915_seqno_passed(seqno, ring->last_submitted_seqno));
2735 }
2736
2737 static bool
2738 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2739 {
2740         if (INTEL_INFO(dev)->gen >= 8) {
2741                 return (ipehr >> 23) == 0x1c;
2742         } else {
2743                 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2744                 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2745                                  MI_SEMAPHORE_REGISTER);
2746         }
2747 }
2748
2749 static struct intel_engine_cs *
2750 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2751 {
2752         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2753         struct intel_engine_cs *signaller;
2754         int i;
2755
2756         if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2757                 for_each_ring(signaller, dev_priv, i) {
2758                         if (ring == signaller)
2759                                 continue;
2760
2761                         if (offset == signaller->semaphore.signal_ggtt[ring->id])
2762                                 return signaller;
2763                 }
2764         } else {
2765                 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2766
2767                 for_each_ring(signaller, dev_priv, i) {
2768                         if(ring == signaller)
2769                                 continue;
2770
2771                         if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2772                                 return signaller;
2773                 }
2774         }
2775
2776         DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2777                   ring->id, ipehr, offset);
2778
2779         return NULL;
2780 }
2781
2782 static struct intel_engine_cs *
2783 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2784 {
2785         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2786         u32 cmd, ipehr, head;
2787         u64 offset = 0;
2788         int i, backwards;
2789
2790         /*
2791          * This function does not support execlist mode - any attempt to
2792          * proceed further into this function will result in a kernel panic
2793          * when dereferencing ring->buffer, which is not set up in execlist
2794          * mode.
2795          *
2796          * The correct way of doing it would be to derive the currently
2797          * executing ring buffer from the current context, which is derived
2798          * from the currently running request. Unfortunately, to get the
2799          * current request we would have to grab the struct_mutex before doing
2800          * anything else, which would be ill-advised since some other thread
2801          * might have grabbed it already and managed to hang itself, causing
2802          * the hang checker to deadlock.
2803          *
2804          * Therefore, this function does not support execlist mode in its
2805          * current form. Just return NULL and move on.
2806          */
2807         if (ring->buffer == NULL)
2808                 return NULL;
2809
2810         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2811         if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2812                 return NULL;
2813
2814         /*
2815          * HEAD is likely pointing to the dword after the actual command,
2816          * so scan backwards until we find the MBOX. But limit it to just 3
2817          * or 4 dwords depending on the semaphore wait command size.
2818          * Note that we don't care about ACTHD here since that might
2819          * point at at batch, and semaphores are always emitted into the
2820          * ringbuffer itself.
2821          */
2822         head = I915_READ_HEAD(ring) & HEAD_ADDR;
2823         backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2824
2825         for (i = backwards; i; --i) {
2826                 /*
2827                  * Be paranoid and presume the hw has gone off into the wild -
2828                  * our ring is smaller than what the hardware (and hence
2829                  * HEAD_ADDR) allows. Also handles wrap-around.
2830                  */
2831                 head &= ring->buffer->size - 1;
2832
2833                 /* This here seems to blow up */
2834                 cmd = ioread32(ring->buffer->virtual_start + head);
2835                 if (cmd == ipehr)
2836                         break;
2837
2838                 head -= 4;
2839         }
2840
2841         if (!i)
2842                 return NULL;
2843
2844         *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2845         if (INTEL_INFO(ring->dev)->gen >= 8) {
2846                 offset = ioread32(ring->buffer->virtual_start + head + 12);
2847                 offset <<= 32;
2848                 offset = ioread32(ring->buffer->virtual_start + head + 8);
2849         }
2850         return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2851 }
2852
2853 static int semaphore_passed(struct intel_engine_cs *ring)
2854 {
2855         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2856         struct intel_engine_cs *signaller;
2857         u32 seqno;
2858
2859         ring->hangcheck.deadlock++;
2860
2861         signaller = semaphore_waits_for(ring, &seqno);
2862         if (signaller == NULL)
2863                 return -1;
2864
2865         /* Prevent pathological recursion due to driver bugs */
2866         if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2867                 return -1;
2868
2869         if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2870                 return 1;
2871
2872         /* cursory check for an unkickable deadlock */
2873         if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2874             semaphore_passed(signaller) < 0)
2875                 return -1;
2876
2877         return 0;
2878 }
2879
2880 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2881 {
2882         struct intel_engine_cs *ring;
2883         int i;
2884
2885         for_each_ring(ring, dev_priv, i)
2886                 ring->hangcheck.deadlock = 0;
2887 }
2888
2889 static enum intel_ring_hangcheck_action
2890 ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2891 {
2892         struct drm_device *dev = ring->dev;
2893         struct drm_i915_private *dev_priv = dev->dev_private;
2894         u32 tmp;
2895
2896         if (acthd != ring->hangcheck.acthd) {
2897                 if (acthd > ring->hangcheck.max_acthd) {
2898                         ring->hangcheck.max_acthd = acthd;
2899                         return HANGCHECK_ACTIVE;
2900                 }
2901
2902                 return HANGCHECK_ACTIVE_LOOP;
2903         }
2904
2905         if (IS_GEN2(dev))
2906                 return HANGCHECK_HUNG;
2907
2908         /* Is the chip hanging on a WAIT_FOR_EVENT?
2909          * If so we can simply poke the RB_WAIT bit
2910          * and break the hang. This should work on
2911          * all but the second generation chipsets.
2912          */
2913         tmp = I915_READ_CTL(ring);
2914         if (tmp & RING_WAIT) {
2915                 i915_handle_error(dev, false,
2916                                   "Kicking stuck wait on %s",
2917                                   ring->name);
2918                 I915_WRITE_CTL(ring, tmp);
2919                 return HANGCHECK_KICK;
2920         }
2921
2922         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2923                 switch (semaphore_passed(ring)) {
2924                 default:
2925                         return HANGCHECK_HUNG;
2926                 case 1:
2927                         i915_handle_error(dev, false,
2928                                           "Kicking stuck semaphore on %s",
2929                                           ring->name);
2930                         I915_WRITE_CTL(ring, tmp);
2931                         return HANGCHECK_KICK;
2932                 case 0:
2933                         return HANGCHECK_WAIT;
2934                 }
2935         }
2936
2937         return HANGCHECK_HUNG;
2938 }
2939
2940 /*
2941  * This is called when the chip hasn't reported back with completed
2942  * batchbuffers in a long time. We keep track per ring seqno progress and
2943  * if there are no progress, hangcheck score for that ring is increased.
2944  * Further, acthd is inspected to see if the ring is stuck. On stuck case
2945  * we kick the ring. If we see no progress on three subsequent calls
2946  * we assume chip is wedged and try to fix it by resetting the chip.
2947  */
2948 static void i915_hangcheck_elapsed(struct work_struct *work)
2949 {
2950         struct drm_i915_private *dev_priv =
2951                 container_of(work, typeof(*dev_priv),
2952                              gpu_error.hangcheck_work.work);
2953         struct drm_device *dev = dev_priv->dev;
2954         struct intel_engine_cs *ring;
2955         int i;
2956         int busy_count = 0, rings_hung = 0;
2957         bool stuck[I915_NUM_RINGS] = { 0 };
2958 #define BUSY 1
2959 #define KICK 5
2960 #define HUNG 20
2961
2962         if (!i915.enable_hangcheck)
2963                 return;
2964
2965         for_each_ring(ring, dev_priv, i) {
2966                 u64 acthd;
2967                 u32 seqno;
2968                 bool busy = true;
2969
2970                 semaphore_clear_deadlocks(dev_priv);
2971
2972                 seqno = ring->get_seqno(ring, false);
2973                 acthd = intel_ring_get_active_head(ring);
2974
2975                 if (ring->hangcheck.seqno == seqno) {
2976                         if (ring_idle(ring, seqno)) {
2977                                 ring->hangcheck.action = HANGCHECK_IDLE;
2978
2979                                 if (waitqueue_active(&ring->irq_queue)) {
2980                                         /* Issue a wake-up to catch stuck h/w. */
2981                                         if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2982                                                 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2983                                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2984                                                                   ring->name);
2985                                                 else
2986                                                         DRM_INFO("Fake missed irq on %s\n",
2987                                                                  ring->name);
2988                                                 wake_up_all(&ring->irq_queue);
2989                                         }
2990                                         /* Safeguard against driver failure */
2991                                         ring->hangcheck.score += BUSY;
2992                                 } else
2993                                         busy = false;
2994                         } else {
2995                                 /* We always increment the hangcheck score
2996                                  * if the ring is busy and still processing
2997                                  * the same request, so that no single request
2998                                  * can run indefinitely (such as a chain of
2999                                  * batches). The only time we do not increment
3000                                  * the hangcheck score on this ring, if this
3001                                  * ring is in a legitimate wait for another
3002                                  * ring. In that case the waiting ring is a
3003                                  * victim and we want to be sure we catch the
3004                                  * right culprit. Then every time we do kick
3005                                  * the ring, add a small increment to the
3006                                  * score so that we can catch a batch that is
3007                                  * being repeatedly kicked and so responsible
3008                                  * for stalling the machine.
3009                                  */
3010                                 ring->hangcheck.action = ring_stuck(ring,
3011                                                                     acthd);
3012
3013                                 switch (ring->hangcheck.action) {
3014                                 case HANGCHECK_IDLE:
3015                                 case HANGCHECK_WAIT:
3016                                 case HANGCHECK_ACTIVE:
3017                                         break;
3018                                 case HANGCHECK_ACTIVE_LOOP:
3019                                         ring->hangcheck.score += BUSY;
3020                                         break;
3021                                 case HANGCHECK_KICK:
3022                                         ring->hangcheck.score += KICK;
3023                                         break;
3024                                 case HANGCHECK_HUNG:
3025                                         ring->hangcheck.score += HUNG;
3026                                         stuck[i] = true;
3027                                         break;
3028                                 }
3029                         }
3030                 } else {
3031                         ring->hangcheck.action = HANGCHECK_ACTIVE;
3032
3033                         /* Gradually reduce the count so that we catch DoS
3034                          * attempts across multiple batches.
3035                          */
3036                         if (ring->hangcheck.score > 0)
3037                                 ring->hangcheck.score--;
3038
3039                         ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
3040                 }
3041
3042                 ring->hangcheck.seqno = seqno;
3043                 ring->hangcheck.acthd = acthd;
3044                 busy_count += busy;
3045         }
3046
3047         for_each_ring(ring, dev_priv, i) {
3048                 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3049                         DRM_INFO("%s on %s\n",
3050                                  stuck[i] ? "stuck" : "no progress",
3051                                  ring->name);
3052                         rings_hung++;
3053                 }
3054         }
3055
3056         if (rings_hung)
3057                 return i915_handle_error(dev, true, "Ring hung");
3058
3059         if (busy_count)
3060                 /* Reset timer case chip hangs without another request
3061                  * being added */
3062                 i915_queue_hangcheck(dev);
3063 }
3064
3065 void i915_queue_hangcheck(struct drm_device *dev)
3066 {
3067         struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3068
3069         if (!i915.enable_hangcheck)
3070                 return;
3071
3072         /* Don't continually defer the hangcheck so that it is always run at
3073          * least once after work has been scheduled on any ring. Otherwise,
3074          * we will ignore a hung ring if a second ring is kept busy.
3075          */
3076
3077         queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
3078                            round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
3079 }
3080
3081 static void ibx_irq_reset(struct drm_device *dev)
3082 {
3083         struct drm_i915_private *dev_priv = dev->dev_private;
3084
3085         if (HAS_PCH_NOP(dev))
3086                 return;
3087
3088         GEN5_IRQ_RESET(SDE);
3089
3090         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
3091                 I915_WRITE(SERR_INT, 0xffffffff);
3092 }
3093
3094 /*
3095  * SDEIER is also touched by the interrupt handler to work around missed PCH
3096  * interrupts. Hence we can't update it after the interrupt handler is enabled -
3097  * instead we unconditionally enable all PCH interrupt sources here, but then
3098  * only unmask them as needed with SDEIMR.
3099  *
3100  * This function needs to be called before interrupts are enabled.
3101  */
3102 static void ibx_irq_pre_postinstall(struct drm_device *dev)
3103 {
3104         struct drm_i915_private *dev_priv = dev->dev_private;
3105
3106         if (HAS_PCH_NOP(dev))
3107                 return;
3108
3109         WARN_ON(I915_READ(SDEIER) != 0);
3110         I915_WRITE(SDEIER, 0xffffffff);
3111         POSTING_READ(SDEIER);
3112 }
3113
3114 static void gen5_gt_irq_reset(struct drm_device *dev)
3115 {
3116         struct drm_i915_private *dev_priv = dev->dev_private;
3117
3118         GEN5_IRQ_RESET(GT);
3119         if (INTEL_INFO(dev)->gen >= 6)
3120                 GEN5_IRQ_RESET(GEN6_PM);
3121 }
3122
3123 /* drm_dma.h hooks
3124 */
3125 static void ironlake_irq_reset(struct drm_device *dev)
3126 {
3127         struct drm_i915_private *dev_priv = dev->dev_private;
3128
3129         I915_WRITE(HWSTAM, 0xffffffff);
3130
3131         GEN5_IRQ_RESET(DE);
3132         if (IS_GEN7(dev))
3133                 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3134
3135         gen5_gt_irq_reset(dev);
3136
3137         ibx_irq_reset(dev);
3138 }
3139
3140 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
3141 {
3142         enum pipe pipe;
3143
3144         i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
3145         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3146
3147         for_each_pipe(dev_priv, pipe)
3148                 I915_WRITE(PIPESTAT(pipe), 0xffff);
3149
3150         GEN5_IRQ_RESET(VLV_);
3151 }
3152
3153 static void valleyview_irq_preinstall(struct drm_device *dev)
3154 {
3155         struct drm_i915_private *dev_priv = dev->dev_private;
3156
3157         /* VLV magic */
3158         I915_WRITE(VLV_IMR, 0);
3159         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
3160         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
3161         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
3162
3163         gen5_gt_irq_reset(dev);
3164
3165         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3166
3167         vlv_display_irq_reset(dev_priv);
3168 }
3169
3170 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3171 {
3172         GEN8_IRQ_RESET_NDX(GT, 0);
3173         GEN8_IRQ_RESET_NDX(GT, 1);
3174         GEN8_IRQ_RESET_NDX(GT, 2);
3175         GEN8_IRQ_RESET_NDX(GT, 3);
3176 }
3177
3178 static void gen8_irq_reset(struct drm_device *dev)
3179 {
3180         struct drm_i915_private *dev_priv = dev->dev_private;
3181         int pipe;
3182
3183         I915_WRITE(GEN8_MASTER_IRQ, 0);
3184         POSTING_READ(GEN8_MASTER_IRQ);
3185
3186         gen8_gt_irq_reset(dev_priv);
3187
3188         for_each_pipe(dev_priv, pipe)
3189                 if (intel_display_power_is_enabled(dev_priv,
3190                                                    POWER_DOMAIN_PIPE(pipe)))
3191                         GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3192
3193         GEN5_IRQ_RESET(GEN8_DE_PORT_);
3194         GEN5_IRQ_RESET(GEN8_DE_MISC_);
3195         GEN5_IRQ_RESET(GEN8_PCU_);
3196
3197         if (HAS_PCH_SPLIT(dev))
3198                 ibx_irq_reset(dev);
3199 }
3200
3201 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3202                                      unsigned int pipe_mask)
3203 {
3204         uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3205
3206         spin_lock_irq(&dev_priv->irq_lock);
3207         if (pipe_mask & 1 << PIPE_A)
3208                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
3209                                   dev_priv->de_irq_mask[PIPE_A],
3210                                   ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
3211         if (pipe_mask & 1 << PIPE_B)
3212                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
3213                                   dev_priv->de_irq_mask[PIPE_B],
3214                                   ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
3215         if (pipe_mask & 1 << PIPE_C)
3216                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
3217                                   dev_priv->de_irq_mask[PIPE_C],
3218                                   ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3219         spin_unlock_irq(&dev_priv->irq_lock);
3220 }
3221
3222 static void cherryview_irq_preinstall(struct drm_device *dev)
3223 {
3224         struct drm_i915_private *dev_priv = dev->dev_private;
3225
3226         I915_WRITE(GEN8_MASTER_IRQ, 0);
3227         POSTING_READ(GEN8_MASTER_IRQ);
3228
3229         gen8_gt_irq_reset(dev_priv);
3230
3231         GEN5_IRQ_RESET(GEN8_PCU_);
3232
3233         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
3234
3235         vlv_display_irq_reset(dev_priv);
3236 }
3237
3238 static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
3239                                   const u32 hpd[HPD_NUM_PINS])
3240 {
3241         struct drm_i915_private *dev_priv = to_i915(dev);
3242         struct intel_encoder *encoder;
3243         u32 enabled_irqs = 0;
3244
3245         for_each_intel_encoder(dev, encoder)
3246                 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
3247                         enabled_irqs |= hpd[encoder->hpd_pin];
3248
3249         return enabled_irqs;
3250 }
3251
3252 static void ibx_hpd_irq_setup(struct drm_device *dev)
3253 {
3254         struct drm_i915_private *dev_priv = dev->dev_private;
3255         u32 hotplug_irqs, hotplug, enabled_irqs;
3256
3257         if (HAS_PCH_IBX(dev)) {
3258                 hotplug_irqs = SDE_HOTPLUG_MASK;
3259                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3260         } else {
3261                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3262                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3263         }
3264
3265         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3266
3267         /*
3268          * Enable digital hotplug on the PCH, and configure the DP short pulse
3269          * duration to 2ms (which is the minimum in the Display Port spec).
3270          * The pulse duration bits are reserved on LPT+.
3271          */
3272         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3273         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3274         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3275         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3276         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3277         /*
3278          * When CPU and PCH are on the same package, port A
3279          * HPD must be enabled in both north and south.
3280          */
3281         if (HAS_PCH_LPT_LP(dev))
3282                 hotplug |= PORTA_HOTPLUG_ENABLE;
3283         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3284 }
3285
3286 static void spt_hpd_irq_setup(struct drm_device *dev)
3287 {
3288         struct drm_i915_private *dev_priv = dev->dev_private;
3289         u32 hotplug_irqs, hotplug, enabled_irqs;
3290
3291         hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3292         enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
3293
3294         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3295
3296         /* Enable digital hotplug on the PCH */
3297         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3298         hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3299                 PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3300         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3301
3302         hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3303         hotplug |= PORTE_HOTPLUG_ENABLE;
3304         I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3305 }
3306
3307 static void ilk_hpd_irq_setup(struct drm_device *dev)
3308 {
3309         struct drm_i915_private *dev_priv = dev->dev_private;
3310         u32 hotplug_irqs, hotplug, enabled_irqs;
3311
3312         if (INTEL_INFO(dev)->gen >= 8) {
3313                 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3314                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
3315
3316                 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3317         } else if (INTEL_INFO(dev)->gen >= 7) {
3318                 hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3319                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3320
3321                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3322         } else {
3323                 hotplug_irqs = DE_DP_A_HOTPLUG;
3324                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3325
3326                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3327         }
3328
3329         /*
3330          * Enable digital hotplug on the CPU, and configure the DP short pulse
3331          * duration to 2ms (which is the minimum in the Display Port spec)
3332          * The pulse duration bits are reserved on HSW+.
3333          */
3334         hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3335         hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3336         hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
3337         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3338
3339         ibx_hpd_irq_setup(dev);
3340 }
3341
3342 static void bxt_hpd_irq_setup(struct drm_device *dev)
3343 {
3344         struct drm_i915_private *dev_priv = dev->dev_private;
3345         u32 hotplug_irqs, hotplug, enabled_irqs;
3346
3347         enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
3348         hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3349
3350         bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3351
3352         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3353         hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
3354                 PORTA_HOTPLUG_ENABLE;
3355         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3356 }
3357
3358 static void ibx_irq_postinstall(struct drm_device *dev)
3359 {
3360         struct drm_i915_private *dev_priv = dev->dev_private;
3361         u32 mask;
3362
3363         if (HAS_PCH_NOP(dev))
3364                 return;
3365
3366         if (HAS_PCH_IBX(dev))
3367                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3368         else
3369                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3370
3371         gen5_assert_iir_is_zero(dev_priv, SDEIIR);
3372         I915_WRITE(SDEIMR, ~mask);
3373 }
3374
3375 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3376 {
3377         struct drm_i915_private *dev_priv = dev->dev_private;
3378         u32 pm_irqs, gt_irqs;
3379
3380         pm_irqs = gt_irqs = 0;
3381
3382         dev_priv->gt_irq_mask = ~0;
3383         if (HAS_L3_DPF(dev)) {
3384                 /* L3 parity interrupt is always unmasked. */
3385                 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3386                 gt_irqs |= GT_PARITY_ERROR(dev);
3387         }
3388
3389         gt_irqs |= GT_RENDER_USER_INTERRUPT;
3390         if (IS_GEN5(dev)) {
3391                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3392                            ILK_BSD_USER_INTERRUPT;
3393         } else {
3394                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3395         }
3396
3397         GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3398
3399         if (INTEL_INFO(dev)->gen >= 6) {
3400                 /*
3401                  * RPS interrupts will get enabled/disabled on demand when RPS
3402                  * itself is enabled/disabled.
3403                  */
3404                 if (HAS_VEBOX(dev))
3405                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3406
3407                 dev_priv->pm_irq_mask = 0xffffffff;
3408                 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3409         }
3410 }
3411
3412 static int ironlake_irq_postinstall(struct drm_device *dev)
3413 {
3414         struct drm_i915_private *dev_priv = dev->dev_private;
3415         u32 display_mask, extra_mask;
3416
3417         if (INTEL_INFO(dev)->gen >= 7) {
3418                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3419                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3420                                 DE_PLANEB_FLIP_DONE_IVB |
3421                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3422                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3423                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3424                               DE_DP_A_HOTPLUG_IVB);
3425         } else {
3426                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3427                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3428                                 DE_AUX_CHANNEL_A |
3429                                 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3430                                 DE_POISON);
3431                 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3432                               DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3433                               DE_DP_A_HOTPLUG);
3434         }
3435
3436         dev_priv->irq_mask = ~display_mask;
3437
3438         I915_WRITE(HWSTAM, 0xeffe);
3439
3440         ibx_irq_pre_postinstall(dev);
3441
3442         GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3443
3444         gen5_gt_irq_postinstall(dev);
3445
3446         ibx_irq_postinstall(dev);
3447
3448         if (IS_IRONLAKE_M(dev)) {
3449                 /* Enable PCU event interrupts
3450                  *
3451                  * spinlocking not required here for correctness since interrupt
3452                  * setup is guaranteed to run in single-threaded context. But we
3453                  * need it to make the assert_spin_locked happy. */
3454                 spin_lock_irq(&dev_priv->irq_lock);
3455                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3456                 spin_unlock_irq(&dev_priv->irq_lock);
3457         }
3458
3459         return 0;
3460 }
3461
3462 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3463 {
3464         u32 pipestat_mask;
3465         u32 iir_mask;
3466         enum pipe pipe;
3467
3468         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3469                         PIPE_FIFO_UNDERRUN_STATUS;
3470
3471         for_each_pipe(dev_priv, pipe)
3472                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3473         POSTING_READ(PIPESTAT(PIPE_A));
3474
3475         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3476                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3477
3478         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3479         for_each_pipe(dev_priv, pipe)
3480                       i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3481
3482         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3483                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3484                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3485         if (IS_CHERRYVIEW(dev_priv))
3486                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3487         dev_priv->irq_mask &= ~iir_mask;
3488
3489         I915_WRITE(VLV_IIR, iir_mask);
3490         I915_WRITE(VLV_IIR, iir_mask);
3491         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3492         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3493         POSTING_READ(VLV_IMR);
3494 }
3495
3496 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3497 {
3498         u32 pipestat_mask;
3499         u32 iir_mask;
3500         enum pipe pipe;
3501
3502         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3503                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3504                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3505         if (IS_CHERRYVIEW(dev_priv))
3506                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3507
3508         dev_priv->irq_mask |= iir_mask;
3509         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3510         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3511         I915_WRITE(VLV_IIR, iir_mask);
3512         I915_WRITE(VLV_IIR, iir_mask);
3513         POSTING_READ(VLV_IIR);
3514
3515         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3516                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3517
3518         i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3519         for_each_pipe(dev_priv, pipe)
3520                 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3521
3522         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3523                         PIPE_FIFO_UNDERRUN_STATUS;
3524
3525         for_each_pipe(dev_priv, pipe)
3526                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3527         POSTING_READ(PIPESTAT(PIPE_A));
3528 }
3529
3530 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3531 {
3532         assert_spin_locked(&dev_priv->irq_lock);
3533
3534         if (dev_priv->display_irqs_enabled)
3535                 return;
3536
3537         dev_priv->display_irqs_enabled = true;
3538
3539         if (intel_irqs_enabled(dev_priv))
3540                 valleyview_display_irqs_install(dev_priv);
3541 }
3542
3543 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3544 {
3545         assert_spin_locked(&dev_priv->irq_lock);
3546
3547         if (!dev_priv->display_irqs_enabled)
3548                 return;
3549
3550         dev_priv->display_irqs_enabled = false;
3551
3552         if (intel_irqs_enabled(dev_priv))
3553                 valleyview_display_irqs_uninstall(dev_priv);
3554 }
3555
3556 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3557 {
3558         dev_priv->irq_mask = ~0;
3559
3560         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3561         POSTING_READ(PORT_HOTPLUG_EN);
3562
3563         I915_WRITE(VLV_IIR, 0xffffffff);
3564         I915_WRITE(VLV_IIR, 0xffffffff);
3565         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3566         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3567         POSTING_READ(VLV_IMR);
3568
3569         /* Interrupt setup is already guaranteed to be single-threaded, this is
3570          * just to make the assert_spin_locked check happy. */
3571         spin_lock_irq(&dev_priv->irq_lock);
3572         if (dev_priv->display_irqs_enabled)
3573                 valleyview_display_irqs_install(dev_priv);
3574         spin_unlock_irq(&dev_priv->irq_lock);
3575 }
3576
3577 static int valleyview_irq_postinstall(struct drm_device *dev)
3578 {
3579         struct drm_i915_private *dev_priv = dev->dev_private;
3580
3581         vlv_display_irq_postinstall(dev_priv);
3582
3583         gen5_gt_irq_postinstall(dev);
3584
3585         /* ack & enable invalid PTE error interrupts */
3586 #if 0 /* FIXME: add support to irq handler for checking these bits */
3587         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3588         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3589 #endif
3590
3591         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3592
3593         return 0;
3594 }
3595
3596 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3597 {
3598         /* These are interrupts we'll toggle with the ring mask register */
3599         uint32_t gt_interrupts[] = {
3600                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3601                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3602                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3603                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3604                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3605                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3606                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3607                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3608                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3609                 0,
3610                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3611                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3612                 };
3613
3614         dev_priv->pm_irq_mask = 0xffffffff;
3615         GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3616         GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3617         /*
3618          * RPS interrupts will get enabled/disabled on demand when RPS itself
3619          * is enabled/disabled.
3620          */
3621         GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3622         GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3623 }
3624
3625 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3626 {
3627         uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3628         uint32_t de_pipe_enables;
3629         u32 de_port_masked = GEN8_AUX_CHANNEL_A;
3630         u32 de_port_enables;
3631         enum pipe pipe;
3632
3633         if (INTEL_INFO(dev_priv)->gen >= 9) {
3634                 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3635                                   GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3636                 de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3637                                   GEN9_AUX_CHANNEL_D;
3638                 if (IS_BROXTON(dev_priv))
3639                         de_port_masked |= BXT_DE_PORT_GMBUS;
3640         } else {
3641                 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3642                                   GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3643         }
3644
3645         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3646                                            GEN8_PIPE_FIFO_UNDERRUN;
3647
3648         de_port_enables = de_port_masked;
3649         if (IS_BROXTON(dev_priv))
3650                 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3651         else if (IS_BROADWELL(dev_priv))
3652                 de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
3653
3654         dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3655         dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3656         dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3657
3658         for_each_pipe(dev_priv, pipe)
3659                 if (intel_display_power_is_enabled(dev_priv,
3660                                 POWER_DOMAIN_PIPE(pipe)))
3661                         GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3662                                           dev_priv->de_irq_mask[pipe],
3663                                           de_pipe_enables);
3664
3665         GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3666 }
3667
3668 static int gen8_irq_postinstall(struct drm_device *dev)
3669 {
3670         struct drm_i915_private *dev_priv = dev->dev_private;
3671
3672         if (HAS_PCH_SPLIT(dev))
3673                 ibx_irq_pre_postinstall(dev);
3674
3675         gen8_gt_irq_postinstall(dev_priv);
3676         gen8_de_irq_postinstall(dev_priv);
3677
3678         if (HAS_PCH_SPLIT(dev))
3679                 ibx_irq_postinstall(dev);
3680
3681         I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3682         POSTING_READ(GEN8_MASTER_IRQ);
3683
3684         return 0;
3685 }
3686
3687 static int cherryview_irq_postinstall(struct drm_device *dev)
3688 {
3689         struct drm_i915_private *dev_priv = dev->dev_private;
3690
3691         vlv_display_irq_postinstall(dev_priv);
3692
3693         gen8_gt_irq_postinstall(dev_priv);
3694
3695         I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3696         POSTING_READ(GEN8_MASTER_IRQ);
3697
3698         return 0;
3699 }
3700
3701 static void gen8_irq_uninstall(struct drm_device *dev)
3702 {
3703         struct drm_i915_private *dev_priv = dev->dev_private;
3704
3705         if (!dev_priv)
3706                 return;
3707
3708         gen8_irq_reset(dev);
3709 }
3710
3711 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3712 {
3713         /* Interrupt setup is already guaranteed to be single-threaded, this is
3714          * just to make the assert_spin_locked check happy. */
3715         spin_lock_irq(&dev_priv->irq_lock);
3716         if (dev_priv->display_irqs_enabled)
3717                 valleyview_display_irqs_uninstall(dev_priv);
3718         spin_unlock_irq(&dev_priv->irq_lock);
3719
3720         vlv_display_irq_reset(dev_priv);
3721
3722         dev_priv->irq_mask = ~0;
3723 }
3724
3725 static void valleyview_irq_uninstall(struct drm_device *dev)
3726 {
3727         struct drm_i915_private *dev_priv = dev->dev_private;
3728
3729         if (!dev_priv)
3730                 return;
3731
3732         I915_WRITE(VLV_MASTER_IER, 0);
3733
3734         gen5_gt_irq_reset(dev);
3735
3736         I915_WRITE(HWSTAM, 0xffffffff);
3737
3738         vlv_display_irq_uninstall(dev_priv);
3739 }
3740
3741 static void cherryview_irq_uninstall(struct drm_device *dev)
3742 {
3743         struct drm_i915_private *dev_priv = dev->dev_private;
3744
3745         if (!dev_priv)
3746                 return;
3747
3748         I915_WRITE(GEN8_MASTER_IRQ, 0);
3749         POSTING_READ(GEN8_MASTER_IRQ);
3750
3751         gen8_gt_irq_reset(dev_priv);
3752
3753         GEN5_IRQ_RESET(GEN8_PCU_);
3754
3755         vlv_display_irq_uninstall(dev_priv);
3756 }
3757
3758 static void ironlake_irq_uninstall(struct drm_device *dev)
3759 {
3760         struct drm_i915_private *dev_priv = dev->dev_private;
3761
3762         if (!dev_priv)
3763                 return;
3764
3765         ironlake_irq_reset(dev);
3766 }
3767
3768 static void i8xx_irq_preinstall(struct drm_device * dev)
3769 {
3770         struct drm_i915_private *dev_priv = dev->dev_private;
3771         int pipe;
3772
3773         for_each_pipe(dev_priv, pipe)
3774                 I915_WRITE(PIPESTAT(pipe), 0);
3775         I915_WRITE16(IMR, 0xffff);
3776         I915_WRITE16(IER, 0x0);
3777         POSTING_READ16(IER);
3778 }
3779
3780 static int i8xx_irq_postinstall(struct drm_device *dev)
3781 {
3782         struct drm_i915_private *dev_priv = dev->dev_private;
3783
3784         I915_WRITE16(EMR,
3785                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3786
3787         /* Unmask the interrupts that we always want on. */
3788         dev_priv->irq_mask =
3789                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3790                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3791                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3792                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3793         I915_WRITE16(IMR, dev_priv->irq_mask);
3794
3795         I915_WRITE16(IER,
3796                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3797                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3798                      I915_USER_INTERRUPT);
3799         POSTING_READ16(IER);
3800
3801         /* Interrupt setup is already guaranteed to be single-threaded, this is
3802          * just to make the assert_spin_locked check happy. */
3803         spin_lock_irq(&dev_priv->irq_lock);
3804         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3805         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3806         spin_unlock_irq(&dev_priv->irq_lock);
3807
3808         return 0;
3809 }
3810
3811 /*
3812  * Returns true when a page flip has completed.
3813  */
3814 static bool i8xx_handle_vblank(struct drm_device *dev,
3815                                int plane, int pipe, u32 iir)
3816 {
3817         struct drm_i915_private *dev_priv = dev->dev_private;
3818         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3819
3820         if (!intel_pipe_handle_vblank(dev, pipe))
3821                 return false;
3822
3823         if ((iir & flip_pending) == 0)
3824                 goto check_page_flip;
3825
3826         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3827          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3828          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3829          * the flip is completed (no longer pending). Since this doesn't raise
3830          * an interrupt per se, we watch for the change at vblank.
3831          */
3832         if (I915_READ16(ISR) & flip_pending)
3833                 goto check_page_flip;
3834
3835         intel_prepare_page_flip(dev, plane);
3836         intel_finish_page_flip(dev, pipe);
3837         return true;
3838
3839 check_page_flip:
3840         intel_check_page_flip(dev, pipe);
3841         return false;
3842 }
3843
3844 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3845 {
3846         struct drm_device *dev = arg;
3847         struct drm_i915_private *dev_priv = dev->dev_private;
3848         u16 iir, new_iir;
3849         u32 pipe_stats[2];
3850         int pipe;
3851         u16 flip_mask =
3852                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3853                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3854
3855         if (!intel_irqs_enabled(dev_priv))
3856                 return IRQ_NONE;
3857
3858         iir = I915_READ16(IIR);
3859         if (iir == 0)
3860                 return IRQ_NONE;
3861
3862         while (iir & ~flip_mask) {
3863                 /* Can't rely on pipestat interrupt bit in iir as it might
3864                  * have been cleared after the pipestat interrupt was received.
3865                  * It doesn't set the bit in iir again, but it still produces
3866                  * interrupts (for non-MSI).
3867                  */
3868                 spin_lock(&dev_priv->irq_lock);
3869                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3870                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3871
3872                 for_each_pipe(dev_priv, pipe) {
3873                         i915_reg_t reg = PIPESTAT(pipe);
3874                         pipe_stats[pipe] = I915_READ(reg);
3875
3876                         /*
3877                          * Clear the PIPE*STAT regs before the IIR
3878                          */
3879                         if (pipe_stats[pipe] & 0x8000ffff)
3880                                 I915_WRITE(reg, pipe_stats[pipe]);
3881                 }
3882                 spin_unlock(&dev_priv->irq_lock);
3883
3884                 I915_WRITE16(IIR, iir & ~flip_mask);
3885                 new_iir = I915_READ16(IIR); /* Flush posted writes */
3886
3887                 if (iir & I915_USER_INTERRUPT)
3888                         notify_ring(&dev_priv->ring[RCS]);
3889
3890                 for_each_pipe(dev_priv, pipe) {
3891                         int plane = pipe;
3892                         if (HAS_FBC(dev))
3893                                 plane = !plane;
3894
3895                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3896                             i8xx_handle_vblank(dev, plane, pipe, iir))
3897                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3898
3899                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3900                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3901
3902                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3903                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3904                                                                     pipe);
3905                 }
3906
3907                 iir = new_iir;
3908         }
3909
3910         return IRQ_HANDLED;
3911 }
3912
3913 static void i8xx_irq_uninstall(struct drm_device * dev)
3914 {
3915         struct drm_i915_private *dev_priv = dev->dev_private;
3916         int pipe;
3917
3918         for_each_pipe(dev_priv, pipe) {
3919                 /* Clear enable bits; then clear status bits */
3920                 I915_WRITE(PIPESTAT(pipe), 0);
3921                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3922         }
3923         I915_WRITE16(IMR, 0xffff);
3924         I915_WRITE16(IER, 0x0);
3925         I915_WRITE16(IIR, I915_READ16(IIR));
3926 }
3927
3928 static void i915_irq_preinstall(struct drm_device * dev)
3929 {
3930         struct drm_i915_private *dev_priv = dev->dev_private;
3931         int pipe;
3932
3933         if (I915_HAS_HOTPLUG(dev)) {
3934                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3935                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3936         }
3937
3938         I915_WRITE16(HWSTAM, 0xeffe);
3939         for_each_pipe(dev_priv, pipe)
3940                 I915_WRITE(PIPESTAT(pipe), 0);
3941         I915_WRITE(IMR, 0xffffffff);
3942         I915_WRITE(IER, 0x0);
3943         POSTING_READ(IER);
3944 }
3945
3946 static int i915_irq_postinstall(struct drm_device *dev)
3947 {
3948         struct drm_i915_private *dev_priv = dev->dev_private;
3949         u32 enable_mask;
3950
3951         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3952
3953         /* Unmask the interrupts that we always want on. */
3954         dev_priv->irq_mask =
3955                 ~(I915_ASLE_INTERRUPT |
3956                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3957                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3958                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3959                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3960
3961         enable_mask =
3962                 I915_ASLE_INTERRUPT |
3963                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3964                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3965                 I915_USER_INTERRUPT;
3966
3967         if (I915_HAS_HOTPLUG(dev)) {
3968                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3969                 POSTING_READ(PORT_HOTPLUG_EN);
3970
3971                 /* Enable in IER... */
3972                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3973                 /* and unmask in IMR */
3974                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3975         }
3976
3977         I915_WRITE(IMR, dev_priv->irq_mask);
3978         I915_WRITE(IER, enable_mask);
3979         POSTING_READ(IER);
3980
3981         i915_enable_asle_pipestat(dev);
3982
3983         /* Interrupt setup is already guaranteed to be single-threaded, this is
3984          * just to make the assert_spin_locked check happy. */
3985         spin_lock_irq(&dev_priv->irq_lock);
3986         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3987         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3988         spin_unlock_irq(&dev_priv->irq_lock);
3989
3990         return 0;
3991 }
3992
3993 /*
3994  * Returns true when a page flip has completed.
3995  */
3996 static bool i915_handle_vblank(struct drm_device *dev,
3997                                int plane, int pipe, u32 iir)
3998 {
3999         struct drm_i915_private *dev_priv = dev->dev_private;
4000         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
4001
4002         if (!intel_pipe_handle_vblank(dev, pipe))
4003                 return false;
4004
4005         if ((iir & flip_pending) == 0)
4006                 goto check_page_flip;
4007
4008         /* We detect FlipDone by looking for the change in PendingFlip from '1'
4009          * to '0' on the following vblank, i.e. IIR has the Pendingflip
4010          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
4011          * the flip is completed (no longer pending). Since this doesn't raise
4012          * an interrupt per se, we watch for the change at vblank.
4013          */
4014         if (I915_READ(ISR) & flip_pending)
4015                 goto check_page_flip;
4016
4017         intel_prepare_page_flip(dev, plane);
4018         intel_finish_page_flip(dev, pipe);
4019         return true;
4020
4021 check_page_flip:
4022         intel_check_page_flip(dev, pipe);
4023         return false;
4024 }
4025
4026 static irqreturn_t i915_irq_handler(int irq, void *arg)
4027 {
4028         struct drm_device *dev = arg;
4029         struct drm_i915_private *dev_priv = dev->dev_private;
4030         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4031         u32 flip_mask =
4032                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4033                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4034         int pipe, ret = IRQ_NONE;
4035
4036         if (!intel_irqs_enabled(dev_priv))
4037                 return IRQ_NONE;
4038
4039         iir = I915_READ(IIR);
4040         do {
4041                 bool irq_received = (iir & ~flip_mask) != 0;
4042                 bool blc_event = false;
4043
4044                 /* Can't rely on pipestat interrupt bit in iir as it might
4045                  * have been cleared after the pipestat interrupt was received.
4046                  * It doesn't set the bit in iir again, but it still produces
4047                  * interrupts (for non-MSI).
4048                  */
4049                 spin_lock(&dev_priv->irq_lock);
4050                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4051                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4052
4053                 for_each_pipe(dev_priv, pipe) {
4054                         i915_reg_t reg = PIPESTAT(pipe);
4055                         pipe_stats[pipe] = I915_READ(reg);
4056
4057                         /* Clear the PIPE*STAT regs before the IIR */
4058                         if (pipe_stats[pipe] & 0x8000ffff) {
4059                                 I915_WRITE(reg, pipe_stats[pipe]);
4060                                 irq_received = true;
4061                         }
4062                 }
4063                 spin_unlock(&dev_priv->irq_lock);
4064
4065                 if (!irq_received)
4066                         break;
4067
4068                 /* Consume port.  Then clear IIR or we'll miss events */
4069                 if (I915_HAS_HOTPLUG(dev) &&
4070                     iir & I915_DISPLAY_PORT_INTERRUPT)
4071                         i9xx_hpd_irq_handler(dev);
4072
4073                 I915_WRITE(IIR, iir & ~flip_mask);
4074                 new_iir = I915_READ(IIR); /* Flush posted writes */
4075
4076                 if (iir & I915_USER_INTERRUPT)
4077                         notify_ring(&dev_priv->ring[RCS]);
4078
4079                 for_each_pipe(dev_priv, pipe) {
4080                         int plane = pipe;
4081                         if (HAS_FBC(dev))
4082                                 plane = !plane;
4083
4084                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4085                             i915_handle_vblank(dev, plane, pipe, iir))
4086                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4087
4088                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4089                                 blc_event = true;
4090
4091                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4092                                 i9xx_pipe_crc_irq_handler(dev, pipe);
4093
4094                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4095                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
4096                                                                     pipe);
4097                 }
4098
4099                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4100                         intel_opregion_asle_intr(dev);
4101
4102                 /* With MSI, interrupts are only generated when iir
4103                  * transitions from zero to nonzero.  If another bit got
4104                  * set while we were handling the existing iir bits, then
4105                  * we would never get another interrupt.
4106                  *
4107                  * This is fine on non-MSI as well, as if we hit this path
4108                  * we avoid exiting the interrupt handler only to generate
4109                  * another one.
4110                  *
4111                  * Note that for MSI this could cause a stray interrupt report
4112                  * if an interrupt landed in the time between writing IIR and
4113                  * the posting read.  This should be rare enough to never
4114                  * trigger the 99% of 100,000 interrupts test for disabling
4115                  * stray interrupts.
4116                  */
4117                 ret = IRQ_HANDLED;
4118                 iir = new_iir;
4119         } while (iir & ~flip_mask);
4120
4121         return ret;
4122 }
4123
4124 static void i915_irq_uninstall(struct drm_device * dev)
4125 {
4126         struct drm_i915_private *dev_priv = dev->dev_private;
4127         int pipe;
4128
4129         if (I915_HAS_HOTPLUG(dev)) {
4130                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4131                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4132         }
4133
4134         I915_WRITE16(HWSTAM, 0xffff);
4135         for_each_pipe(dev_priv, pipe) {
4136                 /* Clear enable bits; then clear status bits */
4137                 I915_WRITE(PIPESTAT(pipe), 0);
4138                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
4139         }
4140         I915_WRITE(IMR, 0xffffffff);
4141         I915_WRITE(IER, 0x0);
4142
4143         I915_WRITE(IIR, I915_READ(IIR));
4144 }
4145
4146 static void i965_irq_preinstall(struct drm_device * dev)
4147 {
4148         struct drm_i915_private *dev_priv = dev->dev_private;
4149         int pipe;
4150
4151         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4152         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4153
4154         I915_WRITE(HWSTAM, 0xeffe);
4155         for_each_pipe(dev_priv, pipe)
4156                 I915_WRITE(PIPESTAT(pipe), 0);
4157         I915_WRITE(IMR, 0xffffffff);
4158         I915_WRITE(IER, 0x0);
4159         POSTING_READ(IER);
4160 }
4161
4162 static int i965_irq_postinstall(struct drm_device *dev)
4163 {
4164         struct drm_i915_private *dev_priv = dev->dev_private;
4165         u32 enable_mask;
4166         u32 error_mask;
4167
4168         /* Unmask the interrupts that we always want on. */
4169         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4170                                I915_DISPLAY_PORT_INTERRUPT |
4171                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4172                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4173                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4174                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
4175                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4176
4177         enable_mask = ~dev_priv->irq_mask;
4178         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4179                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4180         enable_mask |= I915_USER_INTERRUPT;
4181
4182         if (IS_G4X(dev))
4183                 enable_mask |= I915_BSD_USER_INTERRUPT;
4184
4185         /* Interrupt setup is already guaranteed to be single-threaded, this is
4186          * just to make the assert_spin_locked check happy. */
4187         spin_lock_irq(&dev_priv->irq_lock);
4188         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4189         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4190         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4191         spin_unlock_irq(&dev_priv->irq_lock);
4192
4193         /*
4194          * Enable some error detection, note the instruction error mask
4195          * bit is reserved, so we leave it masked.
4196          */
4197         if (IS_G4X(dev)) {
4198                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4199                                GM45_ERROR_MEM_PRIV |
4200                                GM45_ERROR_CP_PRIV |
4201                                I915_ERROR_MEMORY_REFRESH);
4202         } else {
4203                 error_mask = ~(I915_ERROR_PAGE_TABLE |
4204                                I915_ERROR_MEMORY_REFRESH);
4205         }
4206         I915_WRITE(EMR, error_mask);
4207
4208         I915_WRITE(IMR, dev_priv->irq_mask);
4209         I915_WRITE(IER, enable_mask);
4210         POSTING_READ(IER);
4211
4212         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4213         POSTING_READ(PORT_HOTPLUG_EN);
4214
4215         i915_enable_asle_pipestat(dev);
4216
4217         return 0;
4218 }
4219
4220 static void i915_hpd_irq_setup(struct drm_device *dev)
4221 {
4222         struct drm_i915_private *dev_priv = dev->dev_private;
4223         u32 hotplug_en;
4224
4225         assert_spin_locked(&dev_priv->irq_lock);
4226
4227         /* Note HDMI and DP share hotplug bits */
4228         /* enable bits are the same for all generations */
4229         hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4230         /* Programming the CRT detection parameters tends
4231            to generate a spurious hotplug event about three
4232            seconds later.  So just do it once.
4233         */
4234         if (IS_G4X(dev))
4235                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4236         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4237
4238         /* Ignore TV since it's buggy */
4239         i915_hotplug_interrupt_update_locked(dev_priv,
4240                                              HOTPLUG_INT_EN_MASK |
4241                                              CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4242                                              CRT_HOTPLUG_ACTIVATION_PERIOD_64,
4243                                              hotplug_en);
4244 }
4245
4246 static irqreturn_t i965_irq_handler(int irq, void *arg)
4247 {
4248         struct drm_device *dev = arg;
4249         struct drm_i915_private *dev_priv = dev->dev_private;
4250         u32 iir, new_iir;
4251         u32 pipe_stats[I915_MAX_PIPES];
4252         int ret = IRQ_NONE, pipe;
4253         u32 flip_mask =
4254                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4255                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4256
4257         if (!intel_irqs_enabled(dev_priv))
4258                 return IRQ_NONE;
4259
4260         iir = I915_READ(IIR);
4261
4262         for (;;) {
4263                 bool irq_received = (iir & ~flip_mask) != 0;
4264                 bool blc_event = false;
4265
4266                 /* Can't rely on pipestat interrupt bit in iir as it might
4267                  * have been cleared after the pipestat interrupt was received.
4268                  * It doesn't set the bit in iir again, but it still produces
4269                  * interrupts (for non-MSI).
4270                  */
4271                 spin_lock(&dev_priv->irq_lock);
4272                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4273                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4274
4275                 for_each_pipe(dev_priv, pipe) {
4276                         i915_reg_t reg = PIPESTAT(pipe);
4277                         pipe_stats[pipe] = I915_READ(reg);
4278
4279                         /*
4280                          * Clear the PIPE*STAT regs before the IIR
4281                          */
4282                         if (pipe_stats[pipe] & 0x8000ffff) {
4283                                 I915_WRITE(reg, pipe_stats[pipe]);
4284                                 irq_received = true;
4285                         }
4286                 }
4287                 spin_unlock(&dev_priv->irq_lock);
4288
4289                 if (!irq_received)
4290                         break;
4291
4292                 ret = IRQ_HANDLED;
4293
4294                 /* Consume port.  Then clear IIR or we'll miss events */
4295                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4296                         i9xx_hpd_irq_handler(dev);
4297
4298                 I915_WRITE(IIR, iir & ~flip_mask);
4299                 new_iir = I915_READ(IIR); /* Flush posted writes */
4300
4301                 if (iir & I915_USER_INTERRUPT)
4302                         notify_ring(&dev_priv->ring[RCS]);
4303                 if (iir & I915_BSD_USER_INTERRUPT)
4304                         notify_ring(&dev_priv->ring[VCS]);
4305
4306                 for_each_pipe(dev_priv, pipe) {
4307                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4308                             i915_handle_vblank(dev, pipe, pipe, iir))
4309                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4310
4311                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4312                                 blc_event = true;
4313
4314                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4315                                 i9xx_pipe_crc_irq_handler(dev, pipe);
4316
4317                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4318                                 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4319                 }
4320
4321                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4322                         intel_opregion_asle_intr(dev);
4323
4324                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4325                         gmbus_irq_handler(dev);
4326
4327                 /* With MSI, interrupts are only generated when iir
4328                  * transitions from zero to nonzero.  If another bit got
4329                  * set while we were handling the existing iir bits, then
4330                  * we would never get another interrupt.
4331                  *
4332                  * This is fine on non-MSI as well, as if we hit this path
4333                  * we avoid exiting the interrupt handler only to generate
4334                  * another one.
4335                  *
4336                  * Note that for MSI this could cause a stray interrupt report
4337                  * if an interrupt landed in the time between writing IIR and
4338                  * the posting read.  This should be rare enough to never
4339                  * trigger the 99% of 100,000 interrupts test for disabling
4340                  * stray interrupts.
4341                  */
4342                 iir = new_iir;
4343         }
4344
4345         return ret;
4346 }
4347
4348 static void i965_irq_uninstall(struct drm_device * dev)
4349 {
4350         struct drm_i915_private *dev_priv = dev->dev_private;
4351         int pipe;
4352
4353         if (!dev_priv)
4354                 return;
4355
4356         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4357         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4358
4359         I915_WRITE(HWSTAM, 0xffffffff);
4360         for_each_pipe(dev_priv, pipe)
4361                 I915_WRITE(PIPESTAT(pipe), 0);
4362         I915_WRITE(IMR, 0xffffffff);
4363         I915_WRITE(IER, 0x0);
4364
4365         for_each_pipe(dev_priv, pipe)
4366                 I915_WRITE(PIPESTAT(pipe),
4367                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4368         I915_WRITE(IIR, I915_READ(IIR));
4369 }
4370
4371 /**
4372  * intel_irq_init - initializes irq support
4373  * @dev_priv: i915 device instance
4374  *
4375  * This function initializes all the irq support including work items, timers
4376  * and all the vtables. It does not setup the interrupt itself though.
4377  */
4378 void intel_irq_init(struct drm_i915_private *dev_priv)
4379 {
4380         struct drm_device *dev = dev_priv->dev;
4381
4382         intel_hpd_init_work(dev_priv);
4383
4384         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4385         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4386
4387         /* Let's track the enabled rps events */
4388         if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4389                 /* WaGsvRC0ResidencyMethod:vlv */
4390                 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4391         else
4392                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4393
4394         INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4395                           i915_hangcheck_elapsed);
4396
4397         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4398
4399         if (IS_GEN2(dev_priv)) {
4400                 dev->max_vblank_count = 0;
4401                 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4402         } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4403                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4404                 dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4405         } else {
4406                 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4407                 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4408         }
4409
4410         /*
4411          * Opt out of the vblank disable timer on everything except gen2.
4412          * Gen2 doesn't have a hardware frame counter and so depends on
4413          * vblank interrupts to produce sane vblank seuquence numbers.
4414          */
4415         if (!IS_GEN2(dev_priv))
4416                 dev->vblank_disable_immediate = true;
4417
4418         dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4419         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4420
4421         if (IS_CHERRYVIEW(dev_priv)) {
4422                 dev->driver->irq_handler = cherryview_irq_handler;
4423                 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4424                 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4425                 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4426                 dev->driver->enable_vblank = valleyview_enable_vblank;
4427                 dev->driver->disable_vblank = valleyview_disable_vblank;
4428                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4429         } else if (IS_VALLEYVIEW(dev_priv)) {
4430                 dev->driver->irq_handler = valleyview_irq_handler;
4431                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4432                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4433                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4434                 dev->driver->enable_vblank = valleyview_enable_vblank;
4435                 dev->driver->disable_vblank = valleyview_disable_vblank;
4436                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4437         } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4438                 dev->driver->irq_handler = gen8_irq_handler;
4439                 dev->driver->irq_preinstall = gen8_irq_reset;
4440                 dev->driver->irq_postinstall = gen8_irq_postinstall;
4441                 dev->driver->irq_uninstall = gen8_irq_uninstall;
4442                 dev->driver->enable_vblank = gen8_enable_vblank;
4443                 dev->driver->disable_vblank = gen8_disable_vblank;
4444                 if (IS_BROXTON(dev))
4445                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4446                 else if (HAS_PCH_SPT(dev))
4447                         dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4448                 else
4449                         dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4450         } else if (HAS_PCH_SPLIT(dev)) {
4451                 dev->driver->irq_handler = ironlake_irq_handler;
4452                 dev->driver->irq_preinstall = ironlake_irq_reset;
4453                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4454                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4455                 dev->driver->enable_vblank = ironlake_enable_vblank;
4456                 dev->driver->disable_vblank = ironlake_disable_vblank;
4457                 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4458         } else {
4459                 if (INTEL_INFO(dev_priv)->gen == 2) {
4460                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
4461                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
4462                         dev->driver->irq_handler = i8xx_irq_handler;
4463                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
4464                 } else if (INTEL_INFO(dev_priv)->gen == 3) {
4465                         dev->driver->irq_preinstall = i915_irq_preinstall;
4466                         dev->driver->irq_postinstall = i915_irq_postinstall;
4467                         dev->driver->irq_uninstall = i915_irq_uninstall;
4468                         dev->driver->irq_handler = i915_irq_handler;
4469                 } else {
4470                         dev->driver->irq_preinstall = i965_irq_preinstall;
4471                         dev->driver->irq_postinstall = i965_irq_postinstall;
4472                         dev->driver->irq_uninstall = i965_irq_uninstall;
4473                         dev->driver->irq_handler = i965_irq_handler;
4474                 }
4475                 if (I915_HAS_HOTPLUG(dev_priv))
4476                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4477                 dev->driver->enable_vblank = i915_enable_vblank;
4478                 dev->driver->disable_vblank = i915_disable_vblank;
4479         }
4480 }
4481
4482 /**
4483  * intel_irq_install - enables the hardware interrupt
4484  * @dev_priv: i915 device instance
4485  *
4486  * This function enables the hardware interrupt handling, but leaves the hotplug
4487  * handling still disabled. It is called after intel_irq_init().
4488  *
4489  * In the driver load and resume code we need working interrupts in a few places
4490  * but don't want to deal with the hassle of concurrent probe and hotplug
4491  * workers. Hence the split into this two-stage approach.
4492  */
4493 int intel_irq_install(struct drm_i915_private *dev_priv)
4494 {
4495         /*
4496          * We enable some interrupt sources in our postinstall hooks, so mark
4497          * interrupts as enabled _before_ actually enabling them to avoid
4498          * special cases in our ordering checks.
4499          */
4500         dev_priv->pm.irqs_enabled = true;
4501
4502         return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4503 }
4504
4505 /**
4506  * intel_irq_uninstall - finilizes all irq handling
4507  * @dev_priv: i915 device instance
4508  *
4509  * This stops interrupt and hotplug handling and unregisters and frees all
4510  * resources acquired in the init functions.
4511  */
4512 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4513 {
4514         drm_irq_uninstall(dev_priv->dev);
4515         intel_hpd_cancel_work(dev_priv);
4516         dev_priv->pm.irqs_enabled = false;
4517 }
4518
4519 /**
4520  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4521  * @dev_priv: i915 device instance
4522  *
4523  * This function is used to disable interrupts at runtime, both in the runtime
4524  * pm and the system suspend/resume code.
4525  */
4526 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4527 {
4528         dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4529         dev_priv->pm.irqs_enabled = false;
4530         synchronize_irq(dev_priv->dev->irq);
4531 }
4532
4533 /**
4534  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4535  * @dev_priv: i915 device instance
4536  *
4537  * This function is used to enable interrupts at runtime, both in the runtime
4538  * pm and the system suspend/resume code.
4539  */
4540 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4541 {
4542         dev_priv->pm.irqs_enabled = true;
4543         dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4544         dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4545 }