drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <linux/circ_buf.h>
  34 #include <drm/drmP.h>
  35 #include <drm/i915_drm.h>
  36 #include "i915_drv.h"
  37 #include "i915_trace.h"
  38 #include "intel_drv.h"
  39
  40 /**
  41  * DOC: interrupt handling
  42  *
  43  * These functions provide the basic support for enabling and disabling the
  44  * interrupt handling support. There's a lot more functionality in i915_irq.c
  45  * and related files, but that will be described in separate chapters.
  46  */
  47
  48 static const u32 hpd_ibx[HPD_NUM_PINS] = {
  49         [HPD_CRT] = SDE_CRT_HOTPLUG,
  50         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  51         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  52         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  53         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  54 };
  55
  56 static const u32 hpd_cpt[HPD_NUM_PINS] = {
  57         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  58         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  59         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  60         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  61         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  62 };
  63
  64 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
  65         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  66         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  67         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  68         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  69         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  70         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  71 };
  72
  73 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
  74         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  75         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  76         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  77         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  78         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  79         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  80 };
  81
  82 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
  83         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  84         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  85         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  86         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  87         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  88         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  89 };
  90
  91 /* BXT hpd list */
  92 static const u32 hpd_bxt[HPD_NUM_PINS] = {
  93         [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
  94         [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
  95 };
  96
  97 /* IIR can theoretically queue up two events. Be paranoid. */
  98 #define GEN8_IRQ_RESET_NDX(type, which) do { \
  99         I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
 100         POSTING_READ(GEN8_##type##_IMR(which)); \
 101         I915_WRITE(GEN8_##type##_IER(which), 0); \
 102         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 103         POSTING_READ(GEN8_##type##_IIR(which)); \
 104         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 105         POSTING_READ(GEN8_##type##_IIR(which)); \
 106 } while (0)
 107
 108 #define GEN5_IRQ_RESET(type) do { \
 109         I915_WRITE(type##IMR, 0xffffffff); \
 110         POSTING_READ(type##IMR); \
 111         I915_WRITE(type##IER, 0); \
 112         I915_WRITE(type##IIR, 0xffffffff); \
 113         POSTING_READ(type##IIR); \
 114         I915_WRITE(type##IIR, 0xffffffff); \
 115         POSTING_READ(type##IIR); \
 116 } while (0)
 117
 118 /*
 119  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 120  */
 121 #define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
 122         u32 val = I915_READ(reg); \
 123         if (val) { \
 124                 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
 125                      (reg), val); \
 126                 I915_WRITE((reg), 0xffffffff); \
 127                 POSTING_READ(reg); \
 128                 I915_WRITE((reg), 0xffffffff); \
 129                 POSTING_READ(reg); \
 130         } \
 131 } while (0)
 132
 133 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
 134         GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
 135         I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
 136         I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
 137         POSTING_READ(GEN8_##type##_IMR(which)); \
 138 } while (0)
 139
 140 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
 141         GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
 142         I915_WRITE(type##IER, (ier_val)); \
 143         I915_WRITE(type##IMR, (imr_val)); \
 144         POSTING_READ(type##IMR); \
 145 } while (0)
 146
 147 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
 148
 149 /* For display hotplug interrupt */
 150 void
 151 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 152 {
 153         assert_spin_locked(&dev_priv->irq_lock);
 154
 155         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 156                 return;
 157
 158         if ((dev_priv->irq_mask & mask) != 0) {
 159                 dev_priv->irq_mask &= ~mask;
 160                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 161                 POSTING_READ(DEIMR);
 162         }
 163 }
 164
 165 void
 166 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 167 {
 168         assert_spin_locked(&dev_priv->irq_lock);
 169
 170         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 171                 return;
 172
 173         if ((dev_priv->irq_mask & mask) != mask) {
 174                 dev_priv->irq_mask |= mask;
 175                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 176                 POSTING_READ(DEIMR);
 177         }
 178 }
 179
 180 /**
 181  * ilk_update_gt_irq - update GTIMR
 182  * @dev_priv: driver private
 183  * @interrupt_mask: mask of interrupt bits to update
 184  * @enabled_irq_mask: mask of interrupt bits to enable
 185  */
 186 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 187                               uint32_t interrupt_mask,
 188                               uint32_t enabled_irq_mask)
 189 {
 190         assert_spin_locked(&dev_priv->irq_lock);
 191
 192         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 193
 194         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 195                 return;
 196
 197         dev_priv->gt_irq_mask &= ~interrupt_mask;
 198         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 199         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 200         POSTING_READ(GTIMR);
 201 }
 202
 203 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 204 {
 205         ilk_update_gt_irq(dev_priv, mask, mask);
 206 }
 207
 208 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 209 {
 210         ilk_update_gt_irq(dev_priv, mask, 0);
 211 }
 212
 213 static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
 214 {
 215         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 216 }
 217
 218 static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
 219 {
 220         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
 221 }
 222
 223 static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
 224 {
 225         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
 226 }
 227
 228 /**
 229   * snb_update_pm_irq - update GEN6_PMIMR
 230   * @dev_priv: driver private
 231   * @interrupt_mask: mask of interrupt bits to update
 232   * @enabled_irq_mask: mask of interrupt bits to enable
 233   */
 234 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 235                               uint32_t interrupt_mask,
 236                               uint32_t enabled_irq_mask)
 237 {
 238         uint32_t new_val;
 239
 240         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 241
 242         assert_spin_locked(&dev_priv->irq_lock);
 243
 244         new_val = dev_priv->pm_irq_mask;
 245         new_val &= ~interrupt_mask;
 246         new_val |= (~enabled_irq_mask & interrupt_mask);
 247
 248         if (new_val != dev_priv->pm_irq_mask) {
 249                 dev_priv->pm_irq_mask = new_val;
 250                 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
 251                 POSTING_READ(gen6_pm_imr(dev_priv));
 252         }
 253 }
 254
 255 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 256 {
 257         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 258                 return;
 259
 260         snb_update_pm_irq(dev_priv, mask, mask);
 261 }
 262
 263 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
 264                                   uint32_t mask)
 265 {
 266         snb_update_pm_irq(dev_priv, mask, 0);
 267 }
 268
 269 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 270 {
 271         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 272                 return;
 273
 274         __gen6_disable_pm_irq(dev_priv, mask);
 275 }
 276
 277 void gen6_reset_rps_interrupts(struct drm_device *dev)
 278 {
 279         struct drm_i915_private *dev_priv = dev->dev_private;
 280         uint32_t reg = gen6_pm_iir(dev_priv);
 281
 282         spin_lock_irq(&dev_priv->irq_lock);
 283         I915_WRITE(reg, dev_priv->pm_rps_events);
 284         I915_WRITE(reg, dev_priv->pm_rps_events);
 285         POSTING_READ(reg);
 286         dev_priv->rps.pm_iir = 0;
 287         spin_unlock_irq(&dev_priv->irq_lock);
 288 }
 289
 290 void gen6_enable_rps_interrupts(struct drm_device *dev)
 291 {
 292         struct drm_i915_private *dev_priv = dev->dev_private;
 293
 294         spin_lock_irq(&dev_priv->irq_lock);
 295
 296         WARN_ON(dev_priv->rps.pm_iir);
 297         WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
 298         dev_priv->rps.interrupts_enabled = true;
 299         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
 300                                 dev_priv->pm_rps_events);
 301         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 302
 303         spin_unlock_irq(&dev_priv->irq_lock);
 304 }
 305
 306 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 307 {
 308         /*
 309          * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
 310          * if GEN6_PM_UP_EI_EXPIRED is masked.
 311          *
 312          * TODO: verify if this can be reproduced on VLV,CHV.
 313          */
 314         if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
 315                 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
 316
 317         if (INTEL_INFO(dev_priv)->gen >= 8)
 318                 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
 319
 320         return mask;
 321 }
 322
 323 void gen6_disable_rps_interrupts(struct drm_device *dev)
 324 {
 325         struct drm_i915_private *dev_priv = dev->dev_private;
 326
 327         spin_lock_irq(&dev_priv->irq_lock);
 328         dev_priv->rps.interrupts_enabled = false;
 329         spin_unlock_irq(&dev_priv->irq_lock);
 330
 331         cancel_work_sync(&dev_priv->rps.work);
 332
 333         spin_lock_irq(&dev_priv->irq_lock);
 334
 335         I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 336
 337         __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 338         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
 339                                 ~dev_priv->pm_rps_events);
 340
 341         spin_unlock_irq(&dev_priv->irq_lock);
 342
 343         synchronize_irq(dev->irq);
 344 }
 345
 346 /**
 347  * ibx_display_interrupt_update - update SDEIMR
 348  * @dev_priv: driver private
 349  * @interrupt_mask: mask of interrupt bits to update
 350  * @enabled_irq_mask: mask of interrupt bits to enable
 351  */
 352 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 353                                   uint32_t interrupt_mask,
 354                                   uint32_t enabled_irq_mask)
 355 {
 356         uint32_t sdeimr = I915_READ(SDEIMR);
 357         sdeimr &= ~interrupt_mask;
 358         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 359
 360         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 361
 362         assert_spin_locked(&dev_priv->irq_lock);
 363
 364         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 365                 return;
 366
 367         I915_WRITE(SDEIMR, sdeimr);
 368         POSTING_READ(SDEIMR);
 369 }
 370
 371 static void
 372 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 373                        u32 enable_mask, u32 status_mask)
 374 {
 375         u32 reg = PIPESTAT(pipe);
 376         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 377
 378         assert_spin_locked(&dev_priv->irq_lock);
 379         WARN_ON(!intel_irqs_enabled(dev_priv));
 380
 381         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 382                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 383                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 384                       pipe_name(pipe), enable_mask, status_mask))
 385                 return;
 386
 387         if ((pipestat & enable_mask) == enable_mask)
 388                 return;
 389
 390         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
 391
 392         /* Enable the interrupt, clear any pending status */
 393         pipestat |= enable_mask | status_mask;
 394         I915_WRITE(reg, pipestat);
 395         POSTING_READ(reg);
 396 }
 397
 398 static void
 399 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 400                         u32 enable_mask, u32 status_mask)
 401 {
 402         u32 reg = PIPESTAT(pipe);
 403         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 404
 405         assert_spin_locked(&dev_priv->irq_lock);
 406         WARN_ON(!intel_irqs_enabled(dev_priv));
 407
 408         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 409                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 410                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 411                       pipe_name(pipe), enable_mask, status_mask))
 412                 return;
 413
 414         if ((pipestat & enable_mask) == 0)
 415                 return;
 416
 417         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
 418
 419         pipestat &= ~enable_mask;
 420         I915_WRITE(reg, pipestat);
 421         POSTING_READ(reg);
 422 }
 423
 424 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
 425 {
 426         u32 enable_mask = status_mask << 16;
 427
 428         /*
 429          * On pipe A we don't support the PSR interrupt yet,
 430          * on pipe B and C the same bit MBZ.
 431          */
 432         if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
 433                 return 0;
 434         /*
 435          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 436          * A the same bit is for perf counters which we don't use either.
 437          */
 438         if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
 439                 return 0;
 440
 441         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 442                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 443                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 444         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 445                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 446         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 447                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 448
 449         return enable_mask;
 450 }
 451
 452 void
 453 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 454                      u32 status_mask)
 455 {
 456         u32 enable_mask;
 457
 458         if (IS_VALLEYVIEW(dev_priv->dev))
 459                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 460                                                            status_mask);
 461         else
 462                 enable_mask = status_mask << 16;
 463         __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 464 }
 465
 466 void
 467 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 468                       u32 status_mask)
 469 {
 470         u32 enable_mask;
 471
 472         if (IS_VALLEYVIEW(dev_priv->dev))
 473                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 474                                                            status_mask);
 475         else
 476                 enable_mask = status_mask << 16;
 477         __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 478 }
 479
 480 /**
 481  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 482  */
 483 static void i915_enable_asle_pipestat(struct drm_device *dev)
 484 {
 485         struct drm_i915_private *dev_priv = dev->dev_private;
 486
 487         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 488                 return;
 489
 490         spin_lock_irq(&dev_priv->irq_lock);
 491
 492         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 493         if (INTEL_INFO(dev)->gen >= 4)
 494                 i915_enable_pipestat(dev_priv, PIPE_A,
 495                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 496
 497         spin_unlock_irq(&dev_priv->irq_lock);
 498 }
 499
 500 /*
 501  * This timing diagram depicts the video signal in and
 502  * around the vertical blanking period.
 503  *
 504  * Assumptions about the fictitious mode used in this example:
 505  *  vblank_start >= 3
 506  *  vsync_start = vblank_start + 1
 507  *  vsync_end = vblank_start + 2
 508  *  vtotal = vblank_start + 3
 509  *
 510  *           start of vblank:
 511  *           latch double buffered registers
 512  *           increment frame counter (ctg+)
 513  *           generate start of vblank interrupt (gen4+)
 514  *           |
 515  *           |          frame start:
 516  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 517  *           |          may be shifted forward 1-3 extra lines via PIPECONF
 518  *           |          |
 519  *           |          |  start of vsync:
 520  *           |          |  generate vsync interrupt
 521  *           |          |  |
 522  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 523  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 524  * ----va---> <-----------------vb--------------------> <--------va-------------
 525  *       |          |       <----vs----->                     |
 526  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 527  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 528  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 529  *       |          |                                         |
 530  *       last visible pixel                                   first visible pixel
 531  *                  |                                         increment frame counter (gen3/4)
 532  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 533  *
 534  * x  = horizontal active
 535  * _  = horizontal blanking
 536  * hs = horizontal sync
 537  * va = vertical active
 538  * vb = vertical blanking
 539  * vs = vertical sync
 540  * vbs = vblank_start (number)
 541  *
 542  * Summary:
 543  * - most events happen at the start of horizontal sync
 544  * - frame start happens at the start of horizontal blank, 1-4 lines
 545  *   (depending on PIPECONF settings) after the start of vblank
 546  * - gen3/4 pixel and frame counter are synchronized with the start
 547  *   of horizontal active on the first line of vertical active
 548  */
 549
 550 static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
 551 {
 552         /* Gen2 doesn't have a hardware frame counter */
 553         return 0;
 554 }
 555
 556 /* Called from drm generic code, passed a 'crtc', which
 557  * we use as a pipe index
 558  */
 559 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 560 {
 561         struct drm_i915_private *dev_priv = dev->dev_private;
 562         unsigned long high_frame;
 563         unsigned long low_frame;
 564         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 565         struct intel_crtc *intel_crtc =
 566                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 567         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 568
 569         htotal = mode->crtc_htotal;
 570         hsync_start = mode->crtc_hsync_start;
 571         vbl_start = mode->crtc_vblank_start;
 572         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 573                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 574
 575         /* Convert to pixel count */
 576         vbl_start *= htotal;
 577
 578         /* Start of vblank event occurs at start of hsync */
 579         vbl_start -= htotal - hsync_start;
 580
 581         high_frame = PIPEFRAME(pipe);
 582         low_frame = PIPEFRAMEPIXEL(pipe);
 583
 584         /*
 585          * High & low register fields aren't synchronized, so make sure
 586          * we get a low value that's stable across two reads of the high
 587          * register.
 588          */
 589         do {
 590                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 591                 low   = I915_READ(low_frame);
 592                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 593         } while (high1 != high2);
 594
 595         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 596         pixel = low & PIPE_PIXEL_MASK;
 597         low >>= PIPE_FRAME_LOW_SHIFT;
 598
 599         /*
 600          * The frame counter increments at beginning of active.
 601          * Cook up a vblank counter by also checking the pixel
 602          * counter against vblank start.
 603          */
 604         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
 605 }
 606
 607 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 608 {
 609         struct drm_i915_private *dev_priv = dev->dev_private;
 610         int reg = PIPE_FRMCOUNT_GM45(pipe);
 611
 612         return I915_READ(reg);
 613 }
 614
 615 /* raw reads, only for fast reads of display block, no need for forcewake etc. */
 616 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
 617
 618 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 619 {
 620         struct drm_device *dev = crtc->base.dev;
 621         struct drm_i915_private *dev_priv = dev->dev_private;
 622         const struct drm_display_mode *mode = &crtc->base.hwmode;
 623         enum pipe pipe = crtc->pipe;
 624         int position, vtotal;
 625
 626         vtotal = mode->crtc_vtotal;
 627         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 628                 vtotal /= 2;
 629
 630         if (IS_GEN2(dev))
 631                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 632         else
 633                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 634
 635         /*
 636          * See update_scanline_offset() for the details on the
 637          * scanline_offset adjustment.
 638          */
 639         return (position + crtc->scanline_offset) % vtotal;
 640 }
 641
 642 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 643                                     unsigned int flags, int *vpos, int *hpos,
 644                                     ktime_t *stime, ktime_t *etime)
 645 {
 646         struct drm_i915_private *dev_priv = dev->dev_private;
 647         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 648         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 649         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 650         int position;
 651         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 652         bool in_vbl = true;
 653         int ret = 0;
 654         unsigned long irqflags;
 655
 656         if (WARN_ON(!mode->crtc_clock)) {
 657                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 658                                  "pipe %c\n", pipe_name(pipe));
 659                 return 0;
 660         }
 661
 662         htotal = mode->crtc_htotal;
 663         hsync_start = mode->crtc_hsync_start;
 664         vtotal = mode->crtc_vtotal;
 665         vbl_start = mode->crtc_vblank_start;
 666         vbl_end = mode->crtc_vblank_end;
 667
 668         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 669                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 670                 vbl_end /= 2;
 671                 vtotal /= 2;
 672         }
 673
 674         ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 675
 676         /*
 677          * Lock uncore.lock, as we will do multiple timing critical raw
 678          * register reads, potentially with preemption disabled, so the
 679          * following code must not block on uncore.lock.
 680          */
 681         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 682
 683         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 684
 685         /* Get optional system timestamp before query. */
 686         if (stime)
 687                 *stime = ktime_get();
 688
 689         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 690                 /* No obvious pixelcount register. Only query vertical
 691                  * scanout position from Display scan line register.
 692                  */
 693                 position = __intel_get_crtc_scanline(intel_crtc);
 694         } else {
 695                 /* Have access to pixelcount since start of frame.
 696                  * We can split this into vertical and horizontal
 697                  * scanout position.
 698                  */
 699                 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 700
 701                 /* convert to pixel counts */
 702                 vbl_start *= htotal;
 703                 vbl_end *= htotal;
 704                 vtotal *= htotal;
 705
 706                 /*
 707                  * In interlaced modes, the pixel counter counts all pixels,
 708                  * so one field will have htotal more pixels. In order to avoid
 709                  * the reported position from jumping backwards when the pixel
 710                  * counter is beyond the length of the shorter field, just
 711                  * clamp the position the length of the shorter field. This
 712                  * matches how the scanline counter based position works since
 713                  * the scanline counter doesn't count the two half lines.
 714                  */
 715                 if (position >= vtotal)
 716                         position = vtotal - 1;
 717
 718                 /*
 719                  * Start of vblank interrupt is triggered at start of hsync,
 720                  * just prior to the first active line of vblank. However we
 721                  * consider lines to start at the leading edge of horizontal
 722                  * active. So, should we get here before we've crossed into
 723                  * the horizontal active of the first line in vblank, we would
 724                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
 725                  * always add htotal-hsync_start to the current pixel position.
 726                  */
 727                 position = (position + htotal - hsync_start) % vtotal;
 728         }
 729
 730         /* Get optional system timestamp after query. */
 731         if (etime)
 732                 *etime = ktime_get();
 733
 734         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 735
 736         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 737
 738         in_vbl = position >= vbl_start && position < vbl_end;
 739
 740         /*
 741          * While in vblank, position will be negative
 742          * counting up towards 0 at vbl_end. And outside
 743          * vblank, position will be positive counting
 744          * up since vbl_end.
 745          */
 746         if (position >= vbl_start)
 747                 position -= vbl_end;
 748         else
 749                 position += vtotal - vbl_end;
 750
 751         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 752                 *vpos = position;
 753                 *hpos = 0;
 754         } else {
 755                 *vpos = position / htotal;
 756                 *hpos = position - (*vpos * htotal);
 757         }
 758
 759         /* In vblank? */
 760         if (in_vbl)
 761                 ret |= DRM_SCANOUTPOS_IN_VBLANK;
 762
 763         return ret;
 764 }
 765
 766 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 767 {
 768         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 769         unsigned long irqflags;
 770         int position;
 771
 772         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 773         position = __intel_get_crtc_scanline(crtc);
 774         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 775
 776         return position;
 777 }
 778
 779 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 780                               int *max_error,
 781                               struct timeval *vblank_time,
 782                               unsigned flags)
 783 {
 784         struct drm_crtc *crtc;
 785
 786         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 787                 DRM_ERROR("Invalid crtc %d\n", pipe);
 788                 return -EINVAL;
 789         }
 790
 791         /* Get drm_crtc to timestamp: */
 792         crtc = intel_get_crtc_for_pipe(dev, pipe);
 793         if (crtc == NULL) {
 794                 DRM_ERROR("Invalid crtc %d\n", pipe);
 795                 return -EINVAL;
 796         }
 797
 798         if (!crtc->hwmode.crtc_clock) {
 799                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 800                 return -EBUSY;
 801         }
 802
 803         /* Helper routine in DRM core does all the work: */
 804         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 805                                                      vblank_time, flags,
 806                                                      crtc,
 807                                                      &crtc->hwmode);
 808 }
 809
 810 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 811 {
 812         struct drm_i915_private *dev_priv = dev->dev_private;
 813         u32 busy_up, busy_down, max_avg, min_avg;
 814         u8 new_delay;
 815
 816         spin_lock(&mchdev_lock);
 817
 818         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 819
 820         new_delay = dev_priv->ips.cur_delay;
 821
 822         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 823         busy_up = I915_READ(RCPREVBSYTUPAVG);
 824         busy_down = I915_READ(RCPREVBSYTDNAVG);
 825         max_avg = I915_READ(RCBMAXAVG);
 826         min_avg = I915_READ(RCBMINAVG);
 827
 828         /* Handle RCS change request from hw */
 829         if (busy_up > max_avg) {
 830                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 831                         new_delay = dev_priv->ips.cur_delay - 1;
 832                 if (new_delay < dev_priv->ips.max_delay)
 833                         new_delay = dev_priv->ips.max_delay;
 834         } else if (busy_down < min_avg) {
 835                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 836                         new_delay = dev_priv->ips.cur_delay + 1;
 837                 if (new_delay > dev_priv->ips.min_delay)
 838                         new_delay = dev_priv->ips.min_delay;
 839         }
 840
 841         if (ironlake_set_drps(dev, new_delay))
 842                 dev_priv->ips.cur_delay = new_delay;
 843
 844         spin_unlock(&mchdev_lock);
 845
 846         return;
 847 }
 848
 849 static void notify_ring(struct intel_engine_cs *ring)
 850 {
 851         if (!intel_ring_initialized(ring))
 852                 return;
 853
 854         trace_i915_gem_request_notify(ring);
 855
 856         wake_up_all(&ring->irq_queue);
 857 }
 858
 859 static void vlv_c0_read(struct drm_i915_private *dev_priv,
 860                         struct intel_rps_ei *ei)
 861 {
 862         ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
 863         ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
 864         ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
 865 }
 866
 867 static bool vlv_c0_above(struct drm_i915_private *dev_priv,
 868                          const struct intel_rps_ei *old,
 869                          const struct intel_rps_ei *now,
 870                          int threshold)
 871 {
 872         u64 time, c0;
 873
 874         if (old->cz_clock == 0)
 875                 return false;
 876
 877         time = now->cz_clock - old->cz_clock;
 878         time *= threshold * dev_priv->mem_freq;
 879
 880         /* Workload can be split between render + media, e.g. SwapBuffers
 881          * being blitted in X after being rendered in mesa. To account for
 882          * this we need to combine both engines into our activity counter.
 883          */
 884         c0 = now->render_c0 - old->render_c0;
 885         c0 += now->media_c0 - old->media_c0;
 886         c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000;
 887
 888         return c0 >= time;
 889 }
 890
 891 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
 892 {
 893         vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
 894         dev_priv->rps.up_ei = dev_priv->rps.down_ei;
 895 }
 896
 897 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
 898 {
 899         struct intel_rps_ei now;
 900         u32 events = 0;
 901
 902         if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
 903                 return 0;
 904
 905         vlv_c0_read(dev_priv, &now);
 906         if (now.cz_clock == 0)
 907                 return 0;
 908
 909         if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
 910                 if (!vlv_c0_above(dev_priv,
 911                                   &dev_priv->rps.down_ei, &now,
 912                                   dev_priv->rps.down_threshold))
 913                         events |= GEN6_PM_RP_DOWN_THRESHOLD;
 914                 dev_priv->rps.down_ei = now;
 915         }
 916
 917         if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
 918                 if (vlv_c0_above(dev_priv,
 919                                  &dev_priv->rps.up_ei, &now,
 920                                  dev_priv->rps.up_threshold))
 921                         events |= GEN6_PM_RP_UP_THRESHOLD;
 922                 dev_priv->rps.up_ei = now;
 923         }
 924
 925         return events;
 926 }
 927
 928 static bool any_waiters(struct drm_i915_private *dev_priv)
 929 {
 930         struct intel_engine_cs *ring;
 931         int i;
 932
 933         for_each_ring(ring, dev_priv, i)
 934                 if (ring->irq_refcount)
 935                         return true;
 936
 937         return false;
 938 }
 939
 940 static void gen6_pm_rps_work(struct work_struct *work)
 941 {
 942         struct drm_i915_private *dev_priv =
 943                 container_of(work, struct drm_i915_private, rps.work);
 944         bool client_boost;
 945         int new_delay, adj, min, max;
 946         u32 pm_iir;
 947
 948         spin_lock_irq(&dev_priv->irq_lock);
 949         /* Speed up work cancelation during disabling rps interrupts. */
 950         if (!dev_priv->rps.interrupts_enabled) {
 951                 spin_unlock_irq(&dev_priv->irq_lock);
 952                 return;
 953         }
 954         pm_iir = dev_priv->rps.pm_iir;
 955         dev_priv->rps.pm_iir = 0;
 956         /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
 957         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 958         client_boost = dev_priv->rps.client_boost;
 959         dev_priv->rps.client_boost = false;
 960         spin_unlock_irq(&dev_priv->irq_lock);
 961
 962         /* Make sure we didn't queue anything we're not going to process. */
 963         WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
 964
 965         if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
 966                 return;
 967
 968         mutex_lock(&dev_priv->rps.hw_lock);
 969
 970         pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
 971
 972         adj = dev_priv->rps.last_adj;
 973         new_delay = dev_priv->rps.cur_freq;
 974         min = dev_priv->rps.min_freq_softlimit;
 975         max = dev_priv->rps.max_freq_softlimit;
 976
 977         if (client_boost) {
 978                 new_delay = dev_priv->rps.max_freq_softlimit;
 979                 adj = 0;
 980         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 981                 if (adj > 0)
 982                         adj *= 2;
 983                 else /* CHV needs even encode values */
 984                         adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
 985                 /*
 986                  * For better performance, jump directly
 987                  * to RPe if we're below it.
 988                  */
 989                 if (new_delay < dev_priv->rps.efficient_freq - adj) {
 990                         new_delay = dev_priv->rps.efficient_freq;
 991                         adj = 0;
 992                 }
 993         } else if (any_waiters(dev_priv)) {
 994                 adj = 0;
 995         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
 996                 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
 997                         new_delay = dev_priv->rps.efficient_freq;
 998                 else
 999                         new_delay = dev_priv->rps.min_freq_softlimit;
1000                 adj = 0;
1001         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1002                 if (adj < 0)
1003                         adj *= 2;
1004                 else /* CHV needs even encode values */
1005                         adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1006         } else { /* unknown event */
1007                 adj = 0;
1008         }
1009
1010         dev_priv->rps.last_adj = adj;
1011
1012         /* sysfs frequency interfaces may have snuck in while servicing the
1013          * interrupt
1014          */
1015         new_delay += adj;
1016         new_delay = clamp_t(int, new_delay, min, max);
1017
1018         intel_set_rps(dev_priv->dev, new_delay);
1019
1020         mutex_unlock(&dev_priv->rps.hw_lock);
1021 }
1022
1023
1024 /**
1025  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1026  * occurred.
1027  * @work: workqueue struct
1028  *
1029  * Doesn't actually do anything except notify userspace. As a consequence of
1030  * this event, userspace should try to remap the bad rows since statistically
1031  * it is likely the same row is more likely to go bad again.
1032  */
1033 static void ivybridge_parity_work(struct work_struct *work)
1034 {
1035         struct drm_i915_private *dev_priv =
1036                 container_of(work, struct drm_i915_private, l3_parity.error_work);
1037         u32 error_status, row, bank, subbank;
1038         char *parity_event[6];
1039         uint32_t misccpctl;
1040         uint8_t slice = 0;
1041
1042         /* We must turn off DOP level clock gating to access the L3 registers.
1043          * In order to prevent a get/put style interface, acquire struct mutex
1044          * any time we access those registers.
1045          */
1046         mutex_lock(&dev_priv->dev->struct_mutex);
1047
1048         /* If we've screwed up tracking, just let the interrupt fire again */
1049         if (WARN_ON(!dev_priv->l3_parity.which_slice))
1050                 goto out;
1051
1052         misccpctl = I915_READ(GEN7_MISCCPCTL);
1053         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1054         POSTING_READ(GEN7_MISCCPCTL);
1055
1056         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1057                 u32 reg;
1058
1059                 slice--;
1060                 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1061                         break;
1062
1063                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1064
1065                 reg = GEN7_L3CDERRST1 + (slice * 0x200);
1066
1067                 error_status = I915_READ(reg);
1068                 row = GEN7_PARITY_ERROR_ROW(error_status);
1069                 bank = GEN7_PARITY_ERROR_BANK(error_status);
1070                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1071
1072                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1073                 POSTING_READ(reg);
1074
1075                 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1076                 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1077                 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1078                 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1079                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1080                 parity_event[5] = NULL;
1081
1082                 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1083                                    KOBJ_CHANGE, parity_event);
1084
1085                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1086                           slice, row, bank, subbank);
1087
1088                 kfree(parity_event[4]);
1089                 kfree(parity_event[3]);
1090                 kfree(parity_event[2]);
1091                 kfree(parity_event[1]);
1092         }
1093
1094         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1095
1096 out:
1097         WARN_ON(dev_priv->l3_parity.which_slice);
1098         spin_lock_irq(&dev_priv->irq_lock);
1099         gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1100         spin_unlock_irq(&dev_priv->irq_lock);
1101
1102         mutex_unlock(&dev_priv->dev->struct_mutex);
1103 }
1104
1105 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1106 {
1107         struct drm_i915_private *dev_priv = dev->dev_private;
1108
1109         if (!HAS_L3_DPF(dev))
1110                 return;
1111
1112         spin_lock(&dev_priv->irq_lock);
1113         gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1114         spin_unlock(&dev_priv->irq_lock);
1115
1116         iir &= GT_PARITY_ERROR(dev);
1117         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1118                 dev_priv->l3_parity.which_slice |= 1 << 1;
1119
1120         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1121                 dev_priv->l3_parity.which_slice |= 1 << 0;
1122
1123         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1124 }
1125
1126 static void ilk_gt_irq_handler(struct drm_device *dev,
1127                                struct drm_i915_private *dev_priv,
1128                                u32 gt_iir)
1129 {
1130         if (gt_iir &
1131             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1132                 notify_ring(&dev_priv->ring[RCS]);
1133         if (gt_iir & ILK_BSD_USER_INTERRUPT)
1134                 notify_ring(&dev_priv->ring[VCS]);
1135 }
1136
1137 static void snb_gt_irq_handler(struct drm_device *dev,
1138                                struct drm_i915_private *dev_priv,
1139                                u32 gt_iir)
1140 {
1141
1142         if (gt_iir &
1143             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1144                 notify_ring(&dev_priv->ring[RCS]);
1145         if (gt_iir & GT_BSD_USER_INTERRUPT)
1146                 notify_ring(&dev_priv->ring[VCS]);
1147         if (gt_iir & GT_BLT_USER_INTERRUPT)
1148                 notify_ring(&dev_priv->ring[BCS]);
1149
1150         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1151                       GT_BSD_CS_ERROR_INTERRUPT |
1152                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1153                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1154
1155         if (gt_iir & GT_PARITY_ERROR(dev))
1156                 ivybridge_parity_error_irq_handler(dev, gt_iir);
1157 }
1158
1159 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1160                                        u32 master_ctl)
1161 {
1162         irqreturn_t ret = IRQ_NONE;
1163
1164         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1165                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1166                 if (tmp) {
1167                         I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1168                         ret = IRQ_HANDLED;
1169
1170                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1171                                 intel_lrc_irq_handler(&dev_priv->ring[RCS]);
1172                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1173                                 notify_ring(&dev_priv->ring[RCS]);
1174
1175                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1176                                 intel_lrc_irq_handler(&dev_priv->ring[BCS]);
1177                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1178                                 notify_ring(&dev_priv->ring[BCS]);
1179                 } else
1180                         DRM_ERROR("The master control interrupt lied (GT0)!\n");
1181         }
1182
1183         if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1184                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1185                 if (tmp) {
1186                         I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1187                         ret = IRQ_HANDLED;
1188
1189                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1190                                 intel_lrc_irq_handler(&dev_priv->ring[VCS]);
1191                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1192                                 notify_ring(&dev_priv->ring[VCS]);
1193
1194                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1195                                 intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
1196                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1197                                 notify_ring(&dev_priv->ring[VCS2]);
1198                 } else
1199                         DRM_ERROR("The master control interrupt lied (GT1)!\n");
1200         }
1201
1202         if (master_ctl & GEN8_GT_VECS_IRQ) {
1203                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1204                 if (tmp) {
1205                         I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1206                         ret = IRQ_HANDLED;
1207
1208                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1209                                 intel_lrc_irq_handler(&dev_priv->ring[VECS]);
1210                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1211                                 notify_ring(&dev_priv->ring[VECS]);
1212                 } else
1213                         DRM_ERROR("The master control interrupt lied (GT3)!\n");
1214         }
1215
1216         if (master_ctl & GEN8_GT_PM_IRQ) {
1217                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1218                 if (tmp & dev_priv->pm_rps_events) {
1219                         I915_WRITE_FW(GEN8_GT_IIR(2),
1220                                       tmp & dev_priv->pm_rps_events);
1221                         ret = IRQ_HANDLED;
1222                         gen6_rps_irq_handler(dev_priv, tmp);
1223                 } else
1224                         DRM_ERROR("The master control interrupt lied (PM)!\n");
1225         }
1226
1227         return ret;
1228 }
1229
1230 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1231 {
1232         switch (port) {
1233         case PORT_B:
1234                 return val & PORTB_HOTPLUG_LONG_DETECT;
1235         case PORT_C:
1236                 return val & PORTC_HOTPLUG_LONG_DETECT;
1237         case PORT_D:
1238                 return val & PORTD_HOTPLUG_LONG_DETECT;
1239         default:
1240                 return false;
1241         }
1242 }
1243
1244 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1245 {
1246         switch (port) {
1247         case PORT_B:
1248                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1249         case PORT_C:
1250                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1251         case PORT_D:
1252                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1253         default:
1254                 return false;
1255         }
1256 }
1257
1258 /* Get a bit mask of pins that have triggered, and which ones may be long. */
1259 static void pch_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1260                              u32 hotplug_trigger, u32 dig_hotplug_reg,
1261                              const u32 hpd[HPD_NUM_PINS])
1262 {
1263         enum port port;
1264         int i;
1265
1266         *pin_mask = 0;
1267         *long_mask = 0;
1268
1269         for_each_hpd_pin(i) {
1270                 if ((hpd[i] & hotplug_trigger) == 0)
1271                         continue;
1272
1273                 *pin_mask |= BIT(i);
1274
1275                 port = intel_hpd_pin_to_port(i);
1276                 if (pch_port_hotplug_long_detect(port, dig_hotplug_reg))
1277                         *long_mask |= BIT(i);
1278         }
1279
1280         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1281                          hotplug_trigger, dig_hotplug_reg, *pin_mask);
1282
1283 }
1284
1285 /* Get a bit mask of pins that have triggered, and which ones may be long. */
1286 static void i9xx_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1287                               u32 hotplug_trigger, const u32 hpd[HPD_NUM_PINS])
1288 {
1289         enum port port;
1290         int i;
1291
1292         *pin_mask = 0;
1293         *long_mask = 0;
1294
1295         if (!hotplug_trigger)
1296                 return;
1297
1298         for_each_hpd_pin(i) {
1299                 if ((hpd[i] & hotplug_trigger) == 0)
1300                         continue;
1301
1302                 *pin_mask |= BIT(i);
1303
1304                 port = intel_hpd_pin_to_port(i);
1305                 if (i9xx_port_hotplug_long_detect(port, hotplug_trigger))
1306                         *long_mask |= BIT(i);
1307         }
1308
1309         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, pins 0x%08x\n",
1310                          hotplug_trigger, *pin_mask);
1311 }
1312
1313 static void gmbus_irq_handler(struct drm_device *dev)
1314 {
1315         struct drm_i915_private *dev_priv = dev->dev_private;
1316
1317         wake_up_all(&dev_priv->gmbus_wait_queue);
1318 }
1319
1320 static void dp_aux_irq_handler(struct drm_device *dev)
1321 {
1322         struct drm_i915_private *dev_priv = dev->dev_private;
1323
1324         wake_up_all(&dev_priv->gmbus_wait_queue);
1325 }
1326
1327 #if defined(CONFIG_DEBUG_FS)
1328 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1329                                          uint32_t crc0, uint32_t crc1,
1330                                          uint32_t crc2, uint32_t crc3,
1331                                          uint32_t crc4)
1332 {
1333         struct drm_i915_private *dev_priv = dev->dev_private;
1334         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1335         struct intel_pipe_crc_entry *entry;
1336         int head, tail;
1337
1338         spin_lock(&pipe_crc->lock);
1339
1340         if (!pipe_crc->entries) {
1341                 spin_unlock(&pipe_crc->lock);
1342                 DRM_DEBUG_KMS("spurious interrupt\n");
1343                 return;
1344         }
1345
1346         head = pipe_crc->head;
1347         tail = pipe_crc->tail;
1348
1349         if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1350                 spin_unlock(&pipe_crc->lock);
1351                 DRM_ERROR("CRC buffer overflowing\n");
1352                 return;
1353         }
1354
1355         entry = &pipe_crc->entries[head];
1356
1357         entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1358         entry->crc[0] = crc0;
1359         entry->crc[1] = crc1;
1360         entry->crc[2] = crc2;
1361         entry->crc[3] = crc3;
1362         entry->crc[4] = crc4;
1363
1364         head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1365         pipe_crc->head = head;
1366
1367         spin_unlock(&pipe_crc->lock);
1368
1369         wake_up_interruptible(&pipe_crc->wq);
1370 }
1371 #else
1372 static inline void
1373 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1374                              uint32_t crc0, uint32_t crc1,
1375                              uint32_t crc2, uint32_t crc3,
1376                              uint32_t crc4) {}
1377 #endif
1378
1379
1380 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1381 {
1382         struct drm_i915_private *dev_priv = dev->dev_private;
1383
1384         display_pipe_crc_irq_handler(dev, pipe,
1385                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1386                                      0, 0, 0, 0);
1387 }
1388
1389 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1390 {
1391         struct drm_i915_private *dev_priv = dev->dev_private;
1392
1393         display_pipe_crc_irq_handler(dev, pipe,
1394                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1395                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1396                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1397                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1398                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1399 }
1400
1401 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1402 {
1403         struct drm_i915_private *dev_priv = dev->dev_private;
1404         uint32_t res1, res2;
1405
1406         if (INTEL_INFO(dev)->gen >= 3)
1407                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1408         else
1409                 res1 = 0;
1410
1411         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1412                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1413         else
1414                 res2 = 0;
1415
1416         display_pipe_crc_irq_handler(dev, pipe,
1417                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
1418                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1419                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1420                                      res1, res2);
1421 }
1422
1423 /* The RPS events need forcewake, so we add them to a work queue and mask their
1424  * IMR bits until the work is done. Other interrupts can be processed without
1425  * the work queue. */
1426 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1427 {
1428         if (pm_iir & dev_priv->pm_rps_events) {
1429                 spin_lock(&dev_priv->irq_lock);
1430                 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1431                 if (dev_priv->rps.interrupts_enabled) {
1432                         dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1433                         queue_work(dev_priv->wq, &dev_priv->rps.work);
1434                 }
1435                 spin_unlock(&dev_priv->irq_lock);
1436         }
1437
1438         if (INTEL_INFO(dev_priv)->gen >= 8)
1439                 return;
1440
1441         if (HAS_VEBOX(dev_priv->dev)) {
1442                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1443                         notify_ring(&dev_priv->ring[VECS]);
1444
1445                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1446                         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1447         }
1448 }
1449
1450 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1451 {
1452         if (!drm_handle_vblank(dev, pipe))
1453                 return false;
1454
1455         return true;
1456 }
1457
1458 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1459 {
1460         struct drm_i915_private *dev_priv = dev->dev_private;
1461         u32 pipe_stats[I915_MAX_PIPES] = { };
1462         int pipe;
1463
1464         spin_lock(&dev_priv->irq_lock);
1465         for_each_pipe(dev_priv, pipe) {
1466                 int reg;
1467                 u32 mask, iir_bit = 0;
1468
1469                 /*
1470                  * PIPESTAT bits get signalled even when the interrupt is
1471                  * disabled with the mask bits, and some of the status bits do
1472                  * not generate interrupts at all (like the underrun bit). Hence
1473                  * we need to be careful that we only handle what we want to
1474                  * handle.
1475                  */
1476
1477                 /* fifo underruns are filterered in the underrun handler. */
1478                 mask = PIPE_FIFO_UNDERRUN_STATUS;
1479
1480                 switch (pipe) {
1481                 case PIPE_A:
1482                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1483                         break;
1484                 case PIPE_B:
1485                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1486                         break;
1487                 case PIPE_C:
1488                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1489                         break;
1490                 }
1491                 if (iir & iir_bit)
1492                         mask |= dev_priv->pipestat_irq_mask[pipe];
1493
1494                 if (!mask)
1495                         continue;
1496
1497                 reg = PIPESTAT(pipe);
1498                 mask |= PIPESTAT_INT_ENABLE_MASK;
1499                 pipe_stats[pipe] = I915_READ(reg) & mask;
1500
1501                 /*
1502                  * Clear the PIPE*STAT regs before the IIR
1503                  */
1504                 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1505                                         PIPESTAT_INT_STATUS_MASK))
1506                         I915_WRITE(reg, pipe_stats[pipe]);
1507         }
1508         spin_unlock(&dev_priv->irq_lock);
1509
1510         for_each_pipe(dev_priv, pipe) {
1511                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1512                     intel_pipe_handle_vblank(dev, pipe))
1513                         intel_check_page_flip(dev, pipe);
1514
1515                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1516                         intel_prepare_page_flip(dev, pipe);
1517                         intel_finish_page_flip(dev, pipe);
1518                 }
1519
1520                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1521                         i9xx_pipe_crc_irq_handler(dev, pipe);
1522
1523                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1524                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1525         }
1526
1527         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1528                 gmbus_irq_handler(dev);
1529 }
1530
1531 static void i9xx_hpd_irq_handler(struct drm_device *dev)
1532 {
1533         struct drm_i915_private *dev_priv = dev->dev_private;
1534         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1535         u32 pin_mask, long_mask;
1536
1537         if (!hotplug_status)
1538                 return;
1539
1540         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1541         /*
1542          * Make sure hotplug status is cleared before we clear IIR, or else we
1543          * may miss hotplug events.
1544          */
1545         POSTING_READ(PORT_HOTPLUG_STAT);
1546
1547         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1548                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1549
1550                 i9xx_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, hpd_status_g4x);
1551                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1552
1553                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1554                         dp_aux_irq_handler(dev);
1555         } else {
1556                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1557
1558                 i9xx_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, hpd_status_i915);
1559                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1560         }
1561 }
1562
1563 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1564 {
1565         struct drm_device *dev = arg;
1566         struct drm_i915_private *dev_priv = dev->dev_private;
1567         u32 iir, gt_iir, pm_iir;
1568         irqreturn_t ret = IRQ_NONE;
1569
1570         if (!intel_irqs_enabled(dev_priv))
1571                 return IRQ_NONE;
1572
1573         while (true) {
1574                 /* Find, clear, then process each source of interrupt */
1575
1576                 gt_iir = I915_READ(GTIIR);
1577                 if (gt_iir)
1578                         I915_WRITE(GTIIR, gt_iir);
1579
1580                 pm_iir = I915_READ(GEN6_PMIIR);
1581                 if (pm_iir)
1582                         I915_WRITE(GEN6_PMIIR, pm_iir);
1583
1584                 iir = I915_READ(VLV_IIR);
1585                 if (iir) {
1586                         /* Consume port before clearing IIR or we'll miss events */
1587                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1588                                 i9xx_hpd_irq_handler(dev);
1589                         I915_WRITE(VLV_IIR, iir);
1590                 }
1591
1592                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1593                         goto out;
1594
1595                 ret = IRQ_HANDLED;
1596
1597                 if (gt_iir)
1598                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1599                 if (pm_iir)
1600                         gen6_rps_irq_handler(dev_priv, pm_iir);
1601                 /* Call regardless, as some status bits might not be
1602                  * signalled in iir */
1603                 valleyview_pipestat_irq_handler(dev, iir);
1604         }
1605
1606 out:
1607         return ret;
1608 }
1609
1610 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1611 {
1612         struct drm_device *dev = arg;
1613         struct drm_i915_private *dev_priv = dev->dev_private;
1614         u32 master_ctl, iir;
1615         irqreturn_t ret = IRQ_NONE;
1616
1617         if (!intel_irqs_enabled(dev_priv))
1618                 return IRQ_NONE;
1619
1620         for (;;) {
1621                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1622                 iir = I915_READ(VLV_IIR);
1623
1624                 if (master_ctl == 0 && iir == 0)
1625                         break;
1626
1627                 ret = IRQ_HANDLED;
1628
1629                 I915_WRITE(GEN8_MASTER_IRQ, 0);
1630
1631                 /* Find, clear, then process each source of interrupt */
1632
1633                 if (iir) {
1634                         /* Consume port before clearing IIR or we'll miss events */
1635                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1636                                 i9xx_hpd_irq_handler(dev);
1637                         I915_WRITE(VLV_IIR, iir);
1638                 }
1639
1640                 gen8_gt_irq_handler(dev_priv, master_ctl);
1641
1642                 /* Call regardless, as some status bits might not be
1643                  * signalled in iir */
1644                 valleyview_pipestat_irq_handler(dev, iir);
1645
1646                 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1647                 POSTING_READ(GEN8_MASTER_IRQ);
1648         }
1649
1650         return ret;
1651 }
1652
1653 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1654 {
1655         struct drm_i915_private *dev_priv = dev->dev_private;
1656         int pipe;
1657         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1658
1659         if (hotplug_trigger) {
1660                 u32 dig_hotplug_reg, pin_mask, long_mask;
1661
1662                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1663                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1664
1665                 pch_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1666                                  dig_hotplug_reg, hpd_ibx);
1667                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1668         }
1669
1670         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1671                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1672                                SDE_AUDIO_POWER_SHIFT);
1673                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1674                                  port_name(port));
1675         }
1676
1677         if (pch_iir & SDE_AUX_MASK)
1678                 dp_aux_irq_handler(dev);
1679
1680         if (pch_iir & SDE_GMBUS)
1681                 gmbus_irq_handler(dev);
1682
1683         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1684                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1685
1686         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1687                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1688
1689         if (pch_iir & SDE_POISON)
1690                 DRM_ERROR("PCH poison interrupt\n");
1691
1692         if (pch_iir & SDE_FDI_MASK)
1693                 for_each_pipe(dev_priv, pipe)
1694                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1695                                          pipe_name(pipe),
1696                                          I915_READ(FDI_RX_IIR(pipe)));
1697
1698         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1699                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1700
1701         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1702                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1703
1704         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1705                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1706
1707         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1708                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1709 }
1710
1711 static void ivb_err_int_handler(struct drm_device *dev)
1712 {
1713         struct drm_i915_private *dev_priv = dev->dev_private;
1714         u32 err_int = I915_READ(GEN7_ERR_INT);
1715         enum pipe pipe;
1716
1717         if (err_int & ERR_INT_POISON)
1718                 DRM_ERROR("Poison interrupt\n");
1719
1720         for_each_pipe(dev_priv, pipe) {
1721                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1722                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1723
1724                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1725                         if (IS_IVYBRIDGE(dev))
1726                                 ivb_pipe_crc_irq_handler(dev, pipe);
1727                         else
1728                                 hsw_pipe_crc_irq_handler(dev, pipe);
1729                 }
1730         }
1731
1732         I915_WRITE(GEN7_ERR_INT, err_int);
1733 }
1734
1735 static void cpt_serr_int_handler(struct drm_device *dev)
1736 {
1737         struct drm_i915_private *dev_priv = dev->dev_private;
1738         u32 serr_int = I915_READ(SERR_INT);
1739
1740         if (serr_int & SERR_INT_POISON)
1741                 DRM_ERROR("PCH poison interrupt\n");
1742
1743         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1744                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1745
1746         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1747                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1748
1749         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1750                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1751
1752         I915_WRITE(SERR_INT, serr_int);
1753 }
1754
1755 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1756 {
1757         struct drm_i915_private *dev_priv = dev->dev_private;
1758         int pipe;
1759         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1760
1761         if (hotplug_trigger) {
1762                 u32 dig_hotplug_reg, pin_mask, long_mask;
1763
1764                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1765                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1766                 pch_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1767                                  dig_hotplug_reg, hpd_cpt);
1768                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1769         }
1770
1771         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1772                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1773                                SDE_AUDIO_POWER_SHIFT_CPT);
1774                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1775                                  port_name(port));
1776         }
1777
1778         if (pch_iir & SDE_AUX_MASK_CPT)
1779                 dp_aux_irq_handler(dev);
1780
1781         if (pch_iir & SDE_GMBUS_CPT)
1782                 gmbus_irq_handler(dev);
1783
1784         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1785                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1786
1787         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1788                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1789
1790         if (pch_iir & SDE_FDI_MASK_CPT)
1791                 for_each_pipe(dev_priv, pipe)
1792                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1793                                          pipe_name(pipe),
1794                                          I915_READ(FDI_RX_IIR(pipe)));
1795
1796         if (pch_iir & SDE_ERROR_CPT)
1797                 cpt_serr_int_handler(dev);
1798 }
1799
1800 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1801 {
1802         struct drm_i915_private *dev_priv = dev->dev_private;
1803         enum pipe pipe;
1804
1805         if (de_iir & DE_AUX_CHANNEL_A)
1806                 dp_aux_irq_handler(dev);
1807
1808         if (de_iir & DE_GSE)
1809                 intel_opregion_asle_intr(dev);
1810
1811         if (de_iir & DE_POISON)
1812                 DRM_ERROR("Poison interrupt\n");
1813
1814         for_each_pipe(dev_priv, pipe) {
1815                 if (de_iir & DE_PIPE_VBLANK(pipe) &&
1816                     intel_pipe_handle_vblank(dev, pipe))
1817                         intel_check_page_flip(dev, pipe);
1818
1819                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1820                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1821
1822                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
1823                         i9xx_pipe_crc_irq_handler(dev, pipe);
1824
1825                 /* plane/pipes map 1:1 on ilk+ */
1826                 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
1827                         intel_prepare_page_flip(dev, pipe);
1828                         intel_finish_page_flip_plane(dev, pipe);
1829                 }
1830         }
1831
1832         /* check event from PCH */
1833         if (de_iir & DE_PCH_EVENT) {
1834                 u32 pch_iir = I915_READ(SDEIIR);
1835
1836                 if (HAS_PCH_CPT(dev))
1837                         cpt_irq_handler(dev, pch_iir);
1838                 else
1839                         ibx_irq_handler(dev, pch_iir);
1840
1841                 /* should clear PCH hotplug event before clear CPU irq */
1842                 I915_WRITE(SDEIIR, pch_iir);
1843         }
1844
1845         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1846                 ironlake_rps_change_irq_handler(dev);
1847 }
1848
1849 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1850 {
1851         struct drm_i915_private *dev_priv = dev->dev_private;
1852         enum pipe pipe;
1853
1854         if (de_iir & DE_ERR_INT_IVB)
1855                 ivb_err_int_handler(dev);
1856
1857         if (de_iir & DE_AUX_CHANNEL_A_IVB)
1858                 dp_aux_irq_handler(dev);
1859
1860         if (de_iir & DE_GSE_IVB)
1861                 intel_opregion_asle_intr(dev);
1862
1863         for_each_pipe(dev_priv, pipe) {
1864                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
1865                     intel_pipe_handle_vblank(dev, pipe))
1866                         intel_check_page_flip(dev, pipe);
1867
1868                 /* plane/pipes map 1:1 on ilk+ */
1869                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
1870                         intel_prepare_page_flip(dev, pipe);
1871                         intel_finish_page_flip_plane(dev, pipe);
1872                 }
1873         }
1874
1875         /* check event from PCH */
1876         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1877                 u32 pch_iir = I915_READ(SDEIIR);
1878
1879                 cpt_irq_handler(dev, pch_iir);
1880
1881                 /* clear PCH hotplug event before clear CPU irq */
1882                 I915_WRITE(SDEIIR, pch_iir);
1883         }
1884 }
1885
1886 /*
1887  * To handle irqs with the minimum potential races with fresh interrupts, we:
1888  * 1 - Disable Master Interrupt Control.
1889  * 2 - Find the source(s) of the interrupt.
1890  * 3 - Clear the Interrupt Identity bits (IIR).
1891  * 4 - Process the interrupt(s) that had bits set in the IIRs.
1892  * 5 - Re-enable Master Interrupt Control.
1893  */
1894 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1895 {
1896         struct drm_device *dev = arg;
1897         struct drm_i915_private *dev_priv = dev->dev_private;
1898         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
1899         irqreturn_t ret = IRQ_NONE;
1900
1901         if (!intel_irqs_enabled(dev_priv))
1902                 return IRQ_NONE;
1903
1904         /* We get interrupts on unclaimed registers, so check for this before we
1905          * do any I915_{READ,WRITE}. */
1906         intel_uncore_check_errors(dev);
1907
1908         /* disable master interrupt before clearing iir  */
1909         de_ier = I915_READ(DEIER);
1910         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1911         POSTING_READ(DEIER);
1912
1913         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1914          * interrupts will will be stored on its back queue, and then we'll be
1915          * able to process them after we restore SDEIER (as soon as we restore
1916          * it, we'll get an interrupt if SDEIIR still has something to process
1917          * due to its back queue). */
1918         if (!HAS_PCH_NOP(dev)) {
1919                 sde_ier = I915_READ(SDEIER);
1920                 I915_WRITE(SDEIER, 0);
1921                 POSTING_READ(SDEIER);
1922         }
1923
1924         /* Find, clear, then process each source of interrupt */
1925
1926         gt_iir = I915_READ(GTIIR);
1927         if (gt_iir) {
1928                 I915_WRITE(GTIIR, gt_iir);
1929                 ret = IRQ_HANDLED;
1930                 if (INTEL_INFO(dev)->gen >= 6)
1931                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1932                 else
1933                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1934         }
1935
1936         de_iir = I915_READ(DEIIR);
1937         if (de_iir) {
1938                 I915_WRITE(DEIIR, de_iir);
1939                 ret = IRQ_HANDLED;
1940                 if (INTEL_INFO(dev)->gen >= 7)
1941                         ivb_display_irq_handler(dev, de_iir);
1942                 else
1943                         ilk_display_irq_handler(dev, de_iir);
1944         }
1945
1946         if (INTEL_INFO(dev)->gen >= 6) {
1947                 u32 pm_iir = I915_READ(GEN6_PMIIR);
1948                 if (pm_iir) {
1949                         I915_WRITE(GEN6_PMIIR, pm_iir);
1950                         ret = IRQ_HANDLED;
1951                         gen6_rps_irq_handler(dev_priv, pm_iir);
1952                 }
1953         }
1954
1955         I915_WRITE(DEIER, de_ier);
1956         POSTING_READ(DEIER);
1957         if (!HAS_PCH_NOP(dev)) {
1958                 I915_WRITE(SDEIER, sde_ier);
1959                 POSTING_READ(SDEIER);
1960         }
1961
1962         return ret;
1963 }
1964
1965 static void bxt_hpd_handler(struct drm_device *dev, uint32_t iir_status)
1966 {
1967         struct drm_i915_private *dev_priv = dev->dev_private;
1968         u32 hp_control, hp_trigger;
1969         u32 pin_mask, long_mask;
1970
1971         /* Get the status */
1972         hp_trigger = iir_status & BXT_DE_PORT_HOTPLUG_MASK;
1973         hp_control = I915_READ(BXT_HOTPLUG_CTL);
1974
1975         /* Hotplug not enabled ? */
1976         if (!(hp_control & BXT_HOTPLUG_CTL_MASK)) {
1977                 DRM_ERROR("Interrupt when HPD disabled\n");
1978                 return;
1979         }
1980
1981         /* Clear sticky bits in hpd status */
1982         I915_WRITE(BXT_HOTPLUG_CTL, hp_control);
1983
1984         pch_get_hpd_pins(&pin_mask, &long_mask, hp_trigger, hp_control, hpd_bxt);
1985         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1986 }
1987
1988 static irqreturn_t gen8_irq_handler(int irq, void *arg)
1989 {
1990         struct drm_device *dev = arg;
1991         struct drm_i915_private *dev_priv = dev->dev_private;
1992         u32 master_ctl;
1993         irqreturn_t ret = IRQ_NONE;
1994         uint32_t tmp = 0;
1995         enum pipe pipe;
1996         u32 aux_mask = GEN8_AUX_CHANNEL_A;
1997
1998         if (!intel_irqs_enabled(dev_priv))
1999                 return IRQ_NONE;
2000
2001         if (IS_GEN9(dev))
2002                 aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2003                         GEN9_AUX_CHANNEL_D;
2004
2005         master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2006         master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2007         if (!master_ctl)
2008                 return IRQ_NONE;
2009
2010         I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2011
2012         /* Find, clear, then process each source of interrupt */
2013
2014         ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2015
2016         if (master_ctl & GEN8_DE_MISC_IRQ) {
2017                 tmp = I915_READ(GEN8_DE_MISC_IIR);
2018                 if (tmp) {
2019                         I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2020                         ret = IRQ_HANDLED;
2021                         if (tmp & GEN8_DE_MISC_GSE)
2022                                 intel_opregion_asle_intr(dev);
2023                         else
2024                                 DRM_ERROR("Unexpected DE Misc interrupt\n");
2025                 }
2026                 else
2027                         DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2028         }
2029
2030         if (master_ctl & GEN8_DE_PORT_IRQ) {
2031                 tmp = I915_READ(GEN8_DE_PORT_IIR);
2032                 if (tmp) {
2033                         bool found = false;
2034
2035                         I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2036                         ret = IRQ_HANDLED;
2037
2038                         if (tmp & aux_mask) {
2039                                 dp_aux_irq_handler(dev);
2040                                 found = true;
2041                         }
2042
2043                         if (IS_BROXTON(dev) && tmp & BXT_DE_PORT_HOTPLUG_MASK) {
2044                                 bxt_hpd_handler(dev, tmp);
2045                                 found = true;
2046                         }
2047
2048                         if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2049                                 gmbus_irq_handler(dev);
2050                                 found = true;
2051                         }
2052
2053                         if (!found)
2054                                 DRM_ERROR("Unexpected DE Port interrupt\n");
2055                 }
2056                 else
2057                         DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2058         }
2059
2060         for_each_pipe(dev_priv, pipe) {
2061                 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2062
2063                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2064                         continue;
2065
2066                 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2067                 if (pipe_iir) {
2068                         ret = IRQ_HANDLED;
2069                         I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2070
2071                         if (pipe_iir & GEN8_PIPE_VBLANK &&
2072                             intel_pipe_handle_vblank(dev, pipe))
2073                                 intel_check_page_flip(dev, pipe);
2074
2075                         if (IS_GEN9(dev))
2076                                 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2077                         else
2078                                 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2079
2080                         if (flip_done) {
2081                                 intel_prepare_page_flip(dev, pipe);
2082                                 intel_finish_page_flip_plane(dev, pipe);
2083                         }
2084
2085                         if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2086                                 hsw_pipe_crc_irq_handler(dev, pipe);
2087
2088                         if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2089                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2090                                                                     pipe);
2091
2092
2093                         if (IS_GEN9(dev))
2094                                 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2095                         else
2096                                 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2097
2098                         if (fault_errors)
2099                                 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2100                                           pipe_name(pipe),
2101                                           pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2102                 } else
2103                         DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2104         }
2105
2106         if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2107             master_ctl & GEN8_DE_PCH_IRQ) {
2108                 /*
2109                  * FIXME(BDW): Assume for now that the new interrupt handling
2110                  * scheme also closed the SDE interrupt handling race we've seen
2111                  * on older pch-split platforms. But this needs testing.
2112                  */
2113                 u32 pch_iir = I915_READ(SDEIIR);
2114                 if (pch_iir) {
2115                         I915_WRITE(SDEIIR, pch_iir);
2116                         ret = IRQ_HANDLED;
2117                         cpt_irq_handler(dev, pch_iir);
2118                 } else
2119                         DRM_ERROR("The master control interrupt lied (SDE)!\n");
2120
2121         }
2122
2123         I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2124         POSTING_READ_FW(GEN8_MASTER_IRQ);
2125
2126         return ret;
2127 }
2128
2129 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2130                                bool reset_completed)
2131 {
2132         struct intel_engine_cs *ring;
2133         int i;
2134
2135         /*
2136          * Notify all waiters for GPU completion events that reset state has
2137          * been changed, and that they need to restart their wait after
2138          * checking for potential errors (and bail out to drop locks if there is
2139          * a gpu reset pending so that i915_error_work_func can acquire them).
2140          */
2141
2142         /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2143         for_each_ring(ring, dev_priv, i)
2144                 wake_up_all(&ring->irq_queue);
2145
2146         /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2147         wake_up_all(&dev_priv->pending_flip_queue);
2148
2149         /*
2150          * Signal tasks blocked in i915_gem_wait_for_error that the pending
2151          * reset state is cleared.
2152          */
2153         if (reset_completed)
2154                 wake_up_all(&dev_priv->gpu_error.reset_queue);
2155 }
2156
2157 /**
2158  * i915_reset_and_wakeup - do process context error handling work
2159  *
2160  * Fire an error uevent so userspace can see that a hang or error
2161  * was detected.
2162  */
2163 static void i915_reset_and_wakeup(struct drm_device *dev)
2164 {
2165         struct drm_i915_private *dev_priv = to_i915(dev);
2166         struct i915_gpu_error *error = &dev_priv->gpu_error;
2167         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2168         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2169         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2170         int ret;
2171
2172         kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2173
2174         /*
2175          * Note that there's only one work item which does gpu resets, so we
2176          * need not worry about concurrent gpu resets potentially incrementing
2177          * error->reset_counter twice. We only need to take care of another
2178          * racing irq/hangcheck declaring the gpu dead for a second time. A
2179          * quick check for that is good enough: schedule_work ensures the
2180          * correct ordering between hang detection and this work item, and since
2181          * the reset in-progress bit is only ever set by code outside of this
2182          * work we don't need to worry about any other races.
2183          */
2184         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2185                 DRM_DEBUG_DRIVER("resetting chip\n");
2186                 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2187                                    reset_event);
2188
2189                 /*
2190                  * In most cases it's guaranteed that we get here with an RPM
2191                  * reference held, for example because there is a pending GPU
2192                  * request that won't finish until the reset is done. This
2193                  * isn't the case at least when we get here by doing a
2194                  * simulated reset via debugs, so get an RPM reference.
2195                  */
2196                 intel_runtime_pm_get(dev_priv);
2197
2198                 intel_prepare_reset(dev);
2199
2200                 /*
2201                  * All state reset _must_ be completed before we update the
2202                  * reset counter, for otherwise waiters might miss the reset
2203                  * pending state and not properly drop locks, resulting in
2204                  * deadlocks with the reset work.
2205                  */
2206                 ret = i915_reset(dev);
2207
2208                 intel_finish_reset(dev);
2209
2210                 intel_runtime_pm_put(dev_priv);
2211
2212                 if (ret == 0) {
2213                         /*
2214                          * After all the gem state is reset, increment the reset
2215                          * counter and wake up everyone waiting for the reset to
2216                          * complete.
2217                          *
2218                          * Since unlock operations are a one-sided barrier only,
2219                          * we need to insert a barrier here to order any seqno
2220                          * updates before
2221                          * the counter increment.
2222                          */
2223                         smp_mb__before_atomic();
2224                         atomic_inc(&dev_priv->gpu_error.reset_counter);
2225
2226                         kobject_uevent_env(&dev->primary->kdev->kobj,
2227                                            KOBJ_CHANGE, reset_done_event);
2228                 } else {
2229                         atomic_set_mask(I915_WEDGED, &error->reset_counter);
2230                 }
2231
2232                 /*
2233                  * Note: The wake_up also serves as a memory barrier so that
2234                  * waiters see the update value of the reset counter atomic_t.
2235                  */
2236                 i915_error_wake_up(dev_priv, true);
2237         }
2238 }
2239
2240 static void i915_report_and_clear_eir(struct drm_device *dev)
2241 {
2242         struct drm_i915_private *dev_priv = dev->dev_private;
2243         uint32_t instdone[I915_NUM_INSTDONE_REG];
2244         u32 eir = I915_READ(EIR);
2245         int pipe, i;
2246
2247         if (!eir)
2248                 return;
2249
2250         pr_err("render error detected, EIR: 0x%08x\n", eir);
2251
2252         i915_get_extra_instdone(dev, instdone);
2253
2254         if (IS_G4X(dev)) {
2255                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2256                         u32 ipeir = I915_READ(IPEIR_I965);
2257
2258                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2259                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2260                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
2261                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2262                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2263                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2264                         I915_WRITE(IPEIR_I965, ipeir);
2265                         POSTING_READ(IPEIR_I965);
2266                 }
2267                 if (eir & GM45_ERROR_PAGE_TABLE) {
2268                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2269                         pr_err("page table error\n");
2270                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2271                         I915_WRITE(PGTBL_ER, pgtbl_err);
2272                         POSTING_READ(PGTBL_ER);
2273                 }
2274         }
2275
2276         if (!IS_GEN2(dev)) {
2277                 if (eir & I915_ERROR_PAGE_TABLE) {
2278                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2279                         pr_err("page table error\n");
2280                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2281                         I915_WRITE(PGTBL_ER, pgtbl_err);
2282                         POSTING_READ(PGTBL_ER);
2283                 }
2284         }
2285
2286         if (eir & I915_ERROR_MEMORY_REFRESH) {
2287                 pr_err("memory refresh error:\n");
2288                 for_each_pipe(dev_priv, pipe)
2289                         pr_err("pipe %c stat: 0x%08x\n",
2290                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2291                 /* pipestat has already been acked */
2292         }
2293         if (eir & I915_ERROR_INSTRUCTION) {
2294                 pr_err("instruction error\n");
2295                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2296                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2297                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2298                 if (INTEL_INFO(dev)->gen < 4) {
2299                         u32 ipeir = I915_READ(IPEIR);
2300
2301                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2302                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2303                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2304                         I915_WRITE(IPEIR, ipeir);
2305                         POSTING_READ(IPEIR);
2306                 } else {
2307                         u32 ipeir = I915_READ(IPEIR_I965);
2308
2309                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2310                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2311                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2312                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2313                         I915_WRITE(IPEIR_I965, ipeir);
2314                         POSTING_READ(IPEIR_I965);
2315                 }
2316         }
2317
2318         I915_WRITE(EIR, eir);
2319         POSTING_READ(EIR);
2320         eir = I915_READ(EIR);
2321         if (eir) {
2322                 /*
2323                  * some errors might have become stuck,
2324                  * mask them.
2325                  */
2326                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2327                 I915_WRITE(EMR, I915_READ(EMR) | eir);
2328                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2329         }
2330 }
2331
2332 /**
2333  * i915_handle_error - handle a gpu error
2334  * @dev: drm device
2335  *
2336  * Do some basic checking of regsiter state at error time and
2337  * dump it to the syslog.  Also call i915_capture_error_state() to make
2338  * sure we get a record and make it available in debugfs.  Fire a uevent
2339  * so userspace knows something bad happened (should trigger collection
2340  * of a ring dump etc.).
2341  */
2342 void i915_handle_error(struct drm_device *dev, bool wedged,
2343                        const char *fmt, ...)
2344 {
2345         struct drm_i915_private *dev_priv = dev->dev_private;
2346         va_list args;
2347         char error_msg[80];
2348
2349         va_start(args, fmt);
2350         vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2351         va_end(args);
2352
2353         i915_capture_error_state(dev, wedged, error_msg);
2354         i915_report_and_clear_eir(dev);
2355
2356         if (wedged) {
2357                 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2358                                 &dev_priv->gpu_error.reset_counter);
2359
2360                 /*
2361                  * Wakeup waiting processes so that the reset function
2362                  * i915_reset_and_wakeup doesn't deadlock trying to grab
2363                  * various locks. By bumping the reset counter first, the woken
2364                  * processes will see a reset in progress and back off,
2365                  * releasing their locks and then wait for the reset completion.
2366                  * We must do this for _all_ gpu waiters that might hold locks
2367                  * that the reset work needs to acquire.
2368                  *
2369                  * Note: The wake_up serves as the required memory barrier to
2370                  * ensure that the waiters see the updated value of the reset
2371                  * counter atomic_t.
2372                  */
2373                 i915_error_wake_up(dev_priv, false);
2374         }
2375
2376         i915_reset_and_wakeup(dev);
2377 }
2378
2379 /* Called from drm generic code, passed 'crtc' which
2380  * we use as a pipe index
2381  */
2382 static int i915_enable_vblank(struct drm_device *dev, int pipe)
2383 {
2384         struct drm_i915_private *dev_priv = dev->dev_private;
2385         unsigned long irqflags;
2386
2387         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2388         if (INTEL_INFO(dev)->gen >= 4)
2389                 i915_enable_pipestat(dev_priv, pipe,
2390                                      PIPE_START_VBLANK_INTERRUPT_STATUS);
2391         else
2392                 i915_enable_pipestat(dev_priv, pipe,
2393                                      PIPE_VBLANK_INTERRUPT_STATUS);
2394         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2395
2396         return 0;
2397 }
2398
2399 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2400 {
2401         struct drm_i915_private *dev_priv = dev->dev_private;
2402         unsigned long irqflags;
2403         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2404                                                      DE_PIPE_VBLANK(pipe);
2405
2406         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2407         ironlake_enable_display_irq(dev_priv, bit);
2408         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2409
2410         return 0;
2411 }
2412
2413 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2414 {
2415         struct drm_i915_private *dev_priv = dev->dev_private;
2416         unsigned long irqflags;
2417
2418         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2419         i915_enable_pipestat(dev_priv, pipe,
2420                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2421         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2422
2423         return 0;
2424 }
2425
2426 static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2427 {
2428         struct drm_i915_private *dev_priv = dev->dev_private;
2429         unsigned long irqflags;
2430
2431         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2432         dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2433         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2434         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2435         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2436         return 0;
2437 }
2438
2439 /* Called from drm generic code, passed 'crtc' which
2440  * we use as a pipe index
2441  */
2442 static void i915_disable_vblank(struct drm_device *dev, int pipe)
2443 {
2444         struct drm_i915_private *dev_priv = dev->dev_private;
2445         unsigned long irqflags;
2446
2447         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2448         i915_disable_pipestat(dev_priv, pipe,
2449                               PIPE_VBLANK_INTERRUPT_STATUS |
2450                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2451         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2452 }
2453
2454 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2455 {
2456         struct drm_i915_private *dev_priv = dev->dev_private;
2457         unsigned long irqflags;
2458         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2459                                                      DE_PIPE_VBLANK(pipe);
2460
2461         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2462         ironlake_disable_display_irq(dev_priv, bit);
2463         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2464 }
2465
2466 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2467 {
2468         struct drm_i915_private *dev_priv = dev->dev_private;
2469         unsigned long irqflags;
2470
2471         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2472         i915_disable_pipestat(dev_priv, pipe,
2473                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2474         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2475 }
2476
2477 static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2478 {
2479         struct drm_i915_private *dev_priv = dev->dev_private;
2480         unsigned long irqflags;
2481
2482         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2483         dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2484         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2485         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2486         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2487 }
2488
2489 static bool
2490 ring_idle(struct intel_engine_cs *ring, u32 seqno)
2491 {
2492         return (list_empty(&ring->request_list) ||
2493                 i915_seqno_passed(seqno, ring->last_submitted_seqno));
2494 }
2495
2496 static bool
2497 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2498 {
2499         if (INTEL_INFO(dev)->gen >= 8) {
2500                 return (ipehr >> 23) == 0x1c;
2501         } else {
2502                 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2503                 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2504                                  MI_SEMAPHORE_REGISTER);
2505         }
2506 }
2507
2508 static struct intel_engine_cs *
2509 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2510 {
2511         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2512         struct intel_engine_cs *signaller;
2513         int i;
2514
2515         if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2516                 for_each_ring(signaller, dev_priv, i) {
2517                         if (ring == signaller)
2518                                 continue;
2519
2520                         if (offset == signaller->semaphore.signal_ggtt[ring->id])
2521                                 return signaller;
2522                 }
2523         } else {
2524                 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2525
2526                 for_each_ring(signaller, dev_priv, i) {
2527                         if(ring == signaller)
2528                                 continue;
2529
2530                         if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2531                                 return signaller;
2532                 }
2533         }
2534
2535         DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2536                   ring->id, ipehr, offset);
2537
2538         return NULL;
2539 }
2540
2541 static struct intel_engine_cs *
2542 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2543 {
2544         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2545         u32 cmd, ipehr, head;
2546         u64 offset = 0;
2547         int i, backwards;
2548
2549         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2550         if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2551                 return NULL;
2552
2553         /*
2554          * HEAD is likely pointing to the dword after the actual command,
2555          * so scan backwards until we find the MBOX. But limit it to just 3
2556          * or 4 dwords depending on the semaphore wait command size.
2557          * Note that we don't care about ACTHD here since that might
2558          * point at at batch, and semaphores are always emitted into the
2559          * ringbuffer itself.
2560          */
2561         head = I915_READ_HEAD(ring) & HEAD_ADDR;
2562         backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2563
2564         for (i = backwards; i; --i) {
2565                 /*
2566                  * Be paranoid and presume the hw has gone off into the wild -
2567                  * our ring is smaller than what the hardware (and hence
2568                  * HEAD_ADDR) allows. Also handles wrap-around.
2569                  */
2570                 head &= ring->buffer->size - 1;
2571
2572                 /* This here seems to blow up */
2573                 cmd = ioread32(ring->buffer->virtual_start + head);
2574                 if (cmd == ipehr)
2575                         break;
2576
2577                 head -= 4;
2578         }
2579
2580         if (!i)
2581                 return NULL;
2582
2583         *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2584         if (INTEL_INFO(ring->dev)->gen >= 8) {
2585                 offset = ioread32(ring->buffer->virtual_start + head + 12);
2586                 offset <<= 32;
2587                 offset = ioread32(ring->buffer->virtual_start + head + 8);
2588         }
2589         return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2590 }
2591
2592 static int semaphore_passed(struct intel_engine_cs *ring)
2593 {
2594         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2595         struct intel_engine_cs *signaller;
2596         u32 seqno;
2597
2598         ring->hangcheck.deadlock++;
2599
2600         signaller = semaphore_waits_for(ring, &seqno);
2601         if (signaller == NULL)
2602                 return -1;
2603
2604         /* Prevent pathological recursion due to driver bugs */
2605         if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2606                 return -1;
2607
2608         if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2609                 return 1;
2610
2611         /* cursory check for an unkickable deadlock */
2612         if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2613             semaphore_passed(signaller) < 0)
2614                 return -1;
2615
2616         return 0;
2617 }
2618
2619 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2620 {
2621         struct intel_engine_cs *ring;
2622         int i;
2623
2624         for_each_ring(ring, dev_priv, i)
2625                 ring->hangcheck.deadlock = 0;
2626 }
2627
2628 static enum intel_ring_hangcheck_action
2629 ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2630 {
2631         struct drm_device *dev = ring->dev;
2632         struct drm_i915_private *dev_priv = dev->dev_private;
2633         u32 tmp;
2634
2635         if (acthd != ring->hangcheck.acthd) {
2636                 if (acthd > ring->hangcheck.max_acthd) {
2637                         ring->hangcheck.max_acthd = acthd;
2638                         return HANGCHECK_ACTIVE;
2639                 }
2640
2641                 return HANGCHECK_ACTIVE_LOOP;
2642         }
2643
2644         if (IS_GEN2(dev))
2645                 return HANGCHECK_HUNG;
2646
2647         /* Is the chip hanging on a WAIT_FOR_EVENT?
2648          * If so we can simply poke the RB_WAIT bit
2649          * and break the hang. This should work on
2650          * all but the second generation chipsets.
2651          */
2652         tmp = I915_READ_CTL(ring);
2653         if (tmp & RING_WAIT) {
2654                 i915_handle_error(dev, false,
2655                                   "Kicking stuck wait on %s",
2656                                   ring->name);
2657                 I915_WRITE_CTL(ring, tmp);
2658                 return HANGCHECK_KICK;
2659         }
2660
2661         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2662                 switch (semaphore_passed(ring)) {
2663                 default:
2664                         return HANGCHECK_HUNG;
2665                 case 1:
2666                         i915_handle_error(dev, false,
2667                                           "Kicking stuck semaphore on %s",
2668                                           ring->name);
2669                         I915_WRITE_CTL(ring, tmp);
2670                         return HANGCHECK_KICK;
2671                 case 0:
2672                         return HANGCHECK_WAIT;
2673                 }
2674         }
2675
2676         return HANGCHECK_HUNG;
2677 }
2678
2679 /*
2680  * This is called when the chip hasn't reported back with completed
2681  * batchbuffers in a long time. We keep track per ring seqno progress and
2682  * if there are no progress, hangcheck score for that ring is increased.
2683  * Further, acthd is inspected to see if the ring is stuck. On stuck case
2684  * we kick the ring. If we see no progress on three subsequent calls
2685  * we assume chip is wedged and try to fix it by resetting the chip.
2686  */
2687 static void i915_hangcheck_elapsed(struct work_struct *work)
2688 {
2689         struct drm_i915_private *dev_priv =
2690                 container_of(work, typeof(*dev_priv),
2691                              gpu_error.hangcheck_work.work);
2692         struct drm_device *dev = dev_priv->dev;
2693         struct intel_engine_cs *ring;
2694         int i;
2695         int busy_count = 0, rings_hung = 0;
2696         bool stuck[I915_NUM_RINGS] = { 0 };
2697 #define BUSY 1
2698 #define KICK 5
2699 #define HUNG 20
2700
2701         if (!i915.enable_hangcheck)
2702                 return;
2703
2704         for_each_ring(ring, dev_priv, i) {
2705                 u64 acthd;
2706                 u32 seqno;
2707                 bool busy = true;
2708
2709                 semaphore_clear_deadlocks(dev_priv);
2710
2711                 seqno = ring->get_seqno(ring, false);
2712                 acthd = intel_ring_get_active_head(ring);
2713
2714                 if (ring->hangcheck.seqno == seqno) {
2715                         if (ring_idle(ring, seqno)) {
2716                                 ring->hangcheck.action = HANGCHECK_IDLE;
2717
2718                                 if (waitqueue_active(&ring->irq_queue)) {
2719                                         /* Issue a wake-up to catch stuck h/w. */
2720                                         if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2721                                                 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2722                                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2723                                                                   ring->name);
2724                                                 else
2725                                                         DRM_INFO("Fake missed irq on %s\n",
2726                                                                  ring->name);
2727                                                 wake_up_all(&ring->irq_queue);
2728                                         }
2729                                         /* Safeguard against driver failure */
2730                                         ring->hangcheck.score += BUSY;
2731                                 } else
2732                                         busy = false;
2733                         } else {
2734                                 /* We always increment the hangcheck score
2735                                  * if the ring is busy and still processing
2736                                  * the same request, so that no single request
2737                                  * can run indefinitely (such as a chain of
2738                                  * batches). The only time we do not increment
2739                                  * the hangcheck score on this ring, if this
2740                                  * ring is in a legitimate wait for another
2741                                  * ring. In that case the waiting ring is a
2742                                  * victim and we want to be sure we catch the
2743                                  * right culprit. Then every time we do kick
2744                                  * the ring, add a small increment to the
2745                                  * score so that we can catch a batch that is
2746                                  * being repeatedly kicked and so responsible
2747                                  * for stalling the machine.
2748                                  */
2749                                 ring->hangcheck.action = ring_stuck(ring,
2750                                                                     acthd);
2751
2752                                 switch (ring->hangcheck.action) {
2753                                 case HANGCHECK_IDLE:
2754                                 case HANGCHECK_WAIT:
2755                                 case HANGCHECK_ACTIVE:
2756                                         break;
2757                                 case HANGCHECK_ACTIVE_LOOP:
2758                                         ring->hangcheck.score += BUSY;
2759                                         break;
2760                                 case HANGCHECK_KICK:
2761                                         ring->hangcheck.score += KICK;
2762                                         break;
2763                                 case HANGCHECK_HUNG:
2764                                         ring->hangcheck.score += HUNG;
2765                                         stuck[i] = true;
2766                                         break;
2767                                 }
2768                         }
2769                 } else {
2770                         ring->hangcheck.action = HANGCHECK_ACTIVE;
2771
2772                         /* Gradually reduce the count so that we catch DoS
2773                          * attempts across multiple batches.
2774                          */
2775                         if (ring->hangcheck.score > 0)
2776                                 ring->hangcheck.score--;
2777
2778                         ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
2779                 }
2780
2781                 ring->hangcheck.seqno = seqno;
2782                 ring->hangcheck.acthd = acthd;
2783                 busy_count += busy;
2784         }
2785
2786         for_each_ring(ring, dev_priv, i) {
2787                 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
2788                         DRM_INFO("%s on %s\n",
2789                                  stuck[i] ? "stuck" : "no progress",
2790                                  ring->name);
2791                         rings_hung++;
2792                 }
2793         }
2794
2795         if (rings_hung)
2796                 return i915_handle_error(dev, true, "Ring hung");
2797
2798         if (busy_count)
2799                 /* Reset timer case chip hangs without another request
2800                  * being added */
2801                 i915_queue_hangcheck(dev);
2802 }
2803
2804 void i915_queue_hangcheck(struct drm_device *dev)
2805 {
2806         struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
2807
2808         if (!i915.enable_hangcheck)
2809                 return;
2810
2811         /* Don't continually defer the hangcheck so that it is always run at
2812          * least once after work has been scheduled on any ring. Otherwise,
2813          * we will ignore a hung ring if a second ring is kept busy.
2814          */
2815
2816         queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
2817                            round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
2818 }
2819
2820 static void ibx_irq_reset(struct drm_device *dev)
2821 {
2822         struct drm_i915_private *dev_priv = dev->dev_private;
2823
2824         if (HAS_PCH_NOP(dev))
2825                 return;
2826
2827         GEN5_IRQ_RESET(SDE);
2828
2829         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2830                 I915_WRITE(SERR_INT, 0xffffffff);
2831 }
2832
2833 /*
2834  * SDEIER is also touched by the interrupt handler to work around missed PCH
2835  * interrupts. Hence we can't update it after the interrupt handler is enabled -
2836  * instead we unconditionally enable all PCH interrupt sources here, but then
2837  * only unmask them as needed with SDEIMR.
2838  *
2839  * This function needs to be called before interrupts are enabled.
2840  */
2841 static void ibx_irq_pre_postinstall(struct drm_device *dev)
2842 {
2843         struct drm_i915_private *dev_priv = dev->dev_private;
2844
2845         if (HAS_PCH_NOP(dev))
2846                 return;
2847
2848         WARN_ON(I915_READ(SDEIER) != 0);
2849         I915_WRITE(SDEIER, 0xffffffff);
2850         POSTING_READ(SDEIER);
2851 }
2852
2853 static void gen5_gt_irq_reset(struct drm_device *dev)
2854 {
2855         struct drm_i915_private *dev_priv = dev->dev_private;
2856
2857         GEN5_IRQ_RESET(GT);
2858         if (INTEL_INFO(dev)->gen >= 6)
2859                 GEN5_IRQ_RESET(GEN6_PM);
2860 }
2861
2862 /* drm_dma.h hooks
2863 */
2864 static void ironlake_irq_reset(struct drm_device *dev)
2865 {
2866         struct drm_i915_private *dev_priv = dev->dev_private;
2867
2868         I915_WRITE(HWSTAM, 0xffffffff);
2869
2870         GEN5_IRQ_RESET(DE);
2871         if (IS_GEN7(dev))
2872                 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
2873
2874         gen5_gt_irq_reset(dev);
2875
2876         ibx_irq_reset(dev);
2877 }
2878
2879 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2880 {
2881         enum pipe pipe;
2882
2883         I915_WRITE(PORT_HOTPLUG_EN, 0);
2884         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2885
2886         for_each_pipe(dev_priv, pipe)
2887                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2888
2889         GEN5_IRQ_RESET(VLV_);
2890 }
2891
2892 static void valleyview_irq_preinstall(struct drm_device *dev)
2893 {
2894         struct drm_i915_private *dev_priv = dev->dev_private;
2895
2896         /* VLV magic */
2897         I915_WRITE(VLV_IMR, 0);
2898         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2899         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2900         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2901
2902         gen5_gt_irq_reset(dev);
2903
2904         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2905
2906         vlv_display_irq_reset(dev_priv);
2907 }
2908
2909 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
2910 {
2911         GEN8_IRQ_RESET_NDX(GT, 0);
2912         GEN8_IRQ_RESET_NDX(GT, 1);
2913         GEN8_IRQ_RESET_NDX(GT, 2);
2914         GEN8_IRQ_RESET_NDX(GT, 3);
2915 }
2916
2917 static void gen8_irq_reset(struct drm_device *dev)
2918 {
2919         struct drm_i915_private *dev_priv = dev->dev_private;
2920         int pipe;
2921
2922         I915_WRITE(GEN8_MASTER_IRQ, 0);
2923         POSTING_READ(GEN8_MASTER_IRQ);
2924
2925         gen8_gt_irq_reset(dev_priv);
2926
2927         for_each_pipe(dev_priv, pipe)
2928                 if (intel_display_power_is_enabled(dev_priv,
2929                                                    POWER_DOMAIN_PIPE(pipe)))
2930                         GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
2931
2932         GEN5_IRQ_RESET(GEN8_DE_PORT_);
2933         GEN5_IRQ_RESET(GEN8_DE_MISC_);
2934         GEN5_IRQ_RESET(GEN8_PCU_);
2935
2936         if (HAS_PCH_SPLIT(dev))
2937                 ibx_irq_reset(dev);
2938 }
2939
2940 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
2941                                      unsigned int pipe_mask)
2942 {
2943         uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
2944
2945         spin_lock_irq(&dev_priv->irq_lock);
2946         if (pipe_mask & 1 << PIPE_A)
2947                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
2948                                   dev_priv->de_irq_mask[PIPE_A],
2949                                   ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
2950         if (pipe_mask & 1 << PIPE_B)
2951                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
2952                                   dev_priv->de_irq_mask[PIPE_B],
2953                                   ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
2954         if (pipe_mask & 1 << PIPE_C)
2955                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
2956                                   dev_priv->de_irq_mask[PIPE_C],
2957                                   ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
2958         spin_unlock_irq(&dev_priv->irq_lock);
2959 }
2960
2961 static void cherryview_irq_preinstall(struct drm_device *dev)
2962 {
2963         struct drm_i915_private *dev_priv = dev->dev_private;
2964
2965         I915_WRITE(GEN8_MASTER_IRQ, 0);
2966         POSTING_READ(GEN8_MASTER_IRQ);
2967
2968         gen8_gt_irq_reset(dev_priv);
2969
2970         GEN5_IRQ_RESET(GEN8_PCU_);
2971
2972         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2973
2974         vlv_display_irq_reset(dev_priv);
2975 }
2976
2977 static void ibx_hpd_irq_setup(struct drm_device *dev)
2978 {
2979         struct drm_i915_private *dev_priv = dev->dev_private;
2980         struct intel_encoder *intel_encoder;
2981         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
2982
2983         if (HAS_PCH_IBX(dev)) {
2984                 hotplug_irqs = SDE_HOTPLUG_MASK;
2985                 for_each_intel_encoder(dev, intel_encoder)
2986                         if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
2987                                 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
2988         } else {
2989                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2990                 for_each_intel_encoder(dev, intel_encoder)
2991                         if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
2992                                 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
2993         }
2994
2995         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
2996
2997         /*
2998          * Enable digital hotplug on the PCH, and configure the DP short pulse
2999          * duration to 2ms (which is the minimum in the Display Port spec)
3000          *
3001          * This register is the same on all known PCH chips.
3002          */
3003         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3004         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3005         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3006         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3007         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3008         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3009 }
3010
3011 static void bxt_hpd_irq_setup(struct drm_device *dev)
3012 {
3013         struct drm_i915_private *dev_priv = dev->dev_private;
3014         struct intel_encoder *intel_encoder;
3015         u32 hotplug_port = 0;
3016         u32 hotplug_ctrl;
3017
3018         /* Now, enable HPD */
3019         for_each_intel_encoder(dev, intel_encoder) {
3020                 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state
3021                                 == HPD_ENABLED)
3022                         hotplug_port |= hpd_bxt[intel_encoder->hpd_pin];
3023         }
3024
3025         /* Mask all HPD control bits */
3026         hotplug_ctrl = I915_READ(BXT_HOTPLUG_CTL) & ~BXT_HOTPLUG_CTL_MASK;
3027
3028         /* Enable requested port in hotplug control */
3029         /* TODO: implement (short) HPD support on port A */
3030         WARN_ON_ONCE(hotplug_port & BXT_DE_PORT_HP_DDIA);
3031         if (hotplug_port & BXT_DE_PORT_HP_DDIB)
3032                 hotplug_ctrl |= BXT_DDIB_HPD_ENABLE;
3033         if (hotplug_port & BXT_DE_PORT_HP_DDIC)
3034                 hotplug_ctrl |= BXT_DDIC_HPD_ENABLE;
3035         I915_WRITE(BXT_HOTPLUG_CTL, hotplug_ctrl);
3036
3037         /* Unmask DDI hotplug in IMR */
3038         hotplug_ctrl = I915_READ(GEN8_DE_PORT_IMR) & ~hotplug_port;
3039         I915_WRITE(GEN8_DE_PORT_IMR, hotplug_ctrl);
3040
3041         /* Enable DDI hotplug in IER */
3042         hotplug_ctrl = I915_READ(GEN8_DE_PORT_IER) | hotplug_port;
3043         I915_WRITE(GEN8_DE_PORT_IER, hotplug_ctrl);
3044         POSTING_READ(GEN8_DE_PORT_IER);
3045 }
3046
3047 static void ibx_irq_postinstall(struct drm_device *dev)
3048 {
3049         struct drm_i915_private *dev_priv = dev->dev_private;
3050         u32 mask;
3051
3052         if (HAS_PCH_NOP(dev))
3053                 return;
3054
3055         if (HAS_PCH_IBX(dev))
3056                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3057         else
3058                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3059
3060         GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
3061         I915_WRITE(SDEIMR, ~mask);
3062 }
3063
3064 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3065 {
3066         struct drm_i915_private *dev_priv = dev->dev_private;
3067         u32 pm_irqs, gt_irqs;
3068
3069         pm_irqs = gt_irqs = 0;
3070
3071         dev_priv->gt_irq_mask = ~0;
3072         if (HAS_L3_DPF(dev)) {
3073                 /* L3 parity interrupt is always unmasked. */
3074                 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3075                 gt_irqs |= GT_PARITY_ERROR(dev);
3076         }
3077
3078         gt_irqs |= GT_RENDER_USER_INTERRUPT;
3079         if (IS_GEN5(dev)) {
3080                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3081                            ILK_BSD_USER_INTERRUPT;
3082         } else {
3083                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3084         }
3085
3086         GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3087
3088         if (INTEL_INFO(dev)->gen >= 6) {
3089                 /*
3090                  * RPS interrupts will get enabled/disabled on demand when RPS
3091                  * itself is enabled/disabled.
3092                  */
3093                 if (HAS_VEBOX(dev))
3094                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3095
3096                 dev_priv->pm_irq_mask = 0xffffffff;
3097                 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3098         }
3099 }
3100
3101 static int ironlake_irq_postinstall(struct drm_device *dev)
3102 {
3103         struct drm_i915_private *dev_priv = dev->dev_private;
3104         u32 display_mask, extra_mask;
3105
3106         if (INTEL_INFO(dev)->gen >= 7) {
3107                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3108                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3109                                 DE_PLANEB_FLIP_DONE_IVB |
3110                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3111                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3112                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
3113         } else {
3114                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3115                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3116                                 DE_AUX_CHANNEL_A |
3117                                 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3118                                 DE_POISON);
3119                 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3120                                 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
3121         }
3122
3123         dev_priv->irq_mask = ~display_mask;
3124
3125         I915_WRITE(HWSTAM, 0xeffe);
3126
3127         ibx_irq_pre_postinstall(dev);
3128
3129         GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3130
3131         gen5_gt_irq_postinstall(dev);
3132
3133         ibx_irq_postinstall(dev);
3134
3135         if (IS_IRONLAKE_M(dev)) {
3136                 /* Enable PCU event interrupts
3137                  *
3138                  * spinlocking not required here for correctness since interrupt
3139                  * setup is guaranteed to run in single-threaded context. But we
3140                  * need it to make the assert_spin_locked happy. */
3141                 spin_lock_irq(&dev_priv->irq_lock);
3142                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3143                 spin_unlock_irq(&dev_priv->irq_lock);
3144         }
3145
3146         return 0;
3147 }
3148
3149 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3150 {
3151         u32 pipestat_mask;
3152         u32 iir_mask;
3153         enum pipe pipe;
3154
3155         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3156                         PIPE_FIFO_UNDERRUN_STATUS;
3157
3158         for_each_pipe(dev_priv, pipe)
3159                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3160         POSTING_READ(PIPESTAT(PIPE_A));
3161
3162         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3163                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3164
3165         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3166         for_each_pipe(dev_priv, pipe)
3167                       i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3168
3169         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3170                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3171                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3172         if (IS_CHERRYVIEW(dev_priv))
3173                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3174         dev_priv->irq_mask &= ~iir_mask;
3175
3176         I915_WRITE(VLV_IIR, iir_mask);
3177         I915_WRITE(VLV_IIR, iir_mask);
3178         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3179         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3180         POSTING_READ(VLV_IMR);
3181 }
3182
3183 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3184 {
3185         u32 pipestat_mask;
3186         u32 iir_mask;
3187         enum pipe pipe;
3188
3189         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3190                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3191                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3192         if (IS_CHERRYVIEW(dev_priv))
3193                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3194
3195         dev_priv->irq_mask |= iir_mask;
3196         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3197         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3198         I915_WRITE(VLV_IIR, iir_mask);
3199         I915_WRITE(VLV_IIR, iir_mask);
3200         POSTING_READ(VLV_IIR);
3201
3202         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3203                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3204
3205         i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3206         for_each_pipe(dev_priv, pipe)
3207                 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3208
3209         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3210                         PIPE_FIFO_UNDERRUN_STATUS;
3211
3212         for_each_pipe(dev_priv, pipe)
3213                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3214         POSTING_READ(PIPESTAT(PIPE_A));
3215 }
3216
3217 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3218 {
3219         assert_spin_locked(&dev_priv->irq_lock);
3220
3221         if (dev_priv->display_irqs_enabled)
3222                 return;
3223
3224         dev_priv->display_irqs_enabled = true;
3225
3226         if (intel_irqs_enabled(dev_priv))
3227                 valleyview_display_irqs_install(dev_priv);
3228 }
3229
3230 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3231 {
3232         assert_spin_locked(&dev_priv->irq_lock);
3233
3234         if (!dev_priv->display_irqs_enabled)
3235                 return;
3236
3237         dev_priv->display_irqs_enabled = false;
3238
3239         if (intel_irqs_enabled(dev_priv))
3240                 valleyview_display_irqs_uninstall(dev_priv);
3241 }
3242
3243 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3244 {
3245         dev_priv->irq_mask = ~0;
3246
3247         I915_WRITE(PORT_HOTPLUG_EN, 0);
3248         POSTING_READ(PORT_HOTPLUG_EN);
3249
3250         I915_WRITE(VLV_IIR, 0xffffffff);
3251         I915_WRITE(VLV_IIR, 0xffffffff);
3252         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3253         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3254         POSTING_READ(VLV_IMR);
3255
3256         /* Interrupt setup is already guaranteed to be single-threaded, this is
3257          * just to make the assert_spin_locked check happy. */
3258         spin_lock_irq(&dev_priv->irq_lock);
3259         if (dev_priv->display_irqs_enabled)
3260                 valleyview_display_irqs_install(dev_priv);
3261         spin_unlock_irq(&dev_priv->irq_lock);
3262 }
3263
3264 static int valleyview_irq_postinstall(struct drm_device *dev)
3265 {
3266         struct drm_i915_private *dev_priv = dev->dev_private;
3267
3268         vlv_display_irq_postinstall(dev_priv);
3269
3270         gen5_gt_irq_postinstall(dev);
3271
3272         /* ack & enable invalid PTE error interrupts */
3273 #if 0 /* FIXME: add support to irq handler for checking these bits */
3274         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3275         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3276 #endif
3277
3278         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3279
3280         return 0;
3281 }
3282
3283 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3284 {
3285         /* These are interrupts we'll toggle with the ring mask register */
3286         uint32_t gt_interrupts[] = {
3287                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3288                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3289                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3290                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3291                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3292                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3293                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3294                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3295                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3296                 0,
3297                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3298                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3299                 };
3300
3301         dev_priv->pm_irq_mask = 0xffffffff;
3302         GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3303         GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3304         /*
3305          * RPS interrupts will get enabled/disabled on demand when RPS itself
3306          * is enabled/disabled.
3307          */
3308         GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3309         GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3310 }
3311
3312 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3313 {
3314         uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3315         uint32_t de_pipe_enables;
3316         int pipe;
3317         u32 de_port_en = GEN8_AUX_CHANNEL_A;
3318
3319         if (IS_GEN9(dev_priv)) {
3320                 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3321                                   GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3322                 de_port_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3323                         GEN9_AUX_CHANNEL_D;
3324
3325                 if (IS_BROXTON(dev_priv))
3326                         de_port_en |= BXT_DE_PORT_GMBUS;
3327         } else
3328                 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3329                                   GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3330
3331         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3332                                            GEN8_PIPE_FIFO_UNDERRUN;
3333
3334         dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3335         dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3336         dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3337
3338         for_each_pipe(dev_priv, pipe)
3339                 if (intel_display_power_is_enabled(dev_priv,
3340                                 POWER_DOMAIN_PIPE(pipe)))
3341                         GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3342                                           dev_priv->de_irq_mask[pipe],
3343                                           de_pipe_enables);
3344
3345         GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_en, de_port_en);
3346 }
3347
3348 static int gen8_irq_postinstall(struct drm_device *dev)
3349 {
3350         struct drm_i915_private *dev_priv = dev->dev_private;
3351
3352         if (HAS_PCH_SPLIT(dev))
3353                 ibx_irq_pre_postinstall(dev);
3354
3355         gen8_gt_irq_postinstall(dev_priv);
3356         gen8_de_irq_postinstall(dev_priv);
3357
3358         if (HAS_PCH_SPLIT(dev))
3359                 ibx_irq_postinstall(dev);
3360
3361         I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3362         POSTING_READ(GEN8_MASTER_IRQ);
3363
3364         return 0;
3365 }
3366
3367 static int cherryview_irq_postinstall(struct drm_device *dev)
3368 {
3369         struct drm_i915_private *dev_priv = dev->dev_private;
3370
3371         vlv_display_irq_postinstall(dev_priv);
3372
3373         gen8_gt_irq_postinstall(dev_priv);
3374
3375         I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3376         POSTING_READ(GEN8_MASTER_IRQ);
3377
3378         return 0;
3379 }
3380
3381 static void gen8_irq_uninstall(struct drm_device *dev)
3382 {
3383         struct drm_i915_private *dev_priv = dev->dev_private;
3384
3385         if (!dev_priv)
3386                 return;
3387
3388         gen8_irq_reset(dev);
3389 }
3390
3391 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3392 {
3393         /* Interrupt setup is already guaranteed to be single-threaded, this is
3394          * just to make the assert_spin_locked check happy. */
3395         spin_lock_irq(&dev_priv->irq_lock);
3396         if (dev_priv->display_irqs_enabled)
3397                 valleyview_display_irqs_uninstall(dev_priv);
3398         spin_unlock_irq(&dev_priv->irq_lock);
3399
3400         vlv_display_irq_reset(dev_priv);
3401
3402         dev_priv->irq_mask = ~0;
3403 }
3404
3405 static void valleyview_irq_uninstall(struct drm_device *dev)
3406 {
3407         struct drm_i915_private *dev_priv = dev->dev_private;
3408
3409         if (!dev_priv)
3410                 return;
3411
3412         I915_WRITE(VLV_MASTER_IER, 0);
3413
3414         gen5_gt_irq_reset(dev);
3415
3416         I915_WRITE(HWSTAM, 0xffffffff);
3417
3418         vlv_display_irq_uninstall(dev_priv);
3419 }
3420
3421 static void cherryview_irq_uninstall(struct drm_device *dev)
3422 {
3423         struct drm_i915_private *dev_priv = dev->dev_private;
3424
3425         if (!dev_priv)
3426                 return;
3427
3428         I915_WRITE(GEN8_MASTER_IRQ, 0);
3429         POSTING_READ(GEN8_MASTER_IRQ);
3430
3431         gen8_gt_irq_reset(dev_priv);
3432
3433         GEN5_IRQ_RESET(GEN8_PCU_);
3434
3435         vlv_display_irq_uninstall(dev_priv);
3436 }
3437
3438 static void ironlake_irq_uninstall(struct drm_device *dev)
3439 {
3440         struct drm_i915_private *dev_priv = dev->dev_private;
3441
3442         if (!dev_priv)
3443                 return;
3444
3445         ironlake_irq_reset(dev);
3446 }
3447
3448 static void i8xx_irq_preinstall(struct drm_device * dev)
3449 {
3450         struct drm_i915_private *dev_priv = dev->dev_private;
3451         int pipe;
3452
3453         for_each_pipe(dev_priv, pipe)
3454                 I915_WRITE(PIPESTAT(pipe), 0);
3455         I915_WRITE16(IMR, 0xffff);
3456         I915_WRITE16(IER, 0x0);
3457         POSTING_READ16(IER);
3458 }
3459
3460 static int i8xx_irq_postinstall(struct drm_device *dev)
3461 {
3462         struct drm_i915_private *dev_priv = dev->dev_private;
3463
3464         I915_WRITE16(EMR,
3465                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3466
3467         /* Unmask the interrupts that we always want on. */
3468         dev_priv->irq_mask =
3469                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3470                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3471                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3472                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3473         I915_WRITE16(IMR, dev_priv->irq_mask);
3474
3475         I915_WRITE16(IER,
3476                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3477                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3478                      I915_USER_INTERRUPT);
3479         POSTING_READ16(IER);
3480
3481         /* Interrupt setup is already guaranteed to be single-threaded, this is
3482          * just to make the assert_spin_locked check happy. */
3483         spin_lock_irq(&dev_priv->irq_lock);
3484         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3485         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3486         spin_unlock_irq(&dev_priv->irq_lock);
3487
3488         return 0;
3489 }
3490
3491 /*
3492  * Returns true when a page flip has completed.
3493  */
3494 static bool i8xx_handle_vblank(struct drm_device *dev,
3495                                int plane, int pipe, u32 iir)
3496 {
3497         struct drm_i915_private *dev_priv = dev->dev_private;
3498         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3499
3500         if (!intel_pipe_handle_vblank(dev, pipe))
3501                 return false;
3502
3503         if ((iir & flip_pending) == 0)
3504                 goto check_page_flip;
3505
3506         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3507          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3508          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3509          * the flip is completed (no longer pending). Since this doesn't raise
3510          * an interrupt per se, we watch for the change at vblank.
3511          */
3512         if (I915_READ16(ISR) & flip_pending)
3513                 goto check_page_flip;
3514
3515         intel_prepare_page_flip(dev, plane);
3516         intel_finish_page_flip(dev, pipe);
3517         return true;
3518
3519 check_page_flip:
3520         intel_check_page_flip(dev, pipe);
3521         return false;
3522 }
3523
3524 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3525 {
3526         struct drm_device *dev = arg;
3527         struct drm_i915_private *dev_priv = dev->dev_private;
3528         u16 iir, new_iir;
3529         u32 pipe_stats[2];
3530         int pipe;
3531         u16 flip_mask =
3532                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3533                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3534
3535         if (!intel_irqs_enabled(dev_priv))
3536                 return IRQ_NONE;
3537
3538         iir = I915_READ16(IIR);
3539         if (iir == 0)
3540                 return IRQ_NONE;
3541
3542         while (iir & ~flip_mask) {
3543                 /* Can't rely on pipestat interrupt bit in iir as it might
3544                  * have been cleared after the pipestat interrupt was received.
3545                  * It doesn't set the bit in iir again, but it still produces
3546                  * interrupts (for non-MSI).
3547                  */
3548                 spin_lock(&dev_priv->irq_lock);
3549                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3550                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3551
3552                 for_each_pipe(dev_priv, pipe) {
3553                         int reg = PIPESTAT(pipe);
3554                         pipe_stats[pipe] = I915_READ(reg);
3555
3556                         /*
3557                          * Clear the PIPE*STAT regs before the IIR
3558                          */
3559                         if (pipe_stats[pipe] & 0x8000ffff)
3560                                 I915_WRITE(reg, pipe_stats[pipe]);
3561                 }
3562                 spin_unlock(&dev_priv->irq_lock);
3563
3564                 I915_WRITE16(IIR, iir & ~flip_mask);
3565                 new_iir = I915_READ16(IIR); /* Flush posted writes */
3566
3567                 if (iir & I915_USER_INTERRUPT)
3568                         notify_ring(&dev_priv->ring[RCS]);
3569
3570                 for_each_pipe(dev_priv, pipe) {
3571                         int plane = pipe;
3572                         if (HAS_FBC(dev))
3573                                 plane = !plane;
3574
3575                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3576                             i8xx_handle_vblank(dev, plane, pipe, iir))
3577                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3578
3579                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3580                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3581
3582                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3583                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3584                                                                     pipe);
3585                 }
3586
3587                 iir = new_iir;
3588         }
3589
3590         return IRQ_HANDLED;
3591 }
3592
3593 static void i8xx_irq_uninstall(struct drm_device * dev)
3594 {
3595         struct drm_i915_private *dev_priv = dev->dev_private;
3596         int pipe;
3597
3598         for_each_pipe(dev_priv, pipe) {
3599                 /* Clear enable bits; then clear status bits */
3600                 I915_WRITE(PIPESTAT(pipe), 0);
3601                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3602         }
3603         I915_WRITE16(IMR, 0xffff);
3604         I915_WRITE16(IER, 0x0);
3605         I915_WRITE16(IIR, I915_READ16(IIR));
3606 }
3607
3608 static void i915_irq_preinstall(struct drm_device * dev)
3609 {
3610         struct drm_i915_private *dev_priv = dev->dev_private;
3611         int pipe;
3612
3613         if (I915_HAS_HOTPLUG(dev)) {
3614                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3615                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3616         }
3617
3618         I915_WRITE16(HWSTAM, 0xeffe);
3619         for_each_pipe(dev_priv, pipe)
3620                 I915_WRITE(PIPESTAT(pipe), 0);
3621         I915_WRITE(IMR, 0xffffffff);
3622         I915_WRITE(IER, 0x0);
3623         POSTING_READ(IER);
3624 }
3625
3626 static int i915_irq_postinstall(struct drm_device *dev)
3627 {
3628         struct drm_i915_private *dev_priv = dev->dev_private;
3629         u32 enable_mask;
3630
3631         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3632
3633         /* Unmask the interrupts that we always want on. */
3634         dev_priv->irq_mask =
3635                 ~(I915_ASLE_INTERRUPT |
3636                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3637                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3638                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3639                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3640
3641         enable_mask =
3642                 I915_ASLE_INTERRUPT |
3643                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3644                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3645                 I915_USER_INTERRUPT;
3646
3647         if (I915_HAS_HOTPLUG(dev)) {
3648                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3649                 POSTING_READ(PORT_HOTPLUG_EN);
3650
3651                 /* Enable in IER... */
3652                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3653                 /* and unmask in IMR */
3654                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3655         }
3656
3657         I915_WRITE(IMR, dev_priv->irq_mask);
3658         I915_WRITE(IER, enable_mask);
3659         POSTING_READ(IER);
3660
3661         i915_enable_asle_pipestat(dev);
3662
3663         /* Interrupt setup is already guaranteed to be single-threaded, this is
3664          * just to make the assert_spin_locked check happy. */
3665         spin_lock_irq(&dev_priv->irq_lock);
3666         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3667         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3668         spin_unlock_irq(&dev_priv->irq_lock);
3669
3670         return 0;
3671 }
3672
3673 /*
3674  * Returns true when a page flip has completed.
3675  */
3676 static bool i915_handle_vblank(struct drm_device *dev,
3677                                int plane, int pipe, u32 iir)
3678 {
3679         struct drm_i915_private *dev_priv = dev->dev_private;
3680         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3681
3682         if (!intel_pipe_handle_vblank(dev, pipe))
3683                 return false;
3684
3685         if ((iir & flip_pending) == 0)
3686                 goto check_page_flip;
3687
3688         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3689          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3690          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3691          * the flip is completed (no longer pending). Since this doesn't raise
3692          * an interrupt per se, we watch for the change at vblank.
3693          */
3694         if (I915_READ(ISR) & flip_pending)
3695                 goto check_page_flip;
3696
3697         intel_prepare_page_flip(dev, plane);
3698         intel_finish_page_flip(dev, pipe);
3699         return true;
3700
3701 check_page_flip:
3702         intel_check_page_flip(dev, pipe);
3703         return false;
3704 }
3705
3706 static irqreturn_t i915_irq_handler(int irq, void *arg)
3707 {
3708         struct drm_device *dev = arg;
3709         struct drm_i915_private *dev_priv = dev->dev_private;
3710         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3711         u32 flip_mask =
3712                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3713                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3714         int pipe, ret = IRQ_NONE;
3715
3716         if (!intel_irqs_enabled(dev_priv))
3717                 return IRQ_NONE;
3718
3719         iir = I915_READ(IIR);
3720         do {
3721                 bool irq_received = (iir & ~flip_mask) != 0;
3722                 bool blc_event = false;
3723
3724                 /* Can't rely on pipestat interrupt bit in iir as it might
3725                  * have been cleared after the pipestat interrupt was received.
3726                  * It doesn't set the bit in iir again, but it still produces
3727                  * interrupts (for non-MSI).
3728                  */
3729                 spin_lock(&dev_priv->irq_lock);
3730                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3731                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3732
3733                 for_each_pipe(dev_priv, pipe) {
3734                         int reg = PIPESTAT(pipe);
3735                         pipe_stats[pipe] = I915_READ(reg);
3736
3737                         /* Clear the PIPE*STAT regs before the IIR */
3738                         if (pipe_stats[pipe] & 0x8000ffff) {
3739                                 I915_WRITE(reg, pipe_stats[pipe]);
3740                                 irq_received = true;
3741                         }
3742                 }
3743                 spin_unlock(&dev_priv->irq_lock);
3744
3745                 if (!irq_received)
3746                         break;
3747
3748                 /* Consume port.  Then clear IIR or we'll miss events */
3749                 if (I915_HAS_HOTPLUG(dev) &&
3750                     iir & I915_DISPLAY_PORT_INTERRUPT)
3751                         i9xx_hpd_irq_handler(dev);
3752
3753                 I915_WRITE(IIR, iir & ~flip_mask);
3754                 new_iir = I915_READ(IIR); /* Flush posted writes */
3755
3756                 if (iir & I915_USER_INTERRUPT)
3757                         notify_ring(&dev_priv->ring[RCS]);
3758
3759                 for_each_pipe(dev_priv, pipe) {
3760                         int plane = pipe;
3761                         if (HAS_FBC(dev))
3762                                 plane = !plane;
3763
3764                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3765                             i915_handle_vblank(dev, plane, pipe, iir))
3766                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3767
3768                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3769                                 blc_event = true;
3770
3771                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3772                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3773
3774                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3775                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3776                                                                     pipe);
3777                 }
3778
3779                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3780                         intel_opregion_asle_intr(dev);
3781
3782                 /* With MSI, interrupts are only generated when iir
3783                  * transitions from zero to nonzero.  If another bit got
3784                  * set while we were handling the existing iir bits, then
3785                  * we would never get another interrupt.
3786                  *
3787                  * This is fine on non-MSI as well, as if we hit this path
3788                  * we avoid exiting the interrupt handler only to generate
3789                  * another one.
3790                  *
3791                  * Note that for MSI this could cause a stray interrupt report
3792                  * if an interrupt landed in the time between writing IIR and
3793                  * the posting read.  This should be rare enough to never
3794                  * trigger the 99% of 100,000 interrupts test for disabling
3795                  * stray interrupts.
3796                  */
3797                 ret = IRQ_HANDLED;
3798                 iir = new_iir;
3799         } while (iir & ~flip_mask);
3800
3801         return ret;
3802 }
3803
3804 static void i915_irq_uninstall(struct drm_device * dev)
3805 {
3806         struct drm_i915_private *dev_priv = dev->dev_private;
3807         int pipe;
3808
3809         if (I915_HAS_HOTPLUG(dev)) {
3810                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3811                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3812         }
3813
3814         I915_WRITE16(HWSTAM, 0xffff);
3815         for_each_pipe(dev_priv, pipe) {
3816                 /* Clear enable bits; then clear status bits */
3817                 I915_WRITE(PIPESTAT(pipe), 0);
3818                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3819         }
3820         I915_WRITE(IMR, 0xffffffff);
3821         I915_WRITE(IER, 0x0);
3822
3823         I915_WRITE(IIR, I915_READ(IIR));
3824 }
3825
3826 static void i965_irq_preinstall(struct drm_device * dev)
3827 {
3828         struct drm_i915_private *dev_priv = dev->dev_private;
3829         int pipe;
3830
3831         I915_WRITE(PORT_HOTPLUG_EN, 0);
3832         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3833
3834         I915_WRITE(HWSTAM, 0xeffe);
3835         for_each_pipe(dev_priv, pipe)
3836                 I915_WRITE(PIPESTAT(pipe), 0);
3837         I915_WRITE(IMR, 0xffffffff);
3838         I915_WRITE(IER, 0x0);
3839         POSTING_READ(IER);
3840 }
3841
3842 static int i965_irq_postinstall(struct drm_device *dev)
3843 {
3844         struct drm_i915_private *dev_priv = dev->dev_private;
3845         u32 enable_mask;
3846         u32 error_mask;
3847
3848         /* Unmask the interrupts that we always want on. */
3849         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
3850                                I915_DISPLAY_PORT_INTERRUPT |
3851                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3852                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3853                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3854                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3855                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3856
3857         enable_mask = ~dev_priv->irq_mask;
3858         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3859                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3860         enable_mask |= I915_USER_INTERRUPT;
3861
3862         if (IS_G4X(dev))
3863                 enable_mask |= I915_BSD_USER_INTERRUPT;
3864
3865         /* Interrupt setup is already guaranteed to be single-threaded, this is
3866          * just to make the assert_spin_locked check happy. */
3867         spin_lock_irq(&dev_priv->irq_lock);
3868         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3869         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3870         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3871         spin_unlock_irq(&dev_priv->irq_lock);
3872
3873         /*
3874          * Enable some error detection, note the instruction error mask
3875          * bit is reserved, so we leave it masked.
3876          */
3877         if (IS_G4X(dev)) {
3878                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3879                                GM45_ERROR_MEM_PRIV |
3880                                GM45_ERROR_CP_PRIV |
3881                                I915_ERROR_MEMORY_REFRESH);
3882         } else {
3883                 error_mask = ~(I915_ERROR_PAGE_TABLE |
3884                                I915_ERROR_MEMORY_REFRESH);
3885         }
3886         I915_WRITE(EMR, error_mask);
3887
3888         I915_WRITE(IMR, dev_priv->irq_mask);
3889         I915_WRITE(IER, enable_mask);
3890         POSTING_READ(IER);
3891
3892         I915_WRITE(PORT_HOTPLUG_EN, 0);
3893         POSTING_READ(PORT_HOTPLUG_EN);
3894
3895         i915_enable_asle_pipestat(dev);
3896
3897         return 0;
3898 }
3899
3900 static void i915_hpd_irq_setup(struct drm_device *dev)
3901 {
3902         struct drm_i915_private *dev_priv = dev->dev_private;
3903         struct intel_encoder *intel_encoder;
3904         u32 hotplug_en;
3905
3906         assert_spin_locked(&dev_priv->irq_lock);
3907
3908         hotplug_en = I915_READ(PORT_HOTPLUG_EN);
3909         hotplug_en &= ~HOTPLUG_INT_EN_MASK;
3910         /* Note HDMI and DP share hotplug bits */
3911         /* enable bits are the same for all generations */
3912         for_each_intel_encoder(dev, intel_encoder)
3913                 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
3914                         hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
3915         /* Programming the CRT detection parameters tends
3916            to generate a spurious hotplug event about three
3917            seconds later.  So just do it once.
3918         */
3919         if (IS_G4X(dev))
3920                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
3921         hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
3922         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3923
3924         /* Ignore TV since it's buggy */
3925         I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
3926 }
3927
3928 static irqreturn_t i965_irq_handler(int irq, void *arg)
3929 {
3930         struct drm_device *dev = arg;
3931         struct drm_i915_private *dev_priv = dev->dev_private;
3932         u32 iir, new_iir;
3933         u32 pipe_stats[I915_MAX_PIPES];
3934         int ret = IRQ_NONE, pipe;
3935         u32 flip_mask =
3936                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3937                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3938
3939         if (!intel_irqs_enabled(dev_priv))
3940                 return IRQ_NONE;
3941
3942         iir = I915_READ(IIR);
3943
3944         for (;;) {
3945                 bool irq_received = (iir & ~flip_mask) != 0;
3946                 bool blc_event = false;
3947
3948                 /* Can't rely on pipestat interrupt bit in iir as it might
3949                  * have been cleared after the pipestat interrupt was received.
3950                  * It doesn't set the bit in iir again, but it still produces
3951                  * interrupts (for non-MSI).
3952                  */
3953                 spin_lock(&dev_priv->irq_lock);
3954                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3955                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3956
3957                 for_each_pipe(dev_priv, pipe) {
3958                         int reg = PIPESTAT(pipe);
3959                         pipe_stats[pipe] = I915_READ(reg);
3960
3961                         /*
3962                          * Clear the PIPE*STAT regs before the IIR
3963                          */
3964                         if (pipe_stats[pipe] & 0x8000ffff) {
3965                                 I915_WRITE(reg, pipe_stats[pipe]);
3966                                 irq_received = true;
3967                         }
3968                 }
3969                 spin_unlock(&dev_priv->irq_lock);
3970
3971                 if (!irq_received)
3972                         break;
3973
3974                 ret = IRQ_HANDLED;
3975
3976                 /* Consume port.  Then clear IIR or we'll miss events */
3977                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
3978                         i9xx_hpd_irq_handler(dev);
3979
3980                 I915_WRITE(IIR, iir & ~flip_mask);
3981                 new_iir = I915_READ(IIR); /* Flush posted writes */
3982
3983                 if (iir & I915_USER_INTERRUPT)
3984                         notify_ring(&dev_priv->ring[RCS]);
3985                 if (iir & I915_BSD_USER_INTERRUPT)
3986                         notify_ring(&dev_priv->ring[VCS]);
3987
3988                 for_each_pipe(dev_priv, pipe) {
3989                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
3990                             i915_handle_vblank(dev, pipe, pipe, iir))
3991                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
3992
3993                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3994                                 blc_event = true;
3995
3996                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3997                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3998
3999                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4000                                 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4001                 }
4002
4003                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4004                         intel_opregion_asle_intr(dev);
4005
4006                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4007                         gmbus_irq_handler(dev);
4008
4009                 /* With MSI, interrupts are only generated when iir
4010                  * transitions from zero to nonzero.  If another bit got
4011                  * set while we were handling the existing iir bits, then
4012                  * we would never get another interrupt.
4013                  *
4014                  * This is fine on non-MSI as well, as if we hit this path
4015                  * we avoid exiting the interrupt handler only to generate
4016                  * another one.
4017                  *
4018                  * Note that for MSI this could cause a stray interrupt report
4019                  * if an interrupt landed in the time between writing IIR and
4020                  * the posting read.  This should be rare enough to never
4021                  * trigger the 99% of 100,000 interrupts test for disabling
4022                  * stray interrupts.
4023                  */
4024                 iir = new_iir;
4025         }
4026
4027         return ret;
4028 }
4029
4030 static void i965_irq_uninstall(struct drm_device * dev)
4031 {
4032         struct drm_i915_private *dev_priv = dev->dev_private;
4033         int pipe;
4034
4035         if (!dev_priv)
4036                 return;
4037
4038         I915_WRITE(PORT_HOTPLUG_EN, 0);
4039         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4040
4041         I915_WRITE(HWSTAM, 0xffffffff);
4042         for_each_pipe(dev_priv, pipe)
4043                 I915_WRITE(PIPESTAT(pipe), 0);
4044         I915_WRITE(IMR, 0xffffffff);
4045         I915_WRITE(IER, 0x0);
4046
4047         for_each_pipe(dev_priv, pipe)
4048                 I915_WRITE(PIPESTAT(pipe),
4049                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4050         I915_WRITE(IIR, I915_READ(IIR));
4051 }
4052
4053 /**
4054  * intel_irq_init - initializes irq support
4055  * @dev_priv: i915 device instance
4056  *
4057  * This function initializes all the irq support including work items, timers
4058  * and all the vtables. It does not setup the interrupt itself though.
4059  */
4060 void intel_irq_init(struct drm_i915_private *dev_priv)
4061 {
4062         struct drm_device *dev = dev_priv->dev;
4063
4064         intel_hpd_init_work(dev_priv);
4065
4066         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4067         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4068
4069         /* Let's track the enabled rps events */
4070         if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4071                 /* WaGsvRC0ResidencyMethod:vlv */
4072                 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4073         else
4074                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4075
4076         INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4077                           i915_hangcheck_elapsed);
4078
4079         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4080
4081         if (IS_GEN2(dev_priv)) {
4082                 dev->max_vblank_count = 0;
4083                 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4084         } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4085                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4086                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4087         } else {
4088                 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4089                 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4090         }
4091
4092         /*
4093          * Opt out of the vblank disable timer on everything except gen2.
4094          * Gen2 doesn't have a hardware frame counter and so depends on
4095          * vblank interrupts to produce sane vblank seuquence numbers.
4096          */
4097         if (!IS_GEN2(dev_priv))
4098                 dev->vblank_disable_immediate = true;
4099
4100         dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4101         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4102
4103         if (IS_CHERRYVIEW(dev_priv)) {
4104                 dev->driver->irq_handler = cherryview_irq_handler;
4105                 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4106                 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4107                 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4108                 dev->driver->enable_vblank = valleyview_enable_vblank;
4109                 dev->driver->disable_vblank = valleyview_disable_vblank;
4110                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4111         } else if (IS_VALLEYVIEW(dev_priv)) {
4112                 dev->driver->irq_handler = valleyview_irq_handler;
4113                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4114                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4115                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4116                 dev->driver->enable_vblank = valleyview_enable_vblank;
4117                 dev->driver->disable_vblank = valleyview_disable_vblank;
4118                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4119         } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4120                 dev->driver->irq_handler = gen8_irq_handler;
4121                 dev->driver->irq_preinstall = gen8_irq_reset;
4122                 dev->driver->irq_postinstall = gen8_irq_postinstall;
4123                 dev->driver->irq_uninstall = gen8_irq_uninstall;
4124                 dev->driver->enable_vblank = gen8_enable_vblank;
4125                 dev->driver->disable_vblank = gen8_disable_vblank;
4126                 if (HAS_PCH_SPLIT(dev))
4127                         dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4128                 else
4129                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4130         } else if (HAS_PCH_SPLIT(dev)) {
4131                 dev->driver->irq_handler = ironlake_irq_handler;
4132                 dev->driver->irq_preinstall = ironlake_irq_reset;
4133                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4134                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4135                 dev->driver->enable_vblank = ironlake_enable_vblank;
4136                 dev->driver->disable_vblank = ironlake_disable_vblank;
4137                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4138         } else {
4139                 if (INTEL_INFO(dev_priv)->gen == 2) {
4140                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
4141                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
4142                         dev->driver->irq_handler = i8xx_irq_handler;
4143                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
4144                 } else if (INTEL_INFO(dev_priv)->gen == 3) {
4145                         dev->driver->irq_preinstall = i915_irq_preinstall;
4146                         dev->driver->irq_postinstall = i915_irq_postinstall;
4147                         dev->driver->irq_uninstall = i915_irq_uninstall;
4148                         dev->driver->irq_handler = i915_irq_handler;
4149                 } else {
4150                         dev->driver->irq_preinstall = i965_irq_preinstall;
4151                         dev->driver->irq_postinstall = i965_irq_postinstall;
4152                         dev->driver->irq_uninstall = i965_irq_uninstall;
4153                         dev->driver->irq_handler = i965_irq_handler;
4154                 }
4155                 if (I915_HAS_HOTPLUG(dev_priv))
4156                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4157                 dev->driver->enable_vblank = i915_enable_vblank;
4158                 dev->driver->disable_vblank = i915_disable_vblank;
4159         }
4160 }
4161
4162 /**
4163  * intel_irq_install - enables the hardware interrupt
4164  * @dev_priv: i915 device instance
4165  *
4166  * This function enables the hardware interrupt handling, but leaves the hotplug
4167  * handling still disabled. It is called after intel_irq_init().
4168  *
4169  * In the driver load and resume code we need working interrupts in a few places
4170  * but don't want to deal with the hassle of concurrent probe and hotplug
4171  * workers. Hence the split into this two-stage approach.
4172  */
4173 int intel_irq_install(struct drm_i915_private *dev_priv)
4174 {
4175         /*
4176          * We enable some interrupt sources in our postinstall hooks, so mark
4177          * interrupts as enabled _before_ actually enabling them to avoid
4178          * special cases in our ordering checks.
4179          */
4180         dev_priv->pm.irqs_enabled = true;
4181
4182         return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4183 }
4184
4185 /**
4186  * intel_irq_uninstall - finilizes all irq handling
4187  * @dev_priv: i915 device instance
4188  *
4189  * This stops interrupt and hotplug handling and unregisters and frees all
4190  * resources acquired in the init functions.
4191  */
4192 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4193 {
4194         drm_irq_uninstall(dev_priv->dev);
4195         intel_hpd_cancel_work(dev_priv);
4196         dev_priv->pm.irqs_enabled = false;
4197 }
4198
4199 /**
4200  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4201  * @dev_priv: i915 device instance
4202  *
4203  * This function is used to disable interrupts at runtime, both in the runtime
4204  * pm and the system suspend/resume code.
4205  */
4206 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4207 {
4208         dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4209         dev_priv->pm.irqs_enabled = false;
4210         synchronize_irq(dev_priv->dev->irq);
4211 }
4212
4213 /**
4214  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4215  * @dev_priv: i915 device instance
4216  *
4217  * This function is used to enable interrupts at runtime, both in the runtime
4218  * pm and the system suspend/resume code.
4219  */
4220 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4221 {
4222         dev_priv->pm.irqs_enabled = true;
4223         dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4224         dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4225 }