drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26
  27 #define FORCEWAKE_ACK_TIMEOUT_MS 2
  28
  29 #define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
  30 #define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
  31
  32 #define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
  33 #define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
  34
  35 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
  36 #define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
  37
  38 #define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
  39 #define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
  40
  41 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
  42
  43
  44 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
  45 {
  46         u32 gt_thread_status_mask;
  47
  48         if (IS_HASWELL(dev_priv->dev))
  49                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
  50         else
  51                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
  52
  53         /* w/a for a sporadic read returning 0 by waiting for the GT
  54          * thread to wake up.
  55          */
  56         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
  57                 DRM_ERROR("GT thread status wait timed out\n");
  58 }
  59
  60 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
  61 {
  62         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
  63         /* something from same cacheline, but !FORCEWAKE */
  64         __raw_posting_read(dev_priv, ECOBUS);
  65 }
  66
  67 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
  68                                                         int fw_engine)
  69 {
  70         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
  71                             FORCEWAKE_ACK_TIMEOUT_MS))
  72                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
  73
  74         __raw_i915_write32(dev_priv, FORCEWAKE, 1);
  75         /* something from same cacheline, but !FORCEWAKE */
  76         __raw_posting_read(dev_priv, ECOBUS);
  77
  78         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1),
  79                             FORCEWAKE_ACK_TIMEOUT_MS))
  80                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
  81
  82         /* WaRsForcewakeWaitTC0:snb */
  83         __gen6_gt_wait_for_thread_c0(dev_priv);
  84 }
  85
  86 static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
  87 {
  88         __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
  89         /* something from same cacheline, but !FORCEWAKE_MT */
  90         __raw_posting_read(dev_priv, ECOBUS);
  91 }
  92
  93 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
  94                                                         int fw_engine)
  95 {
  96         u32 forcewake_ack;
  97
  98         if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev))
  99                 forcewake_ack = FORCEWAKE_ACK_HSW;
 100         else
 101                 forcewake_ack = FORCEWAKE_MT_ACK;
 102
 103         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0,
 104                             FORCEWAKE_ACK_TIMEOUT_MS))
 105                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 106
 107         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 108                            _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 109         /* something from same cacheline, but !FORCEWAKE_MT */
 110         __raw_posting_read(dev_priv, ECOBUS);
 111
 112         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL),
 113                             FORCEWAKE_ACK_TIMEOUT_MS))
 114                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 115
 116         /* WaRsForcewakeWaitTC0:ivb,hsw */
 117         if (INTEL_INFO(dev_priv->dev)->gen < 8)
 118                 __gen6_gt_wait_for_thread_c0(dev_priv);
 119 }
 120
 121 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 122 {
 123         u32 gtfifodbg;
 124
 125         gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 126         if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
 127                 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
 128 }
 129
 130 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 131                                                         int fw_engine)
 132 {
 133         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
 134         /* something from same cacheline, but !FORCEWAKE */
 135         __raw_posting_read(dev_priv, ECOBUS);
 136         gen6_gt_check_fifodbg(dev_priv);
 137 }
 138
 139 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
 140                                                         int fw_engine)
 141 {
 142         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 143                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 144         /* something from same cacheline, but !FORCEWAKE_MT */
 145         __raw_posting_read(dev_priv, ECOBUS);
 146         gen6_gt_check_fifodbg(dev_priv);
 147 }
 148
 149 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 150 {
 151         int ret = 0;
 152
 153         /* On VLV, FIFO will be shared by both SW and HW.
 154          * So, we need to read the FREE_ENTRIES everytime */
 155         if (IS_VALLEYVIEW(dev_priv->dev))
 156                 dev_priv->uncore.fifo_count =
 157                         __raw_i915_read32(dev_priv, GTFIFOCTL) &
 158                                                 GT_FIFO_FREE_ENTRIES_MASK;
 159
 160         if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 161                 int loop = 500;
 162                 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 163                 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 164                         udelay(10);
 165                         fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 166                 }
 167                 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
 168                         ++ret;
 169                 dev_priv->uncore.fifo_count = fifo;
 170         }
 171         dev_priv->uncore.fifo_count--;
 172
 173         return ret;
 174 }
 175
 176 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 177 {
 178         __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 179                            _MASKED_BIT_DISABLE(0xffff));
 180         /* something from same cacheline, but !FORCEWAKE_VLV */
 181         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 182 }
 183
 184 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 185                                                 int fw_engine)
 186 {
 187         /* Check for Render Engine */
 188         if (FORCEWAKE_RENDER & fw_engine) {
 189                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 190                                                 FORCEWAKE_ACK_VLV) &
 191                                                 FORCEWAKE_KERNEL) == 0,
 192                                         FORCEWAKE_ACK_TIMEOUT_MS))
 193                         DRM_ERROR("Timed out: Render forcewake old ack to clear.\n");
 194
 195                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 196                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 197
 198                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 199                                                 FORCEWAKE_ACK_VLV) &
 200                                                 FORCEWAKE_KERNEL),
 201                                         FORCEWAKE_ACK_TIMEOUT_MS))
 202                         DRM_ERROR("Timed out: waiting for Render to ack.\n");
 203         }
 204
 205         /* Check for Media Engine */
 206         if (FORCEWAKE_MEDIA & fw_engine) {
 207                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 208                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 209                                                 FORCEWAKE_KERNEL) == 0,
 210                                         FORCEWAKE_ACK_TIMEOUT_MS))
 211                         DRM_ERROR("Timed out: Media forcewake old ack to clear.\n");
 212
 213                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 214                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 215
 216                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 217                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 218                                                 FORCEWAKE_KERNEL),
 219                                         FORCEWAKE_ACK_TIMEOUT_MS))
 220                         DRM_ERROR("Timed out: waiting for media to ack.\n");
 221         }
 222
 223         /* WaRsForcewakeWaitTC0:vlv */
 224         __gen6_gt_wait_for_thread_c0(dev_priv);
 225
 226 }
 227
 228 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
 229                                         int fw_engine)
 230 {
 231
 232         /* Check for Render Engine */
 233         if (FORCEWAKE_RENDER & fw_engine)
 234                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 235                                         _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 236
 237
 238         /* Check for Media Engine */
 239         if (FORCEWAKE_MEDIA & fw_engine)
 240                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 241                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 242
 243         /* The below doubles as a POSTING_READ */
 244         gen6_gt_check_fifodbg(dev_priv);
 245
 246 }
 247
 248 void vlv_force_wake_get(struct drm_i915_private *dev_priv,
 249                                                 int fw_engine)
 250 {
 251         unsigned long irqflags;
 252
 253         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 254         if (FORCEWAKE_RENDER & fw_engine) {
 255                 if (dev_priv->uncore.fw_rendercount++ == 0)
 256                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 257                                                         FORCEWAKE_RENDER);
 258         }
 259         if (FORCEWAKE_MEDIA & fw_engine) {
 260                 if (dev_priv->uncore.fw_mediacount++ == 0)
 261                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 262                                                         FORCEWAKE_MEDIA);
 263         }
 264
 265         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 266 }
 267
 268 void vlv_force_wake_put(struct drm_i915_private *dev_priv,
 269                                                 int fw_engine)
 270 {
 271         unsigned long irqflags;
 272
 273         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 274
 275         if (FORCEWAKE_RENDER & fw_engine) {
 276                 WARN_ON(dev_priv->uncore.fw_rendercount == 0);
 277                 if (--dev_priv->uncore.fw_rendercount == 0)
 278                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 279                                                         FORCEWAKE_RENDER);
 280         }
 281
 282         if (FORCEWAKE_MEDIA & fw_engine) {
 283                 WARN_ON(dev_priv->uncore.fw_mediacount == 0);
 284                 if (--dev_priv->uncore.fw_mediacount == 0)
 285                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 286                                                         FORCEWAKE_MEDIA);
 287         }
 288
 289         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 290 }
 291
 292 static void gen6_force_wake_work(struct work_struct *work)
 293 {
 294         struct drm_i915_private *dev_priv =
 295                 container_of(work, typeof(*dev_priv), uncore.force_wake_work.work);
 296         unsigned long irqflags;
 297
 298         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 299         if (--dev_priv->uncore.forcewake_count == 0)
 300                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 301         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 302 }
 303
 304 static void intel_uncore_forcewake_reset(struct drm_device *dev)
 305 {
 306         struct drm_i915_private *dev_priv = dev->dev_private;
 307
 308         if (IS_VALLEYVIEW(dev)) {
 309                 vlv_force_wake_reset(dev_priv);
 310         } else if (INTEL_INFO(dev)->gen >= 6) {
 311                 __gen6_gt_force_wake_reset(dev_priv);
 312                 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
 313                         __gen6_gt_force_wake_mt_reset(dev_priv);
 314         }
 315 }
 316
 317 void intel_uncore_early_sanitize(struct drm_device *dev)
 318 {
 319         struct drm_i915_private *dev_priv = dev->dev_private;
 320
 321         if (HAS_FPGA_DBG_UNCLAIMED(dev))
 322                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 323
 324         if (IS_HASWELL(dev) &&
 325             (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) {
 326                 /* The docs do not explain exactly how the calculation can be
 327                  * made. It is somewhat guessable, but for now, it's always
 328                  * 128MB.
 329                  * NB: We can't write IDICR yet because we do not have gt funcs
 330                  * set up */
 331                 dev_priv->ellc_size = 128;
 332                 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size);
 333         }
 334
 335         /* clear out old GT FIFO errors */
 336         if (IS_GEN6(dev) || IS_GEN7(dev))
 337                 __raw_i915_write32(dev_priv, GTFIFODBG,
 338                                    __raw_i915_read32(dev_priv, GTFIFODBG));
 339
 340         intel_uncore_forcewake_reset(dev);
 341 }
 342
 343 void intel_uncore_sanitize(struct drm_device *dev)
 344 {
 345         struct drm_i915_private *dev_priv = dev->dev_private;
 346         u32 reg_val;
 347
 348         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 349         intel_disable_gt_powersave(dev);
 350
 351         /* Turn off power gate, require especially for the BIOS less system */
 352         if (IS_VALLEYVIEW(dev)) {
 353
 354                 mutex_lock(&dev_priv->rps.hw_lock);
 355                 reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS);
 356
 357                 if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT))
 358                         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0);
 359
 360                 mutex_unlock(&dev_priv->rps.hw_lock);
 361
 362         }
 363 }
 364
 365 /*
 366  * Generally this is called implicitly by the register read function. However,
 367  * if some sequence requires the GT to not power down then this function should
 368  * be called at the beginning of the sequence followed by a call to
 369  * gen6_gt_force_wake_put() at the end of the sequence.
 370  */
 371 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 372 {
 373         unsigned long irqflags;
 374
 375         if (!dev_priv->uncore.funcs.force_wake_get)
 376                 return;
 377
 378         intel_runtime_pm_get(dev_priv);
 379
 380         /* Redirect to VLV specific routine */
 381         if (IS_VALLEYVIEW(dev_priv->dev))
 382                 return vlv_force_wake_get(dev_priv, fw_engine);
 383
 384         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 385         if (dev_priv->uncore.forcewake_count++ == 0)
 386                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 387         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 388 }
 389
 390 /*
 391  * see gen6_gt_force_wake_get()
 392  */
 393 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 394 {
 395         unsigned long irqflags;
 396
 397         if (!dev_priv->uncore.funcs.force_wake_put)
 398                 return;
 399
 400         /* Redirect to VLV specific routine */
 401         if (IS_VALLEYVIEW(dev_priv->dev))
 402                 return vlv_force_wake_put(dev_priv, fw_engine);
 403
 404
 405         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 406         if (--dev_priv->uncore.forcewake_count == 0) {
 407                 dev_priv->uncore.forcewake_count++;
 408                 mod_delayed_work(dev_priv->wq,
 409                                  &dev_priv->uncore.force_wake_work,
 410                                  1);
 411         }
 412         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 413
 414         intel_runtime_pm_put(dev_priv);
 415 }
 416
 417 /* We give fast paths for the really cool registers */
 418 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
 419          ((reg) < 0x40000 && (reg) != FORCEWAKE)
 420
 421 static void
 422 ilk_dummy_write(struct drm_i915_private *dev_priv)
 423 {
 424         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 425          * the chip from rc6 before touching it for real. MI_MODE is masked,
 426          * hence harmless to write 0 into. */
 427         __raw_i915_write32(dev_priv, MI_MODE, 0);
 428 }
 429
 430 static void
 431 hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg)
 432 {
 433         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 434                 DRM_ERROR("Unknown unclaimed register before writing to %x\n",
 435                           reg);
 436                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 437         }
 438 }
 439
 440 static void
 441 hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg)
 442 {
 443         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 444                 DRM_ERROR("Unclaimed write to %x\n", reg);
 445                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 446         }
 447 }
 448
 449 static void
 450 assert_device_not_suspended(struct drm_i915_private *dev_priv)
 451 {
 452         WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended,
 453              "Device suspended\n");
 454 }
 455
 456 #define REG_READ_HEADER(x) \
 457         unsigned long irqflags; \
 458         u##x val = 0; \
 459         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 460
 461 #define REG_READ_FOOTER \
 462         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 463         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 464         return val
 465
 466 #define __gen4_read(x) \
 467 static u##x \
 468 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 469         REG_READ_HEADER(x); \
 470         val = __raw_i915_read##x(dev_priv, reg); \
 471         REG_READ_FOOTER; \
 472 }
 473
 474 #define __gen5_read(x) \
 475 static u##x \
 476 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 477         REG_READ_HEADER(x); \
 478         ilk_dummy_write(dev_priv); \
 479         val = __raw_i915_read##x(dev_priv, reg); \
 480         REG_READ_FOOTER; \
 481 }
 482
 483 #define __gen6_read(x) \
 484 static u##x \
 485 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 486         REG_READ_HEADER(x); \
 487         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 488                 if (dev_priv->uncore.forcewake_count == 0) \
 489                         dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 490                                                         FORCEWAKE_ALL); \
 491                 val = __raw_i915_read##x(dev_priv, reg); \
 492                 if (dev_priv->uncore.forcewake_count == 0) \
 493                         dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 494                                                         FORCEWAKE_ALL); \
 495         } else { \
 496                 val = __raw_i915_read##x(dev_priv, reg); \
 497         } \
 498         REG_READ_FOOTER; \
 499 }
 500
 501 #define __vlv_read(x) \
 502 static u##x \
 503 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 504         unsigned fwengine = 0; \
 505         unsigned *fwcount; \
 506         REG_READ_HEADER(x); \
 507         if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) {   \
 508                 fwengine = FORCEWAKE_RENDER;            \
 509                 fwcount = &dev_priv->uncore.fw_rendercount;    \
 510         }                                               \
 511         else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) {       \
 512                 fwengine = FORCEWAKE_MEDIA;             \
 513                 fwcount = &dev_priv->uncore.fw_mediacount;     \
 514         }  \
 515         if (fwengine != 0) {            \
 516                 if ((*fwcount)++ == 0) \
 517                         (dev_priv)->uncore.funcs.force_wake_get(dev_priv, \
 518                                                                 fwengine); \
 519                 val = __raw_i915_read##x(dev_priv, reg); \
 520                 if (--(*fwcount) == 0) \
 521                         (dev_priv)->uncore.funcs.force_wake_put(dev_priv, \
 522                                                         fwengine); \
 523         } else { \
 524                 val = __raw_i915_read##x(dev_priv, reg); \
 525         } \
 526         REG_READ_FOOTER; \
 527 }
 528
 529
 530 __vlv_read(8)
 531 __vlv_read(16)
 532 __vlv_read(32)
 533 __vlv_read(64)
 534 __gen6_read(8)
 535 __gen6_read(16)
 536 __gen6_read(32)
 537 __gen6_read(64)
 538 __gen5_read(8)
 539 __gen5_read(16)
 540 __gen5_read(32)
 541 __gen5_read(64)
 542 __gen4_read(8)
 543 __gen4_read(16)
 544 __gen4_read(32)
 545 __gen4_read(64)
 546
 547 #undef __vlv_read
 548 #undef __gen6_read
 549 #undef __gen5_read
 550 #undef __gen4_read
 551 #undef REG_READ_FOOTER
 552 #undef REG_READ_HEADER
 553
 554 #define REG_WRITE_HEADER \
 555         unsigned long irqflags; \
 556         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
 557         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 558
 559 #define REG_WRITE_FOOTER \
 560         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
 561
 562 #define __gen4_write(x) \
 563 static void \
 564 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 565         REG_WRITE_HEADER; \
 566         __raw_i915_write##x(dev_priv, reg, val); \
 567         REG_WRITE_FOOTER; \
 568 }
 569
 570 #define __gen5_write(x) \
 571 static void \
 572 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 573         REG_WRITE_HEADER; \
 574         ilk_dummy_write(dev_priv); \
 575         __raw_i915_write##x(dev_priv, reg, val); \
 576         REG_WRITE_FOOTER; \
 577 }
 578
 579 #define __gen6_write(x) \
 580 static void \
 581 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 582         u32 __fifo_ret = 0; \
 583         REG_WRITE_HEADER; \
 584         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 585                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 586         } \
 587         assert_device_not_suspended(dev_priv); \
 588         __raw_i915_write##x(dev_priv, reg, val); \
 589         if (unlikely(__fifo_ret)) { \
 590                 gen6_gt_check_fifodbg(dev_priv); \
 591         } \
 592         REG_WRITE_FOOTER; \
 593 }
 594
 595 #define __hsw_write(x) \
 596 static void \
 597 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 598         u32 __fifo_ret = 0; \
 599         REG_WRITE_HEADER; \
 600         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 601                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 602         } \
 603         assert_device_not_suspended(dev_priv); \
 604         hsw_unclaimed_reg_clear(dev_priv, reg); \
 605         __raw_i915_write##x(dev_priv, reg, val); \
 606         if (unlikely(__fifo_ret)) { \
 607                 gen6_gt_check_fifodbg(dev_priv); \
 608         } \
 609         hsw_unclaimed_reg_check(dev_priv, reg); \
 610         REG_WRITE_FOOTER; \
 611 }
 612
 613 static const u32 gen8_shadowed_regs[] = {
 614         FORCEWAKE_MT,
 615         GEN6_RPNSWREQ,
 616         GEN6_RC_VIDEO_FREQ,
 617         RING_TAIL(RENDER_RING_BASE),
 618         RING_TAIL(GEN6_BSD_RING_BASE),
 619         RING_TAIL(VEBOX_RING_BASE),
 620         RING_TAIL(BLT_RING_BASE),
 621         /* TODO: Other registers are not yet used */
 622 };
 623
 624 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 625 {
 626         int i;
 627         for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
 628                 if (reg == gen8_shadowed_regs[i])
 629                         return true;
 630
 631         return false;
 632 }
 633
 634 #define __gen8_write(x) \
 635 static void \
 636 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 637         bool __needs_put = reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg); \
 638         REG_WRITE_HEADER; \
 639         if (__needs_put) { \
 640                 dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 641                                                         FORCEWAKE_ALL); \
 642         } \
 643         __raw_i915_write##x(dev_priv, reg, val); \
 644         if (__needs_put) { \
 645                 dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 646                                                         FORCEWAKE_ALL); \
 647         } \
 648         REG_WRITE_FOOTER; \
 649 }
 650
 651 __gen8_write(8)
 652 __gen8_write(16)
 653 __gen8_write(32)
 654 __gen8_write(64)
 655 __hsw_write(8)
 656 __hsw_write(16)
 657 __hsw_write(32)
 658 __hsw_write(64)
 659 __gen6_write(8)
 660 __gen6_write(16)
 661 __gen6_write(32)
 662 __gen6_write(64)
 663 __gen5_write(8)
 664 __gen5_write(16)
 665 __gen5_write(32)
 666 __gen5_write(64)
 667 __gen4_write(8)
 668 __gen4_write(16)
 669 __gen4_write(32)
 670 __gen4_write(64)
 671
 672 #undef __gen8_write
 673 #undef __hsw_write
 674 #undef __gen6_write
 675 #undef __gen5_write
 676 #undef __gen4_write
 677 #undef REG_WRITE_FOOTER
 678 #undef REG_WRITE_HEADER
 679
 680 void intel_uncore_init(struct drm_device *dev)
 681 {
 682         struct drm_i915_private *dev_priv = dev->dev_private;
 683
 684         INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work,
 685                           gen6_force_wake_work);
 686
 687         if (IS_VALLEYVIEW(dev)) {
 688                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
 689                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
 690         } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
 691                 dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get;
 692                 dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put;
 693         } else if (IS_IVYBRIDGE(dev)) {
 694                 u32 ecobus;
 695
 696                 /* IVB configs may use multi-threaded forcewake */
 697
 698                 /* A small trick here - if the bios hasn't configured
 699                  * MT forcewake, and if the device is in RC6, then
 700                  * force_wake_mt_get will not wake the device and the
 701                  * ECOBUS read will return zero. Which will be
 702                  * (correctly) interpreted by the test below as MT
 703                  * forcewake being disabled.
 704                  */
 705                 mutex_lock(&dev->struct_mutex);
 706                 __gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
 707                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
 708                 __gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
 709                 mutex_unlock(&dev->struct_mutex);
 710
 711                 if (ecobus & FORCEWAKE_MT_ENABLE) {
 712                         dev_priv->uncore.funcs.force_wake_get =
 713                                 __gen6_gt_force_wake_mt_get;
 714                         dev_priv->uncore.funcs.force_wake_put =
 715                                 __gen6_gt_force_wake_mt_put;
 716                 } else {
 717                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
 718                         DRM_INFO("when using vblank-synced partial screen updates.\n");
 719                         dev_priv->uncore.funcs.force_wake_get =
 720                                 __gen6_gt_force_wake_get;
 721                         dev_priv->uncore.funcs.force_wake_put =
 722                                 __gen6_gt_force_wake_put;
 723                 }
 724         } else if (IS_GEN6(dev)) {
 725                 dev_priv->uncore.funcs.force_wake_get =
 726                         __gen6_gt_force_wake_get;
 727                 dev_priv->uncore.funcs.force_wake_put =
 728                         __gen6_gt_force_wake_put;
 729         }
 730
 731         switch (INTEL_INFO(dev)->gen) {
 732         default:
 733                 dev_priv->uncore.funcs.mmio_writeb  = gen8_write8;
 734                 dev_priv->uncore.funcs.mmio_writew  = gen8_write16;
 735                 dev_priv->uncore.funcs.mmio_writel  = gen8_write32;
 736                 dev_priv->uncore.funcs.mmio_writeq  = gen8_write64;
 737                 dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 738                 dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 739                 dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 740                 dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 741                 break;
 742         case 7:
 743         case 6:
 744                 if (IS_HASWELL(dev)) {
 745                         dev_priv->uncore.funcs.mmio_writeb  = hsw_write8;
 746                         dev_priv->uncore.funcs.mmio_writew  = hsw_write16;
 747                         dev_priv->uncore.funcs.mmio_writel  = hsw_write32;
 748                         dev_priv->uncore.funcs.mmio_writeq  = hsw_write64;
 749                 } else {
 750                         dev_priv->uncore.funcs.mmio_writeb  = gen6_write8;
 751                         dev_priv->uncore.funcs.mmio_writew  = gen6_write16;
 752                         dev_priv->uncore.funcs.mmio_writel  = gen6_write32;
 753                         dev_priv->uncore.funcs.mmio_writeq  = gen6_write64;
 754                 }
 755
 756                 if (IS_VALLEYVIEW(dev)) {
 757                         dev_priv->uncore.funcs.mmio_readb  = vlv_read8;
 758                         dev_priv->uncore.funcs.mmio_readw  = vlv_read16;
 759                         dev_priv->uncore.funcs.mmio_readl  = vlv_read32;
 760                         dev_priv->uncore.funcs.mmio_readq  = vlv_read64;
 761                 } else {
 762                         dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 763                         dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 764                         dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 765                         dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 766                 }
 767                 break;
 768         case 5:
 769                 dev_priv->uncore.funcs.mmio_writeb  = gen5_write8;
 770                 dev_priv->uncore.funcs.mmio_writew  = gen5_write16;
 771                 dev_priv->uncore.funcs.mmio_writel  = gen5_write32;
 772                 dev_priv->uncore.funcs.mmio_writeq  = gen5_write64;
 773                 dev_priv->uncore.funcs.mmio_readb  = gen5_read8;
 774                 dev_priv->uncore.funcs.mmio_readw  = gen5_read16;
 775                 dev_priv->uncore.funcs.mmio_readl  = gen5_read32;
 776                 dev_priv->uncore.funcs.mmio_readq  = gen5_read64;
 777                 break;
 778         case 4:
 779         case 3:
 780         case 2:
 781                 dev_priv->uncore.funcs.mmio_writeb  = gen4_write8;
 782                 dev_priv->uncore.funcs.mmio_writew  = gen4_write16;
 783                 dev_priv->uncore.funcs.mmio_writel  = gen4_write32;
 784                 dev_priv->uncore.funcs.mmio_writeq  = gen4_write64;
 785                 dev_priv->uncore.funcs.mmio_readb  = gen4_read8;
 786                 dev_priv->uncore.funcs.mmio_readw  = gen4_read16;
 787                 dev_priv->uncore.funcs.mmio_readl  = gen4_read32;
 788                 dev_priv->uncore.funcs.mmio_readq  = gen4_read64;
 789                 break;
 790         }
 791 }
 792
 793 void intel_uncore_fini(struct drm_device *dev)
 794 {
 795         struct drm_i915_private *dev_priv = dev->dev_private;
 796
 797         flush_delayed_work(&dev_priv->uncore.force_wake_work);
 798
 799         /* Paranoia: make sure we have disabled everything before we exit. */
 800         intel_uncore_sanitize(dev);
 801 }
 802
 803 static const struct register_whitelist {
 804         uint64_t offset;
 805         uint32_t size;
 806         uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
 807 } whitelist[] = {
 808         { RING_TIMESTAMP(RENDER_RING_BASE), 8, 0xF0 },
 809 };
 810
 811 int i915_reg_read_ioctl(struct drm_device *dev,
 812                         void *data, struct drm_file *file)
 813 {
 814         struct drm_i915_private *dev_priv = dev->dev_private;
 815         struct drm_i915_reg_read *reg = data;
 816         struct register_whitelist const *entry = whitelist;
 817         int i;
 818
 819         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
 820                 if (entry->offset == reg->offset &&
 821                     (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
 822                         break;
 823         }
 824
 825         if (i == ARRAY_SIZE(whitelist))
 826                 return -EINVAL;
 827
 828         switch (entry->size) {
 829         case 8:
 830                 reg->val = I915_READ64(reg->offset);
 831                 break;
 832         case 4:
 833                 reg->val = I915_READ(reg->offset);
 834                 break;
 835         case 2:
 836                 reg->val = I915_READ16(reg->offset);
 837                 break;
 838         case 1:
 839                 reg->val = I915_READ8(reg->offset);
 840                 break;
 841         default:
 842                 WARN_ON(1);
 843                 return -EINVAL;
 844         }
 845
 846         return 0;
 847 }
 848
 849 int i915_get_reset_stats_ioctl(struct drm_device *dev,
 850                                void *data, struct drm_file *file)
 851 {
 852         struct drm_i915_private *dev_priv = dev->dev_private;
 853         struct drm_i915_reset_stats *args = data;
 854         struct i915_ctx_hang_stats *hs;
 855         int ret;
 856
 857         if (args->flags || args->pad)
 858                 return -EINVAL;
 859
 860         if (args->ctx_id == DEFAULT_CONTEXT_ID && !capable(CAP_SYS_ADMIN))
 861                 return -EPERM;
 862
 863         ret = mutex_lock_interruptible(&dev->struct_mutex);
 864         if (ret)
 865                 return ret;
 866
 867         hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id);
 868         if (IS_ERR(hs)) {
 869                 mutex_unlock(&dev->struct_mutex);
 870                 return PTR_ERR(hs);
 871         }
 872
 873         if (capable(CAP_SYS_ADMIN))
 874                 args->reset_count = i915_reset_count(&dev_priv->gpu_error);
 875         else
 876                 args->reset_count = 0;
 877
 878         args->batch_active = hs->batch_active;
 879         args->batch_pending = hs->batch_pending;
 880
 881         mutex_unlock(&dev->struct_mutex);
 882
 883         return 0;
 884 }
 885
 886 static int i965_reset_complete(struct drm_device *dev)
 887 {
 888         u8 gdrst;
 889         pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
 890         return (gdrst & GRDOM_RESET_ENABLE) == 0;
 891 }
 892
 893 static int i965_do_reset(struct drm_device *dev)
 894 {
 895         int ret;
 896
 897         /*
 898          * Set the domains we want to reset (GRDOM/bits 2 and 3) as
 899          * well as the reset bit (GR/bit 0).  Setting the GR bit
 900          * triggers the reset; when done, the hardware will clear it.
 901          */
 902         pci_write_config_byte(dev->pdev, I965_GDRST,
 903                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
 904         ret =  wait_for(i965_reset_complete(dev), 500);
 905         if (ret)
 906                 return ret;
 907
 908         /* We can't reset render&media without also resetting display ... */
 909         pci_write_config_byte(dev->pdev, I965_GDRST,
 910                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 911
 912         ret =  wait_for(i965_reset_complete(dev), 500);
 913         if (ret)
 914                 return ret;
 915
 916         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
 917
 918         return 0;
 919 }
 920
 921 static int ironlake_do_reset(struct drm_device *dev)
 922 {
 923         struct drm_i915_private *dev_priv = dev->dev_private;
 924         u32 gdrst;
 925         int ret;
 926
 927         gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 928         gdrst &= ~GRDOM_MASK;
 929         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
 930                    gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE);
 931         ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 932         if (ret)
 933                 return ret;
 934
 935         /* We can't reset render&media without also resetting display ... */
 936         gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 937         gdrst &= ~GRDOM_MASK;
 938         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
 939                    gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 940         return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 941 }
 942
 943 static int gen6_do_reset(struct drm_device *dev)
 944 {
 945         struct drm_i915_private *dev_priv = dev->dev_private;
 946         int     ret;
 947         unsigned long irqflags;
 948
 949         /* Hold uncore.lock across reset to prevent any register access
 950          * with forcewake not set correctly
 951          */
 952         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 953
 954         /* Reset the chip */
 955
 956         /* GEN6_GDRST is not in the gt power well, no need to check
 957          * for fifo space for the write or forcewake the chip for
 958          * the read
 959          */
 960         __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL);
 961
 962         /* Spin waiting for the device to ack the reset request */
 963         ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
 964
 965         intel_uncore_forcewake_reset(dev);
 966
 967         /* If reset with a user forcewake, try to restore, otherwise turn it off */
 968         if (dev_priv->uncore.forcewake_count)
 969                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 970         else
 971                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 972
 973         /* Restore fifo count */
 974         dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 975
 976         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 977         return ret;
 978 }
 979
 980 int intel_gpu_reset(struct drm_device *dev)
 981 {
 982         switch (INTEL_INFO(dev)->gen) {
 983         case 8:
 984         case 7:
 985         case 6: return gen6_do_reset(dev);
 986         case 5: return ironlake_do_reset(dev);
 987         case 4: return i965_do_reset(dev);
 988         default: return -ENODEV;
 989         }
 990 }
 991
 992 void intel_uncore_check_errors(struct drm_device *dev)
 993 {
 994         struct drm_i915_private *dev_priv = dev->dev_private;
 995
 996         if (HAS_FPGA_DBG_UNCLAIMED(dev) &&
 997             (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
 998                 DRM_ERROR("Unclaimed register before interrupt\n");
 999                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
1000         }
1001 }