Merge tag 'v4.4-rc2' into drm-intel-next-queued
[platform/kernel/linux-rpi.git] / drivers / gpu / drm / i915 / intel_runtime_pm.c
index d89c1d0..a1dc815 100644 (file)
@@ -49,9 +49,6 @@
  * present for a given platform.
  */
 
-#define GEN9_ENABLE_DC5(dev) 0
-#define SKL_ENABLE_DC6(dev) IS_SKYLAKE(dev)
-
 #define for_each_power_well(i, power_well, domain_mask, power_domains) \
        for (i = 0;                                                     \
             i < (power_domains)->power_well_count &&                   \
@@ -244,12 +241,6 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
                gen8_irq_power_well_post_enable(dev_priv,
                                                1 << PIPE_C | 1 << PIPE_B);
        }
-
-       if (power_well->data == SKL_DISP_PW_1) {
-               if (!dev_priv->power_domains.initializing)
-                       intel_prepare_ddi(dev);
-               gen8_irq_power_well_post_enable(dev_priv, 1 << PIPE_A);
-       }
 }
 
 static void hsw_set_power_well(struct drm_i915_private *dev_priv,
@@ -292,58 +283,38 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
        BIT(POWER_DOMAIN_TRANSCODER_C) |                \
        BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |         \
        BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |         \
-       BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_E_2_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_E_LANES) |            \
        BIT(POWER_DOMAIN_AUX_B) |                       \
        BIT(POWER_DOMAIN_AUX_C) |                       \
        BIT(POWER_DOMAIN_AUX_D) |                       \
        BIT(POWER_DOMAIN_AUDIO) |                       \
        BIT(POWER_DOMAIN_VGA) |                         \
        BIT(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS (                \
-       SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
-       BIT(POWER_DOMAIN_PLLS) |                        \
-       BIT(POWER_DOMAIN_PIPE_A) |                      \
-       BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
-       BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |         \
-       BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
-       BIT(POWER_DOMAIN_AUX_A) |                       \
-       BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_A_E_POWER_DOMAINS (            \
-       BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_E_2_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_E_LANES) |            \
        BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_B_POWER_DOMAINS (              \
-       BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |            \
        BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_C_POWER_DOMAINS (              \
-       BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |            \
        BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DDI_D_POWER_DOMAINS (              \
-       BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |            \
        BIT(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_MISC_IO_POWER_DOMAINS (            \
-       SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS |         \
-       BIT(POWER_DOMAIN_PLLS) |                        \
+#define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (             \
+       SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
+       BIT(POWER_DOMAIN_MODESET) |                     \
+       BIT(POWER_DOMAIN_AUX_A) |                       \
        BIT(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS (          \
-       (POWER_DOMAIN_MASK & ~(SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS |  \
+       (POWER_DOMAIN_MASK & ~(                         \
        SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
-       SKL_DISPLAY_DDI_A_E_POWER_DOMAINS |             \
-       SKL_DISPLAY_DDI_B_POWER_DOMAINS |               \
-       SKL_DISPLAY_DDI_C_POWER_DOMAINS |               \
-       SKL_DISPLAY_DDI_D_POWER_DOMAINS |               \
-       SKL_DISPLAY_MISC_IO_POWER_DOMAINS)) |           \
+       SKL_DISPLAY_DC_OFF_POWER_DOMAINS)) |            \
        BIT(POWER_DOMAIN_INIT))
 
 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (                \
@@ -354,25 +325,28 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
        BIT(POWER_DOMAIN_TRANSCODER_C) |                \
        BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |         \
        BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |         \
-       BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |            \
        BIT(POWER_DOMAIN_AUX_B) |                       \
        BIT(POWER_DOMAIN_AUX_C) |                       \
        BIT(POWER_DOMAIN_AUDIO) |                       \
        BIT(POWER_DOMAIN_VGA) |                         \
+       BIT(POWER_DOMAIN_GMBUS) |                       \
        BIT(POWER_DOMAIN_INIT))
 #define BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS (                \
        BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
        BIT(POWER_DOMAIN_PIPE_A) |                      \
        BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
        BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |         \
-       BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |            \
        BIT(POWER_DOMAIN_AUX_A) |                       \
        BIT(POWER_DOMAIN_PLLS) |                        \
        BIT(POWER_DOMAIN_INIT))
+#define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (             \
+       BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
+       BIT(POWER_DOMAIN_MODESET) |                     \
+       BIT(POWER_DOMAIN_AUX_A) |                       \
+       BIT(POWER_DOMAIN_INIT))
 #define BXT_DISPLAY_ALWAYS_ON_POWER_DOMAINS (          \
        (POWER_DOMAIN_MASK & ~(BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS |  \
        BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS)) |       \
@@ -416,46 +390,74 @@ static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
          */
 }
 
-void bxt_enable_dc9(struct drm_i915_private *dev_priv)
+static void gen9_set_dc_state_debugmask_memory_up(
+                       struct drm_i915_private *dev_priv)
 {
        uint32_t val;
 
-       assert_can_enable_dc9(dev_priv);
+       /* The below bit doesn't need to be cleared ever afterwards */
+       val = I915_READ(DC_STATE_DEBUG);
+       if (!(val & DC_STATE_DEBUG_MASK_MEMORY_UP)) {
+               val |= DC_STATE_DEBUG_MASK_MEMORY_UP;
+               I915_WRITE(DC_STATE_DEBUG, val);
+               POSTING_READ(DC_STATE_DEBUG);
+       }
+}
 
-       DRM_DEBUG_KMS("Enabling DC9\n");
+static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
+{
+       uint32_t val;
+       uint32_t mask;
+
+       mask = DC_STATE_EN_UPTO_DC5;
+       if (IS_BROXTON(dev_priv))
+               mask |= DC_STATE_EN_DC9;
+       else
+               mask |= DC_STATE_EN_UPTO_DC6;
+
+       WARN_ON_ONCE(state & ~mask);
+
+       if (i915.enable_dc == 0)
+               state = DC_STATE_DISABLE;
+       else if (i915.enable_dc == 1 && state > DC_STATE_EN_UPTO_DC5)
+               state = DC_STATE_EN_UPTO_DC5;
+
+       if (state & DC_STATE_EN_UPTO_DC5_DC6_MASK)
+               gen9_set_dc_state_debugmask_memory_up(dev_priv);
 
        val = I915_READ(DC_STATE_EN);
-       val |= DC_STATE_EN_DC9;
+       DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
+                     val & mask, state);
+       val &= ~mask;
+       val |= state;
        I915_WRITE(DC_STATE_EN, val);
        POSTING_READ(DC_STATE_EN);
 }
 
-void bxt_disable_dc9(struct drm_i915_private *dev_priv)
+void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 {
-       uint32_t val;
+       assert_can_enable_dc9(dev_priv);
+
+       DRM_DEBUG_KMS("Enabling DC9\n");
+
+       gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
+}
 
+void bxt_disable_dc9(struct drm_i915_private *dev_priv)
+{
        assert_can_disable_dc9(dev_priv);
 
        DRM_DEBUG_KMS("Disabling DC9\n");
 
-       val = I915_READ(DC_STATE_EN);
-       val &= ~DC_STATE_EN_DC9;
-       I915_WRITE(DC_STATE_EN, val);
-       POSTING_READ(DC_STATE_EN);
+       gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 }
 
-static void gen9_set_dc_state_debugmask_memory_up(
-                       struct drm_i915_private *dev_priv)
+static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 {
-       uint32_t val;
-
-       /* The below bit doesn't need to be cleared ever afterwards */
-       val = I915_READ(DC_STATE_DEBUG);
-       if (!(val & DC_STATE_DEBUG_MASK_MEMORY_UP)) {
-               val |= DC_STATE_DEBUG_MASK_MEMORY_UP;
-               I915_WRITE(DC_STATE_DEBUG, val);
-               POSTING_READ(DC_STATE_DEBUG);
-       }
+       WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
+                 "CSR program storage start is NULL\n");
+       WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
+       WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 }
 
 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
@@ -478,8 +480,6 @@ static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 
 static void assert_can_disable_dc5(struct drm_i915_private *dev_priv)
 {
-       bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
-                                       SKL_DISP_PW_2);
        /*
         * During initialization, the firmware may not be loaded yet.
         * We still want to make sure that the DC enabling flag is cleared.
@@ -487,40 +487,17 @@ static void assert_can_disable_dc5(struct drm_i915_private *dev_priv)
        if (dev_priv->power_domains.initializing)
                return;
 
-       WARN_ONCE(!pg2_enabled, "PG2 not enabled to disable DC5.\n");
        WARN_ONCE(dev_priv->pm.suspended,
                "Disabling of DC5 while platform is runtime-suspended should never happen.\n");
 }
 
 static void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 {
-       uint32_t val;
-
        assert_can_enable_dc5(dev_priv);
 
        DRM_DEBUG_KMS("Enabling DC5\n");
 
-       gen9_set_dc_state_debugmask_memory_up(dev_priv);
-
-       val = I915_READ(DC_STATE_EN);
-       val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK;
-       val |= DC_STATE_EN_UPTO_DC5;
-       I915_WRITE(DC_STATE_EN, val);
-       POSTING_READ(DC_STATE_EN);
-}
-
-static void gen9_disable_dc5(struct drm_i915_private *dev_priv)
-{
-       uint32_t val;
-
-       assert_can_disable_dc5(dev_priv);
-
-       DRM_DEBUG_KMS("Disabling DC5\n");
-
-       val = I915_READ(DC_STATE_EN);
-       val &= ~DC_STATE_EN_UPTO_DC5;
-       I915_WRITE(DC_STATE_EN, val);
-       POSTING_READ(DC_STATE_EN);
+       gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 }
 
 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
@@ -546,40 +523,37 @@ static void assert_can_disable_dc6(struct drm_i915_private *dev_priv)
        if (dev_priv->power_domains.initializing)
                return;
 
-       assert_csr_loaded(dev_priv);
        WARN_ONCE(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
                  "DC6 already programmed to be disabled.\n");
 }
 
-static void skl_enable_dc6(struct drm_i915_private *dev_priv)
+static void gen9_disable_dc5_dc6(struct drm_i915_private *dev_priv)
 {
-       uint32_t val;
+       assert_can_disable_dc5(dev_priv);
 
+       if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 && i915.enable_dc != 1)
+               assert_can_disable_dc6(dev_priv);
+
+       gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+}
+
+void skl_enable_dc6(struct drm_i915_private *dev_priv)
+{
        assert_can_enable_dc6(dev_priv);
 
        DRM_DEBUG_KMS("Enabling DC6\n");
 
-       gen9_set_dc_state_debugmask_memory_up(dev_priv);
+       gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 
-       val = I915_READ(DC_STATE_EN);
-       val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK;
-       val |= DC_STATE_EN_UPTO_DC6;
-       I915_WRITE(DC_STATE_EN, val);
-       POSTING_READ(DC_STATE_EN);
 }
 
-static void skl_disable_dc6(struct drm_i915_private *dev_priv)
+void skl_disable_dc6(struct drm_i915_private *dev_priv)
 {
-       uint32_t val;
-
        assert_can_disable_dc6(dev_priv);
 
        DRM_DEBUG_KMS("Disabling DC6\n");
 
-       val = I915_READ(DC_STATE_EN);
-       val &= ~DC_STATE_EN_UPTO_DC6;
-       I915_WRITE(DC_STATE_EN, val);
-       POSTING_READ(DC_STATE_EN);
+       gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 }
 
 static void skl_set_power_well(struct drm_i915_private *dev_priv,
@@ -629,20 +603,16 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
                                !I915_READ(HSW_PWR_WELL_BIOS),
                                "Invalid for power well status to be enabled, unless done by the BIOS, \
                                when request is to disable!\n");
-                       if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) &&
-                               power_well->data == SKL_DISP_PW_2) {
-                               if (SKL_ENABLE_DC6(dev)) {
-                                       skl_disable_dc6(dev_priv);
-                                       /*
-                                        * DDI buffer programming unnecessary during driver-load/resume
-                                        * as it's already done during modeset initialization then.
-                                        * It's also invalid here as encoder list is still uninitialized.
-                                        */
-                                       if (!dev_priv->power_domains.initializing)
-                                               intel_prepare_ddi(dev);
-                               } else {
-                                       gen9_disable_dc5(dev_priv);
-                               }
+                       if (power_well->data == SKL_DISP_PW_2) {
+                               /*
+                                * DDI buffer programming unnecessary during
+                                * driver-load/resume as it's already done
+                                * during modeset initialization then. It's
+                                * also invalid here as encoder list is still
+                                * uninitialized.
+                                */
+                               if (!dev_priv->power_domains.initializing)
+                                       intel_prepare_ddi(dev);
                        }
                        I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask);
                }
@@ -657,34 +627,9 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
                }
        } else {
                if (enable_requested) {
-                       if (IS_SKYLAKE(dev) &&
-                               (power_well->data == SKL_DISP_PW_1) &&
-                               (intel_csr_load_status_get(dev_priv) == FW_LOADED))
-                               DRM_DEBUG_KMS("Not Disabling PW1, dmc will handle\n");
-                       else {
-                               I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask);
-                               POSTING_READ(HSW_PWR_WELL_DRIVER);
-                               DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
-                       }
-
-                       if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) &&
-                               power_well->data == SKL_DISP_PW_2) {
-                               enum csr_state state;
-                               /* TODO: wait for a completion event or
-                                * similar here instead of busy
-                                * waiting using wait_for function.
-                                */
-                               wait_for((state = intel_csr_load_status_get(dev_priv)) !=
-                                               FW_UNINITIALIZED, 1000);
-                               if (state != FW_LOADED)
-                                       DRM_DEBUG("CSR firmware not ready (%d)\n",
-                                                       state);
-                               else
-                                       if (SKL_ENABLE_DC6(dev))
-                                               skl_enable_dc6(dev_priv);
-                                       else
-                                               gen9_enable_dc5(dev_priv);
-                       }
+                       I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask);
+                       POSTING_READ(HSW_PWR_WELL_DRIVER);
+                       DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
                }
        }
 
@@ -759,6 +704,41 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv,
        skl_set_power_well(dev_priv, power_well, false);
 }
 
+static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
+                                          struct i915_power_well *power_well)
+{
+       return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
+}
+
+static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
+                                         struct i915_power_well *power_well)
+{
+       gen9_disable_dc5_dc6(dev_priv);
+}
+
+static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
+                                          struct i915_power_well *power_well)
+{
+       if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 && i915.enable_dc != 1)
+               skl_enable_dc6(dev_priv);
+       else
+               gen9_enable_dc5(dev_priv);
+}
+
+static void gen9_dc_off_power_well_sync_hw(struct drm_i915_private *dev_priv,
+                                          struct i915_power_well *power_well)
+{
+       if (power_well->count > 0) {
+               gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+       } else {
+               if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 &&
+                   i915.enable_dc != 1)
+                       gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
+               else
+                       gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
+       }
+}
+
 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
                                           struct i915_power_well *power_well)
 {
@@ -973,10 +953,12 @@ static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_pr
                                                 int power_well_id)
 {
        struct i915_power_domains *power_domains = &dev_priv->power_domains;
-       struct i915_power_well *power_well;
        int i;
 
-       for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
+       for (i = 0; i < power_domains->power_well_count; i++) {
+               struct i915_power_well *power_well;
+
+               power_well = &power_domains->power_wells[i];
                if (power_well->data == power_well_id)
                        return power_well;
        }
@@ -1457,7 +1439,7 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
        for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
                WARN_ON(!power_well->count);
 
-               if (!--power_well->count && i915.disable_power_well)
+               if (!--power_well->count)
                        intel_power_well_disable(dev_priv, power_well);
        }
 
@@ -1469,20 +1451,17 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 #define HSW_ALWAYS_ON_POWER_DOMAINS (                  \
        BIT(POWER_DOMAIN_PIPE_A) |                      \
        BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
-       BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
-       BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
+       BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |            \
+       BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |            \
        BIT(POWER_DOMAIN_PORT_CRT) |                    \
        BIT(POWER_DOMAIN_PLLS) |                        \
        BIT(POWER_DOMAIN_AUX_A) |                       \
        BIT(POWER_DOMAIN_AUX_B) |                       \
        BIT(POWER_DOMAIN_AUX_C) |                       \
        BIT(POWER_DOMAIN_AUX_D) |                       \
+       BIT(POWER_DOMAIN_GMBUS) |                       \
        BIT(POWER_DOMAIN_INIT))
 #define HSW_DISPLAY_POWER_DOMAINS (                            \
        (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) |    \
@@ -1499,49 +1478,42 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 #define VLV_DISPLAY_POWER_DOMAINS      POWER_DOMAIN_MASK
 
 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (                \
-       BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |    \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |    \
        BIT(POWER_DOMAIN_PORT_CRT) |            \
        BIT(POWER_DOMAIN_AUX_B) |               \
        BIT(POWER_DOMAIN_AUX_C) |               \
        BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \
-       BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |    \
        BIT(POWER_DOMAIN_AUX_B) |               \
        BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |    \
        BIT(POWER_DOMAIN_AUX_B) |               \
        BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \
-       BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |    \
        BIT(POWER_DOMAIN_AUX_C) |               \
        BIT(POWER_DOMAIN_INIT))
 
 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |    \
        BIT(POWER_DOMAIN_AUX_C) |               \
        BIT(POWER_DOMAIN_INIT))
 
 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (                \
-       BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+       BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |    \
+       BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |    \
        BIT(POWER_DOMAIN_AUX_B) |               \
        BIT(POWER_DOMAIN_AUX_C) |               \
        BIT(POWER_DOMAIN_INIT))
 
 #define CHV_DPIO_CMN_D_POWER_DOMAINS (         \
-       BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |  \
-       BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
+       BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |    \
        BIT(POWER_DOMAIN_AUX_D) |               \
        BIT(POWER_DOMAIN_INIT))
 
@@ -1589,6 +1561,13 @@ static const struct i915_power_well_ops skl_power_well_ops = {
        .is_enabled = skl_power_well_enabled,
 };
 
+static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
+       .sync_hw = gen9_dc_off_power_well_sync_hw,
+       .enable = gen9_dc_off_power_well_enable,
+       .disable = gen9_dc_off_power_well_disable,
+       .is_enabled = gen9_dc_off_power_well_enabled,
+};
+
 static struct i915_power_well hsw_power_wells[] = {
        {
                .name = "always-on",
@@ -1644,6 +1623,7 @@ static struct i915_power_well vlv_power_wells[] = {
                .always_on = 1,
                .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
                .ops = &i9xx_always_on_power_well_ops,
+               .data = PUNIT_POWER_WELL_ALWAYS_ON,
        },
        {
                .name = "display",
@@ -1745,20 +1725,29 @@ static struct i915_power_well skl_power_wells[] = {
                .always_on = 1,
                .domains = SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS,
                .ops = &i9xx_always_on_power_well_ops,
+               .data = SKL_DISP_PW_ALWAYS_ON,
        },
        {
                .name = "power well 1",
-               .domains = SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS,
+               /* Handled by the DMC firmware */
+               .domains = 0,
                .ops = &skl_power_well_ops,
                .data = SKL_DISP_PW_1,
        },
        {
                .name = "MISC IO power well",
-               .domains = SKL_DISPLAY_MISC_IO_POWER_DOMAINS,
+               /* Handled by the DMC firmware */
+               .domains = 0,
                .ops = &skl_power_well_ops,
                .data = SKL_DISP_PW_MISC_IO,
        },
        {
+               .name = "DC off",
+               .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
+               .ops = &gen9_dc_off_power_well_ops,
+               .data = SKL_DISP_PW_DC_OFF,
+       },
+       {
                .name = "power well 2",
                .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
                .ops = &skl_power_well_ops,
@@ -1790,6 +1779,34 @@ static struct i915_power_well skl_power_wells[] = {
        },
 };
 
+void skl_pw1_misc_io_init(struct drm_i915_private *dev_priv)
+{
+       struct i915_power_well *well;
+
+       if (!IS_SKYLAKE(dev_priv))
+               return;
+
+       well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
+       intel_power_well_enable(dev_priv, well);
+
+       well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
+       intel_power_well_enable(dev_priv, well);
+}
+
+void skl_pw1_misc_io_fini(struct drm_i915_private *dev_priv)
+{
+       struct i915_power_well *well;
+
+       if (!IS_SKYLAKE(dev_priv))
+               return;
+
+       well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
+       intel_power_well_disable(dev_priv, well);
+
+       well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
+       intel_power_well_disable(dev_priv, well);
+}
+
 static struct i915_power_well bxt_power_wells[] = {
        {
                .name = "always-on",
@@ -1804,11 +1821,17 @@ static struct i915_power_well bxt_power_wells[] = {
                .data = SKL_DISP_PW_1,
        },
        {
+               .name = "DC off",
+               .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
+               .ops = &gen9_dc_off_power_well_ops,
+               .data = SKL_DISP_PW_DC_OFF,
+       },
+       {
                .name = "power well 2",
                .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
                .ops = &skl_power_well_ops,
                .data = SKL_DISP_PW_2,
-       }
+       },
 };
 
 static int
@@ -1845,6 +1868,8 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
        i915.disable_power_well = sanitize_disable_power_well_option(dev_priv,
                                                     i915.disable_power_well);
 
+       BUILD_BUG_ON(POWER_DOMAIN_NUM > 31);
+
        mutex_init(&power_domains->lock);
 
        /*
@@ -1855,7 +1880,7 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
                set_power_wells(power_domains, hsw_power_wells);
        } else if (IS_BROADWELL(dev_priv->dev)) {
                set_power_wells(power_domains, bdw_power_wells);
-       } else if (IS_SKYLAKE(dev_priv->dev)) {
+       } else if (IS_SKYLAKE(dev_priv->dev) || IS_KABYLAKE(dev_priv->dev)) {
                set_power_wells(power_domains, skl_power_wells);
        } else if (IS_BROXTON(dev_priv->dev)) {
                set_power_wells(power_domains, bxt_power_wells);
@@ -1870,21 +1895,6 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
        return 0;
 }
 
-static void intel_runtime_pm_disable(struct drm_i915_private *dev_priv)
-{
-       struct drm_device *dev = dev_priv->dev;
-       struct device *device = &dev->pdev->dev;
-
-       if (!HAS_RUNTIME_PM(dev))
-               return;
-
-       if (!intel_enable_rc6(dev))
-               return;
-
-       /* Make sure we're not suspended first. */
-       pm_runtime_get_sync(device);
-}
-
 /**
  * intel_power_domains_fini - finalizes the power domain structures
  * @dev_priv: i915 device instance
@@ -1895,15 +1905,17 @@ static void intel_runtime_pm_disable(struct drm_i915_private *dev_priv)
  */
 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
 {
-       intel_runtime_pm_disable(dev_priv);
-
        /* The i915.ko module is still not prepared to be loaded when
         * the power well is not enabled, so just enable it in case
         * we're going to unload/reload. */
        intel_display_set_init_power(dev_priv, true);
+
+       /* Remove the refcount we took to keep power well support disabled. */
+       if (!i915.disable_power_well)
+               intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 }
 
-static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
+static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
 {
        struct i915_power_domains *power_domains = &dev_priv->power_domains;
        struct i915_power_well *power_well;
@@ -1918,6 +1930,47 @@ static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
        mutex_unlock(&power_domains->lock);
 }
 
+static void skl_display_core_init(struct drm_i915_private *dev_priv,
+                                 bool resume)
+{
+       struct i915_power_domains *power_domains = &dev_priv->power_domains;
+       uint32_t val;
+
+       gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+
+       /* enable PCH reset handshake */
+       val = I915_READ(HSW_NDE_RSTWRN_OPT);
+       I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
+
+       /* enable PG1 and Misc I/O */
+       mutex_lock(&power_domains->lock);
+       skl_pw1_misc_io_init(dev_priv);
+       mutex_unlock(&power_domains->lock);
+
+       if (!resume)
+               return;
+
+       skl_init_cdclk(dev_priv);
+
+       if (dev_priv->csr.dmc_payload)
+               intel_csr_load_program(dev_priv);
+}
+
+static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
+{
+       struct i915_power_domains *power_domains = &dev_priv->power_domains;
+
+       gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+
+       skl_uninit_cdclk(dev_priv);
+
+       /* The spec doesn't call for removing the reset handshake flag */
+       /* disable PG1 and Misc I/O */
+       mutex_lock(&power_domains->lock);
+       skl_pw1_misc_io_fini(dev_priv);
+       mutex_unlock(&power_domains->lock);
+}
+
 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
 {
        struct i915_power_well *cmn_bc =
@@ -2040,14 +2093,16 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
  * This function initializes the hardware power domain state and enables all
  * power domains using intel_display_set_init_power().
  */
-void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
+void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
 {
        struct drm_device *dev = dev_priv->dev;
        struct i915_power_domains *power_domains = &dev_priv->power_domains;
 
        power_domains->initializing = true;
 
-       if (IS_CHERRYVIEW(dev)) {
+       if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+               skl_display_core_init(dev_priv, resume);
+       } else if (IS_CHERRYVIEW(dev)) {
                mutex_lock(&power_domains->lock);
                chv_phy_control_init(dev_priv);
                mutex_unlock(&power_domains->lock);
@@ -2059,38 +2114,31 @@ void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
 
        /* For now, we need the power well to be always enabled. */
        intel_display_set_init_power(dev_priv, true);
-       intel_power_domains_resume(dev_priv);
+       /* Disable power support if the user asked so. */
+       if (!i915.disable_power_well)
+               intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
+       intel_power_domains_sync_hw(dev_priv);
        power_domains->initializing = false;
 }
 
 /**
- * intel_aux_display_runtime_get - grab an auxiliary power domain reference
+ * intel_power_domains_suspend - suspend power domain state
  * @dev_priv: i915 device instance
  *
- * This function grabs a power domain reference for the auxiliary power domain
- * (for access to the GMBUS and DP AUX blocks) and ensures that it and all its
- * parents are powered up. Therefore users should only grab a reference to the
- * innermost power domain they need.
- *
- * Any power domain reference obtained by this function must have a symmetric
- * call to intel_aux_display_runtime_put() to release the reference again.
+ * This function prepares the hardware power domain state before entering
+ * system suspend. It must be paired with intel_power_domains_init_hw().
  */
-void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
+void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
 {
-       intel_runtime_pm_get(dev_priv);
-}
+       if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+               skl_display_core_uninit(dev_priv);
 
-/**
- * intel_aux_display_runtime_put - release an auxiliary power domain reference
- * @dev_priv: i915 device instance
- *
- * This function drops the auxiliary power domain reference obtained by
- * intel_aux_display_runtime_get() and might power down the corresponding
- * hardware block right away if this is the last reference.
- */
-void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
-{
-       intel_runtime_pm_put(dev_priv);
+       /*
+        * Even if power well support was disabled we still want to disable
+        * power wells while we are system suspended.
+        */
+       if (!i915.disable_power_well)
+               intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 }
 
 /**