1 /* SPDX-License-Identifier: MIT */
3 * Copyright © 2019 Intel Corporation
6 #include <linux/string_helpers.h>
11 #include "intel_backlight_regs.h"
12 #include "intel_cdclk.h"
13 #include "intel_clock_gating.h"
14 #include "intel_combo_phy.h"
16 #include "intel_display_power.h"
17 #include "intel_display_power_map.h"
18 #include "intel_display_power_well.h"
19 #include "intel_display_types.h"
20 #include "intel_dmc.h"
21 #include "intel_mchbar_regs.h"
22 #include "intel_pch_refclk.h"
23 #include "intel_pcode.h"
24 #include "intel_pmdemand.h"
25 #include "intel_pps_regs.h"
26 #include "intel_snps_phy.h"
27 #include "skl_watermark.h"
28 #include "skl_watermark_regs.h"
29 #include "vlv_sideband.h"
31 #define for_each_power_domain_well(__dev_priv, __power_well, __domain) \
32 for_each_power_well(__dev_priv, __power_well) \
33 for_each_if(test_bit((__domain), (__power_well)->domains.bits))
35 #define for_each_power_domain_well_reverse(__dev_priv, __power_well, __domain) \
36 for_each_power_well_reverse(__dev_priv, __power_well) \
37 for_each_if(test_bit((__domain), (__power_well)->domains.bits))
40 intel_display_power_domain_str(enum intel_display_power_domain domain)
43 case POWER_DOMAIN_DISPLAY_CORE:
44 return "DISPLAY_CORE";
45 case POWER_DOMAIN_PIPE_A:
47 case POWER_DOMAIN_PIPE_B:
49 case POWER_DOMAIN_PIPE_C:
51 case POWER_DOMAIN_PIPE_D:
53 case POWER_DOMAIN_PIPE_PANEL_FITTER_A:
54 return "PIPE_PANEL_FITTER_A";
55 case POWER_DOMAIN_PIPE_PANEL_FITTER_B:
56 return "PIPE_PANEL_FITTER_B";
57 case POWER_DOMAIN_PIPE_PANEL_FITTER_C:
58 return "PIPE_PANEL_FITTER_C";
59 case POWER_DOMAIN_PIPE_PANEL_FITTER_D:
60 return "PIPE_PANEL_FITTER_D";
61 case POWER_DOMAIN_TRANSCODER_A:
62 return "TRANSCODER_A";
63 case POWER_DOMAIN_TRANSCODER_B:
64 return "TRANSCODER_B";
65 case POWER_DOMAIN_TRANSCODER_C:
66 return "TRANSCODER_C";
67 case POWER_DOMAIN_TRANSCODER_D:
68 return "TRANSCODER_D";
69 case POWER_DOMAIN_TRANSCODER_EDP:
70 return "TRANSCODER_EDP";
71 case POWER_DOMAIN_TRANSCODER_DSI_A:
72 return "TRANSCODER_DSI_A";
73 case POWER_DOMAIN_TRANSCODER_DSI_C:
74 return "TRANSCODER_DSI_C";
75 case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
76 return "TRANSCODER_VDSC_PW2";
77 case POWER_DOMAIN_PORT_DDI_LANES_A:
78 return "PORT_DDI_LANES_A";
79 case POWER_DOMAIN_PORT_DDI_LANES_B:
80 return "PORT_DDI_LANES_B";
81 case POWER_DOMAIN_PORT_DDI_LANES_C:
82 return "PORT_DDI_LANES_C";
83 case POWER_DOMAIN_PORT_DDI_LANES_D:
84 return "PORT_DDI_LANES_D";
85 case POWER_DOMAIN_PORT_DDI_LANES_E:
86 return "PORT_DDI_LANES_E";
87 case POWER_DOMAIN_PORT_DDI_LANES_F:
88 return "PORT_DDI_LANES_F";
89 case POWER_DOMAIN_PORT_DDI_LANES_TC1:
90 return "PORT_DDI_LANES_TC1";
91 case POWER_DOMAIN_PORT_DDI_LANES_TC2:
92 return "PORT_DDI_LANES_TC2";
93 case POWER_DOMAIN_PORT_DDI_LANES_TC3:
94 return "PORT_DDI_LANES_TC3";
95 case POWER_DOMAIN_PORT_DDI_LANES_TC4:
96 return "PORT_DDI_LANES_TC4";
97 case POWER_DOMAIN_PORT_DDI_LANES_TC5:
98 return "PORT_DDI_LANES_TC5";
99 case POWER_DOMAIN_PORT_DDI_LANES_TC6:
100 return "PORT_DDI_LANES_TC6";
101 case POWER_DOMAIN_PORT_DDI_IO_A:
102 return "PORT_DDI_IO_A";
103 case POWER_DOMAIN_PORT_DDI_IO_B:
104 return "PORT_DDI_IO_B";
105 case POWER_DOMAIN_PORT_DDI_IO_C:
106 return "PORT_DDI_IO_C";
107 case POWER_DOMAIN_PORT_DDI_IO_D:
108 return "PORT_DDI_IO_D";
109 case POWER_DOMAIN_PORT_DDI_IO_E:
110 return "PORT_DDI_IO_E";
111 case POWER_DOMAIN_PORT_DDI_IO_F:
112 return "PORT_DDI_IO_F";
113 case POWER_DOMAIN_PORT_DDI_IO_TC1:
114 return "PORT_DDI_IO_TC1";
115 case POWER_DOMAIN_PORT_DDI_IO_TC2:
116 return "PORT_DDI_IO_TC2";
117 case POWER_DOMAIN_PORT_DDI_IO_TC3:
118 return "PORT_DDI_IO_TC3";
119 case POWER_DOMAIN_PORT_DDI_IO_TC4:
120 return "PORT_DDI_IO_TC4";
121 case POWER_DOMAIN_PORT_DDI_IO_TC5:
122 return "PORT_DDI_IO_TC5";
123 case POWER_DOMAIN_PORT_DDI_IO_TC6:
124 return "PORT_DDI_IO_TC6";
125 case POWER_DOMAIN_PORT_DSI:
127 case POWER_DOMAIN_PORT_CRT:
129 case POWER_DOMAIN_PORT_OTHER:
131 case POWER_DOMAIN_VGA:
133 case POWER_DOMAIN_AUDIO_MMIO:
135 case POWER_DOMAIN_AUDIO_PLAYBACK:
136 return "AUDIO_PLAYBACK";
137 case POWER_DOMAIN_AUX_IO_A:
139 case POWER_DOMAIN_AUX_IO_B:
141 case POWER_DOMAIN_AUX_IO_C:
143 case POWER_DOMAIN_AUX_IO_D:
145 case POWER_DOMAIN_AUX_IO_E:
147 case POWER_DOMAIN_AUX_IO_F:
149 case POWER_DOMAIN_AUX_A:
151 case POWER_DOMAIN_AUX_B:
153 case POWER_DOMAIN_AUX_C:
155 case POWER_DOMAIN_AUX_D:
157 case POWER_DOMAIN_AUX_E:
159 case POWER_DOMAIN_AUX_F:
161 case POWER_DOMAIN_AUX_USBC1:
163 case POWER_DOMAIN_AUX_USBC2:
165 case POWER_DOMAIN_AUX_USBC3:
167 case POWER_DOMAIN_AUX_USBC4:
169 case POWER_DOMAIN_AUX_USBC5:
171 case POWER_DOMAIN_AUX_USBC6:
173 case POWER_DOMAIN_AUX_TBT1:
175 case POWER_DOMAIN_AUX_TBT2:
177 case POWER_DOMAIN_AUX_TBT3:
179 case POWER_DOMAIN_AUX_TBT4:
181 case POWER_DOMAIN_AUX_TBT5:
183 case POWER_DOMAIN_AUX_TBT6:
185 case POWER_DOMAIN_GMBUS:
187 case POWER_DOMAIN_INIT:
189 case POWER_DOMAIN_MODESET:
191 case POWER_DOMAIN_GT_IRQ:
193 case POWER_DOMAIN_DC_OFF:
195 case POWER_DOMAIN_TC_COLD_OFF:
196 return "TC_COLD_OFF";
198 MISSING_CASE(domain);
204 * __intel_display_power_is_enabled - unlocked check for a power domain
205 * @dev_priv: i915 device instance
206 * @domain: power domain to check
208 * This is the unlocked version of intel_display_power_is_enabled() and should
209 * only be used from error capture and recovery code where deadlocks are
213 * True when the power domain is enabled, false otherwise.
215 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
216 enum intel_display_power_domain domain)
218 struct i915_power_well *power_well;
221 if (dev_priv->runtime_pm.suspended)
226 for_each_power_domain_well_reverse(dev_priv, power_well, domain) {
227 if (intel_power_well_is_always_on(power_well))
230 if (!intel_power_well_is_enabled_cached(power_well)) {
240 * intel_display_power_is_enabled - check for a power domain
241 * @dev_priv: i915 device instance
242 * @domain: power domain to check
244 * This function can be used to check the hw power domain state. It is mostly
245 * used in hardware state readout functions. Everywhere else code should rely
246 * upon explicit power domain reference counting to ensure that the hardware
247 * block is powered up before accessing it.
249 * Callers must hold the relevant modesetting locks to ensure that concurrent
250 * threads can't disable the power well while the caller tries to read a few
254 * True when the power domain is enabled, false otherwise.
256 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
257 enum intel_display_power_domain domain)
259 struct i915_power_domains *power_domains;
262 power_domains = &dev_priv->display.power.domains;
264 mutex_lock(&power_domains->lock);
265 ret = __intel_display_power_is_enabled(dev_priv, domain);
266 mutex_unlock(&power_domains->lock);
272 sanitize_target_dc_state(struct drm_i915_private *i915,
275 struct i915_power_domains *power_domains = &i915->display.power.domains;
276 static const u32 states[] = {
277 DC_STATE_EN_UPTO_DC6,
278 DC_STATE_EN_UPTO_DC5,
284 for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
285 if (target_dc_state != states[i])
288 if (power_domains->allowed_dc_mask & target_dc_state)
291 target_dc_state = states[i + 1];
294 return target_dc_state;
298 * intel_display_power_set_target_dc_state - Set target dc state.
299 * @dev_priv: i915 device
300 * @state: state which needs to be set as target_dc_state.
302 * This function set the "DC off" power well target_dc_state,
303 * based upon this target_dc_stste, "DC off" power well will
304 * enable desired DC state.
306 void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
309 struct i915_power_well *power_well;
311 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
313 mutex_lock(&power_domains->lock);
314 power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
316 if (drm_WARN_ON(&dev_priv->drm, !power_well))
319 state = sanitize_target_dc_state(dev_priv, state);
321 if (state == power_domains->target_dc_state)
324 dc_off_enabled = intel_power_well_is_enabled(dev_priv, power_well);
326 * If DC off power well is disabled, need to enable and disable the
327 * DC off power well to effect target DC state.
330 intel_power_well_enable(dev_priv, power_well);
332 power_domains->target_dc_state = state;
335 intel_power_well_disable(dev_priv, power_well);
338 mutex_unlock(&power_domains->lock);
341 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
343 static void __async_put_domains_mask(struct i915_power_domains *power_domains,
344 struct intel_power_domain_mask *mask)
346 bitmap_or(mask->bits,
347 power_domains->async_put_domains[0].bits,
348 power_domains->async_put_domains[1].bits,
352 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
355 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
357 struct drm_i915_private *i915 = container_of(power_domains,
358 struct drm_i915_private,
359 display.power.domains);
361 return !drm_WARN_ON(&i915->drm,
362 bitmap_intersects(power_domains->async_put_domains[0].bits,
363 power_domains->async_put_domains[1].bits,
368 __async_put_domains_state_ok(struct i915_power_domains *power_domains)
370 struct drm_i915_private *i915 = container_of(power_domains,
371 struct drm_i915_private,
372 display.power.domains);
373 struct intel_power_domain_mask async_put_mask;
374 enum intel_display_power_domain domain;
377 err |= !assert_async_put_domain_masks_disjoint(power_domains);
378 __async_put_domains_mask(power_domains, &async_put_mask);
379 err |= drm_WARN_ON(&i915->drm,
380 !!power_domains->async_put_wakeref !=
381 !bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM));
383 for_each_power_domain(domain, &async_put_mask)
384 err |= drm_WARN_ON(&i915->drm,
385 power_domains->domain_use_count[domain] != 1);
390 static void print_power_domains(struct i915_power_domains *power_domains,
391 const char *prefix, struct intel_power_domain_mask *mask)
393 struct drm_i915_private *i915 = container_of(power_domains,
394 struct drm_i915_private,
395 display.power.domains);
396 enum intel_display_power_domain domain;
398 drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM));
399 for_each_power_domain(domain, mask)
400 drm_dbg(&i915->drm, "%s use_count %d\n",
401 intel_display_power_domain_str(domain),
402 power_domains->domain_use_count[domain]);
406 print_async_put_domains_state(struct i915_power_domains *power_domains)
408 struct drm_i915_private *i915 = container_of(power_domains,
409 struct drm_i915_private,
410 display.power.domains);
412 drm_dbg(&i915->drm, "async_put_wakeref %u\n",
413 power_domains->async_put_wakeref);
415 print_power_domains(power_domains, "async_put_domains[0]",
416 &power_domains->async_put_domains[0]);
417 print_power_domains(power_domains, "async_put_domains[1]",
418 &power_domains->async_put_domains[1]);
422 verify_async_put_domains_state(struct i915_power_domains *power_domains)
424 if (!__async_put_domains_state_ok(power_domains))
425 print_async_put_domains_state(power_domains);
431 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
436 verify_async_put_domains_state(struct i915_power_domains *power_domains)
440 #endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
442 static void async_put_domains_mask(struct i915_power_domains *power_domains,
443 struct intel_power_domain_mask *mask)
446 assert_async_put_domain_masks_disjoint(power_domains);
448 __async_put_domains_mask(power_domains, mask);
452 async_put_domains_clear_domain(struct i915_power_domains *power_domains,
453 enum intel_display_power_domain domain)
455 assert_async_put_domain_masks_disjoint(power_domains);
457 clear_bit(domain, power_domains->async_put_domains[0].bits);
458 clear_bit(domain, power_domains->async_put_domains[1].bits);
462 intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
463 enum intel_display_power_domain domain)
465 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
466 struct intel_power_domain_mask async_put_mask;
469 async_put_domains_mask(power_domains, &async_put_mask);
470 if (!test_bit(domain, async_put_mask.bits))
473 async_put_domains_clear_domain(power_domains, domain);
477 async_put_domains_mask(power_domains, &async_put_mask);
478 if (!bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM))
481 cancel_delayed_work(&power_domains->async_put_work);
482 intel_runtime_pm_put_raw(&dev_priv->runtime_pm,
483 fetch_and_zero(&power_domains->async_put_wakeref));
485 verify_async_put_domains_state(power_domains);
491 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
492 enum intel_display_power_domain domain)
494 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
495 struct i915_power_well *power_well;
497 if (intel_display_power_grab_async_put_ref(dev_priv, domain))
500 for_each_power_domain_well(dev_priv, power_well, domain)
501 intel_power_well_get(dev_priv, power_well);
503 power_domains->domain_use_count[domain]++;
507 * intel_display_power_get - grab a power domain reference
508 * @dev_priv: i915 device instance
509 * @domain: power domain to reference
511 * This function grabs a power domain reference for @domain and ensures that the
512 * power domain and all its parents are powered up. Therefore users should only
513 * grab a reference to the innermost power domain they need.
515 * Any power domain reference obtained by this function must have a symmetric
516 * call to intel_display_power_put() to release the reference again.
518 intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
519 enum intel_display_power_domain domain)
521 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
522 intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
524 mutex_lock(&power_domains->lock);
525 __intel_display_power_get_domain(dev_priv, domain);
526 mutex_unlock(&power_domains->lock);
532 * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
533 * @dev_priv: i915 device instance
534 * @domain: power domain to reference
536 * This function grabs a power domain reference for @domain and ensures that the
537 * power domain and all its parents are powered up. Therefore users should only
538 * grab a reference to the innermost power domain they need.
540 * Any power domain reference obtained by this function must have a symmetric
541 * call to intel_display_power_put() to release the reference again.
544 intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
545 enum intel_display_power_domain domain)
547 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
548 intel_wakeref_t wakeref;
551 wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
555 mutex_lock(&power_domains->lock);
557 if (__intel_display_power_is_enabled(dev_priv, domain)) {
558 __intel_display_power_get_domain(dev_priv, domain);
564 mutex_unlock(&power_domains->lock);
567 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
575 __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
576 enum intel_display_power_domain domain)
578 struct i915_power_domains *power_domains;
579 struct i915_power_well *power_well;
580 const char *name = intel_display_power_domain_str(domain);
581 struct intel_power_domain_mask async_put_mask;
583 power_domains = &dev_priv->display.power.domains;
585 drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
586 "Use count on domain %s is already zero\n",
588 async_put_domains_mask(power_domains, &async_put_mask);
589 drm_WARN(&dev_priv->drm,
590 test_bit(domain, async_put_mask.bits),
591 "Async disabling of domain %s is pending\n",
594 power_domains->domain_use_count[domain]--;
596 for_each_power_domain_well_reverse(dev_priv, power_well, domain)
597 intel_power_well_put(dev_priv, power_well);
600 static void __intel_display_power_put(struct drm_i915_private *dev_priv,
601 enum intel_display_power_domain domain)
603 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
605 mutex_lock(&power_domains->lock);
606 __intel_display_power_put_domain(dev_priv, domain);
607 mutex_unlock(&power_domains->lock);
611 queue_async_put_domains_work(struct i915_power_domains *power_domains,
612 intel_wakeref_t wakeref)
614 struct drm_i915_private *i915 = container_of(power_domains,
615 struct drm_i915_private,
616 display.power.domains);
617 drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
618 power_domains->async_put_wakeref = wakeref;
619 drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
620 &power_domains->async_put_work,
621 msecs_to_jiffies(100)));
625 release_async_put_domains(struct i915_power_domains *power_domains,
626 struct intel_power_domain_mask *mask)
628 struct drm_i915_private *dev_priv =
629 container_of(power_domains, struct drm_i915_private,
630 display.power.domains);
631 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
632 enum intel_display_power_domain domain;
633 intel_wakeref_t wakeref;
636 * The caller must hold already raw wakeref, upgrade that to a proper
637 * wakeref to make the state checker happy about the HW access during
638 * power well disabling.
640 assert_rpm_raw_wakeref_held(rpm);
641 wakeref = intel_runtime_pm_get(rpm);
643 for_each_power_domain(domain, mask) {
644 /* Clear before put, so put's sanity check is happy. */
645 async_put_domains_clear_domain(power_domains, domain);
646 __intel_display_power_put_domain(dev_priv, domain);
649 intel_runtime_pm_put(rpm, wakeref);
653 intel_display_power_put_async_work(struct work_struct *work)
655 struct drm_i915_private *dev_priv =
656 container_of(work, struct drm_i915_private,
657 display.power.domains.async_put_work.work);
658 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
659 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
660 intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
661 intel_wakeref_t old_work_wakeref = 0;
663 mutex_lock(&power_domains->lock);
666 * Bail out if all the domain refs pending to be released were grabbed
667 * by subsequent gets or a flush_work.
669 old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
670 if (!old_work_wakeref)
673 release_async_put_domains(power_domains,
674 &power_domains->async_put_domains[0]);
676 /* Requeue the work if more domains were async put meanwhile. */
677 if (!bitmap_empty(power_domains->async_put_domains[1].bits, POWER_DOMAIN_NUM)) {
678 bitmap_copy(power_domains->async_put_domains[0].bits,
679 power_domains->async_put_domains[1].bits,
681 bitmap_zero(power_domains->async_put_domains[1].bits,
683 queue_async_put_domains_work(power_domains,
684 fetch_and_zero(&new_work_wakeref));
687 * Cancel the work that got queued after this one got dequeued,
688 * since here we released the corresponding async-put reference.
690 cancel_delayed_work(&power_domains->async_put_work);
694 verify_async_put_domains_state(power_domains);
696 mutex_unlock(&power_domains->lock);
698 if (old_work_wakeref)
699 intel_runtime_pm_put_raw(rpm, old_work_wakeref);
700 if (new_work_wakeref)
701 intel_runtime_pm_put_raw(rpm, new_work_wakeref);
705 * __intel_display_power_put_async - release a power domain reference asynchronously
706 * @i915: i915 device instance
707 * @domain: power domain to reference
708 * @wakeref: wakeref acquired for the reference that is being released
710 * This function drops the power domain reference obtained by
711 * intel_display_power_get*() and schedules a work to power down the
712 * corresponding hardware block if this is the last reference.
714 void __intel_display_power_put_async(struct drm_i915_private *i915,
715 enum intel_display_power_domain domain,
716 intel_wakeref_t wakeref)
718 struct i915_power_domains *power_domains = &i915->display.power.domains;
719 struct intel_runtime_pm *rpm = &i915->runtime_pm;
720 intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
722 mutex_lock(&power_domains->lock);
724 if (power_domains->domain_use_count[domain] > 1) {
725 __intel_display_power_put_domain(i915, domain);
730 drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
732 /* Let a pending work requeue itself or queue a new one. */
733 if (power_domains->async_put_wakeref) {
734 set_bit(domain, power_domains->async_put_domains[1].bits);
736 set_bit(domain, power_domains->async_put_domains[0].bits);
737 queue_async_put_domains_work(power_domains,
738 fetch_and_zero(&work_wakeref));
742 verify_async_put_domains_state(power_domains);
744 mutex_unlock(&power_domains->lock);
747 intel_runtime_pm_put_raw(rpm, work_wakeref);
749 intel_runtime_pm_put(rpm, wakeref);
753 * intel_display_power_flush_work - flushes the async display power disabling work
754 * @i915: i915 device instance
756 * Flushes any pending work that was scheduled by a preceding
757 * intel_display_power_put_async() call, completing the disabling of the
758 * corresponding power domains.
760 * Note that the work handler function may still be running after this
761 * function returns; to ensure that the work handler isn't running use
762 * intel_display_power_flush_work_sync() instead.
764 void intel_display_power_flush_work(struct drm_i915_private *i915)
766 struct i915_power_domains *power_domains = &i915->display.power.domains;
767 struct intel_power_domain_mask async_put_mask;
768 intel_wakeref_t work_wakeref;
770 mutex_lock(&power_domains->lock);
772 work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
776 async_put_domains_mask(power_domains, &async_put_mask);
777 release_async_put_domains(power_domains, &async_put_mask);
778 cancel_delayed_work(&power_domains->async_put_work);
781 verify_async_put_domains_state(power_domains);
783 mutex_unlock(&power_domains->lock);
786 intel_runtime_pm_put_raw(&i915->runtime_pm, work_wakeref);
790 * intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
791 * @i915: i915 device instance
793 * Like intel_display_power_flush_work(), but also ensure that the work
794 * handler function is not running any more when this function returns.
797 intel_display_power_flush_work_sync(struct drm_i915_private *i915)
799 struct i915_power_domains *power_domains = &i915->display.power.domains;
801 intel_display_power_flush_work(i915);
802 cancel_delayed_work_sync(&power_domains->async_put_work);
804 verify_async_put_domains_state(power_domains);
806 drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
809 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
811 * intel_display_power_put - release a power domain reference
812 * @dev_priv: i915 device instance
813 * @domain: power domain to reference
814 * @wakeref: wakeref acquired for the reference that is being released
816 * This function drops the power domain reference obtained by
817 * intel_display_power_get() and might power down the corresponding hardware
818 * block right away if this is the last reference.
820 void intel_display_power_put(struct drm_i915_private *dev_priv,
821 enum intel_display_power_domain domain,
822 intel_wakeref_t wakeref)
824 __intel_display_power_put(dev_priv, domain);
825 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
829 * intel_display_power_put_unchecked - release an unchecked power domain reference
830 * @dev_priv: i915 device instance
831 * @domain: power domain to reference
833 * This function drops the power domain reference obtained by
834 * intel_display_power_get() and might power down the corresponding hardware
835 * block right away if this is the last reference.
837 * This function is only for the power domain code's internal use to suppress wakeref
838 * tracking when the correspondig debug kconfig option is disabled, should not
841 void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
842 enum intel_display_power_domain domain)
844 __intel_display_power_put(dev_priv, domain);
845 intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
850 intel_display_power_get_in_set(struct drm_i915_private *i915,
851 struct intel_display_power_domain_set *power_domain_set,
852 enum intel_display_power_domain domain)
854 intel_wakeref_t __maybe_unused wf;
856 drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
858 wf = intel_display_power_get(i915, domain);
859 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
860 power_domain_set->wakerefs[domain] = wf;
862 set_bit(domain, power_domain_set->mask.bits);
866 intel_display_power_get_in_set_if_enabled(struct drm_i915_private *i915,
867 struct intel_display_power_domain_set *power_domain_set,
868 enum intel_display_power_domain domain)
872 drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
874 wf = intel_display_power_get_if_enabled(i915, domain);
878 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
879 power_domain_set->wakerefs[domain] = wf;
881 set_bit(domain, power_domain_set->mask.bits);
887 intel_display_power_put_mask_in_set(struct drm_i915_private *i915,
888 struct intel_display_power_domain_set *power_domain_set,
889 struct intel_power_domain_mask *mask)
891 enum intel_display_power_domain domain;
893 drm_WARN_ON(&i915->drm,
894 !bitmap_subset(mask->bits, power_domain_set->mask.bits, POWER_DOMAIN_NUM));
896 for_each_power_domain(domain, mask) {
897 intel_wakeref_t __maybe_unused wf = -1;
899 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
900 wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
902 intel_display_power_put(i915, domain, wf);
903 clear_bit(domain, power_domain_set->mask.bits);
908 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
909 int disable_power_well)
911 if (disable_power_well >= 0)
912 return !!disable_power_well;
917 static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
924 if (!HAS_DISPLAY(dev_priv))
927 if (IS_DG2(dev_priv))
929 else if (IS_DG1(dev_priv))
931 else if (DISPLAY_VER(dev_priv) >= 12)
933 else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
935 else if (DISPLAY_VER(dev_priv) >= 9)
941 * DC9 has a separate HW flow from the rest of the DC states,
942 * not depending on the DMC firmware. It's needed by system
943 * suspend/resume, so allow it unconditionally.
945 mask = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ||
946 DISPLAY_VER(dev_priv) >= 11 ?
949 if (!dev_priv->params.disable_power_well)
952 if (enable_dc >= 0 && enable_dc <= max_dc) {
953 requested_dc = enable_dc;
954 } else if (enable_dc == -1) {
955 requested_dc = max_dc;
956 } else if (enable_dc > max_dc && enable_dc <= 4) {
957 drm_dbg_kms(&dev_priv->drm,
958 "Adjusting requested max DC state (%d->%d)\n",
960 requested_dc = max_dc;
962 drm_err(&dev_priv->drm,
963 "Unexpected value for enable_dc (%d)\n", enable_dc);
964 requested_dc = max_dc;
967 switch (requested_dc) {
969 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
972 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
975 mask |= DC_STATE_EN_UPTO_DC6;
978 mask |= DC_STATE_EN_UPTO_DC5;
982 drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
988 * intel_power_domains_init - initializes the power domain structures
989 * @dev_priv: i915 device instance
991 * Initializes the power domain structures for @dev_priv depending upon the
992 * supported platform.
994 int intel_power_domains_init(struct drm_i915_private *dev_priv)
996 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
998 dev_priv->params.disable_power_well =
999 sanitize_disable_power_well_option(dev_priv,
1000 dev_priv->params.disable_power_well);
1001 power_domains->allowed_dc_mask =
1002 get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc);
1004 power_domains->target_dc_state =
1005 sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
1007 mutex_init(&power_domains->lock);
1009 INIT_DELAYED_WORK(&power_domains->async_put_work,
1010 intel_display_power_put_async_work);
1012 return intel_display_power_map_init(power_domains);
1016 * intel_power_domains_cleanup - clean up power domains resources
1017 * @dev_priv: i915 device instance
1019 * Release any resources acquired by intel_power_domains_init()
1021 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
1023 intel_display_power_map_cleanup(&dev_priv->display.power.domains);
1026 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
1028 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1029 struct i915_power_well *power_well;
1031 mutex_lock(&power_domains->lock);
1032 for_each_power_well(dev_priv, power_well)
1033 intel_power_well_sync_hw(dev_priv, power_well);
1034 mutex_unlock(&power_domains->lock);
1037 static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
1038 enum dbuf_slice slice, bool enable)
1040 i915_reg_t reg = DBUF_CTL_S(slice);
1043 intel_de_rmw(dev_priv, reg, DBUF_POWER_REQUEST,
1044 enable ? DBUF_POWER_REQUEST : 0);
1045 intel_de_posting_read(dev_priv, reg);
1048 state = intel_de_read(dev_priv, reg) & DBUF_POWER_STATE;
1049 drm_WARN(&dev_priv->drm, enable != state,
1050 "DBuf slice %d power %s timeout!\n",
1051 slice, str_enable_disable(enable));
1054 void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
1057 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1058 u8 slice_mask = DISPLAY_INFO(dev_priv)->dbuf.slice_mask;
1059 enum dbuf_slice slice;
1061 drm_WARN(&dev_priv->drm, req_slices & ~slice_mask,
1062 "Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
1063 req_slices, slice_mask);
1065 drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
1069 * Might be running this in parallel to gen9_dc_off_power_well_enable
1070 * being called from intel_dp_detect for instance,
1071 * which causes assertion triggered by race condition,
1072 * as gen9_assert_dbuf_enabled might preempt this when registers
1073 * were already updated, while dev_priv was not.
1075 mutex_lock(&power_domains->lock);
1077 for_each_dbuf_slice(dev_priv, slice)
1078 gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice));
1080 dev_priv->display.dbuf.enabled_slices = req_slices;
1082 mutex_unlock(&power_domains->lock);
1085 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
1089 dev_priv->display.dbuf.enabled_slices =
1090 intel_enabled_dbuf_slices_mask(dev_priv);
1092 slices_mask = BIT(DBUF_S1) | dev_priv->display.dbuf.enabled_slices;
1094 if (DISPLAY_VER(dev_priv) >= 14)
1095 intel_pmdemand_program_dbuf(dev_priv, slices_mask);
1098 * Just power up at least 1 slice, we will
1099 * figure out later which slices we have and what we need.
1101 gen9_dbuf_slices_update(dev_priv, slices_mask);
1104 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
1106 gen9_dbuf_slices_update(dev_priv, 0);
1108 if (DISPLAY_VER(dev_priv) >= 14)
1109 intel_pmdemand_program_dbuf(dev_priv, 0);
1112 static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv)
1114 enum dbuf_slice slice;
1116 if (IS_ALDERLAKE_P(dev_priv))
1119 for_each_dbuf_slice(dev_priv, slice)
1120 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
1121 DBUF_TRACKER_STATE_SERVICE_MASK,
1122 DBUF_TRACKER_STATE_SERVICE(8));
1125 static void icl_mbus_init(struct drm_i915_private *dev_priv)
1127 unsigned long abox_regs = DISPLAY_INFO(dev_priv)->abox_mask;
1130 if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
1133 mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
1134 MBUS_ABOX_BT_CREDIT_POOL2_MASK |
1135 MBUS_ABOX_B_CREDIT_MASK |
1136 MBUS_ABOX_BW_CREDIT_MASK;
1137 val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
1138 MBUS_ABOX_BT_CREDIT_POOL2(16) |
1139 MBUS_ABOX_B_CREDIT(1) |
1140 MBUS_ABOX_BW_CREDIT(1);
1143 * gen12 platforms that use abox1 and abox2 for pixel data reads still
1144 * expect us to program the abox_ctl0 register as well, even though
1145 * we don't have to program other instance-0 registers like BW_BUDDY.
1147 if (DISPLAY_VER(dev_priv) == 12)
1148 abox_regs |= BIT(0);
1150 for_each_set_bit(i, &abox_regs, sizeof(abox_regs))
1151 intel_de_rmw(dev_priv, MBUS_ABOX_CTL(i), mask, val);
1154 static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
1156 u32 val = intel_de_read(dev_priv, LCPLL_CTL);
1159 * The LCPLL register should be turned on by the BIOS. For now
1160 * let's just check its state and print errors in case
1161 * something is wrong. Don't even try to turn it on.
1164 if (val & LCPLL_CD_SOURCE_FCLK)
1165 drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
1167 if (val & LCPLL_PLL_DISABLE)
1168 drm_err(&dev_priv->drm, "LCPLL is disabled\n");
1170 if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
1171 drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
1174 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
1176 struct intel_crtc *crtc;
1178 for_each_intel_crtc(&dev_priv->drm, crtc)
1179 I915_STATE_WARN(dev_priv, crtc->active,
1180 "CRTC for pipe %c enabled\n",
1181 pipe_name(crtc->pipe));
1183 I915_STATE_WARN(dev_priv, intel_de_read(dev_priv, HSW_PWR_WELL_CTL2),
1184 "Display power well on\n");
1185 I915_STATE_WARN(dev_priv,
1186 intel_de_read(dev_priv, SPLL_CTL) & SPLL_PLL_ENABLE,
1188 I915_STATE_WARN(dev_priv,
1189 intel_de_read(dev_priv, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
1190 "WRPLL1 enabled\n");
1191 I915_STATE_WARN(dev_priv,
1192 intel_de_read(dev_priv, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
1193 "WRPLL2 enabled\n");
1194 I915_STATE_WARN(dev_priv,
1195 intel_de_read(dev_priv, PP_STATUS(0)) & PP_ON,
1196 "Panel power on\n");
1197 I915_STATE_WARN(dev_priv,
1198 intel_de_read(dev_priv, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
1199 "CPU PWM1 enabled\n");
1200 if (IS_HASWELL(dev_priv))
1201 I915_STATE_WARN(dev_priv,
1202 intel_de_read(dev_priv, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
1203 "CPU PWM2 enabled\n");
1204 I915_STATE_WARN(dev_priv,
1205 intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
1206 "PCH PWM1 enabled\n");
1207 I915_STATE_WARN(dev_priv,
1208 (intel_de_read(dev_priv, UTIL_PIN_CTL) & (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) == (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
1209 "Utility pin enabled in PWM mode\n");
1210 I915_STATE_WARN(dev_priv,
1211 intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
1212 "PCH GTC enabled\n");
1215 * In theory we can still leave IRQs enabled, as long as only the HPD
1216 * interrupts remain enabled. We used to check for that, but since it's
1217 * gen-specific and since we only disable LCPLL after we fully disable
1218 * the interrupts, the check below should be enough.
1220 I915_STATE_WARN(dev_priv, intel_irqs_enabled(dev_priv),
1224 static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
1226 if (IS_HASWELL(dev_priv))
1227 return intel_de_read(dev_priv, D_COMP_HSW);
1229 return intel_de_read(dev_priv, D_COMP_BDW);
1232 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
1234 if (IS_HASWELL(dev_priv)) {
1235 if (snb_pcode_write(&dev_priv->uncore, GEN6_PCODE_WRITE_D_COMP, val))
1236 drm_dbg_kms(&dev_priv->drm,
1237 "Failed to write to D_COMP\n");
1239 intel_de_write(dev_priv, D_COMP_BDW, val);
1240 intel_de_posting_read(dev_priv, D_COMP_BDW);
1245 * This function implements pieces of two sequences from BSpec:
1246 * - Sequence for display software to disable LCPLL
1247 * - Sequence for display software to allow package C8+
1248 * The steps implemented here are just the steps that actually touch the LCPLL
1249 * register. Callers should take care of disabling all the display engine
1250 * functions, doing the mode unset, fixing interrupts, etc.
1252 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
1253 bool switch_to_fclk, bool allow_power_down)
1257 assert_can_disable_lcpll(dev_priv);
1259 val = intel_de_read(dev_priv, LCPLL_CTL);
1261 if (switch_to_fclk) {
1262 val |= LCPLL_CD_SOURCE_FCLK;
1263 intel_de_write(dev_priv, LCPLL_CTL, val);
1265 if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
1266 LCPLL_CD_SOURCE_FCLK_DONE, 1))
1267 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
1269 val = intel_de_read(dev_priv, LCPLL_CTL);
1272 val |= LCPLL_PLL_DISABLE;
1273 intel_de_write(dev_priv, LCPLL_CTL, val);
1274 intel_de_posting_read(dev_priv, LCPLL_CTL);
1276 if (intel_de_wait_for_clear(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 1))
1277 drm_err(&dev_priv->drm, "LCPLL still locked\n");
1279 val = hsw_read_dcomp(dev_priv);
1280 val |= D_COMP_COMP_DISABLE;
1281 hsw_write_dcomp(dev_priv, val);
1284 if (wait_for((hsw_read_dcomp(dev_priv) &
1285 D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
1286 drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
1288 if (allow_power_down) {
1289 intel_de_rmw(dev_priv, LCPLL_CTL, 0, LCPLL_POWER_DOWN_ALLOW);
1290 intel_de_posting_read(dev_priv, LCPLL_CTL);
1295 * Fully restores LCPLL, disallowing power down and switching back to LCPLL
1298 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
1302 val = intel_de_read(dev_priv, LCPLL_CTL);
1304 if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
1305 LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
1309 * Make sure we're not on PC8 state before disabling PC8, otherwise
1310 * we'll hang the machine. To prevent PC8 state, just enable force_wake.
1312 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
1314 if (val & LCPLL_POWER_DOWN_ALLOW) {
1315 val &= ~LCPLL_POWER_DOWN_ALLOW;
1316 intel_de_write(dev_priv, LCPLL_CTL, val);
1317 intel_de_posting_read(dev_priv, LCPLL_CTL);
1320 val = hsw_read_dcomp(dev_priv);
1321 val |= D_COMP_COMP_FORCE;
1322 val &= ~D_COMP_COMP_DISABLE;
1323 hsw_write_dcomp(dev_priv, val);
1325 val = intel_de_read(dev_priv, LCPLL_CTL);
1326 val &= ~LCPLL_PLL_DISABLE;
1327 intel_de_write(dev_priv, LCPLL_CTL, val);
1329 if (intel_de_wait_for_set(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 5))
1330 drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
1332 if (val & LCPLL_CD_SOURCE_FCLK) {
1333 intel_de_rmw(dev_priv, LCPLL_CTL, LCPLL_CD_SOURCE_FCLK, 0);
1335 if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
1336 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
1337 drm_err(&dev_priv->drm,
1338 "Switching back to LCPLL failed\n");
1341 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1343 intel_update_cdclk(dev_priv);
1344 intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1348 * Package states C8 and deeper are really deep PC states that can only be
1349 * reached when all the devices on the system allow it, so even if the graphics
1350 * device allows PC8+, it doesn't mean the system will actually get to these
1351 * states. Our driver only allows PC8+ when going into runtime PM.
1353 * The requirements for PC8+ are that all the outputs are disabled, the power
1354 * well is disabled and most interrupts are disabled, and these are also
1355 * requirements for runtime PM. When these conditions are met, we manually do
1356 * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
1357 * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
1360 * When we really reach PC8 or deeper states (not just when we allow it) we lose
1361 * the state of some registers, so when we come back from PC8+ we need to
1362 * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
1363 * need to take care of the registers kept by RC6. Notice that this happens even
1364 * if we don't put the device in PCI D3 state (which is what currently happens
1365 * because of the runtime PM support).
1367 * For more, read "Display Sequences for Package C8" on the hardware
1370 static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
1372 drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
1374 if (HAS_PCH_LPT_LP(dev_priv))
1375 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1376 PCH_LP_PARTITION_LEVEL_DISABLE, 0);
1378 lpt_disable_clkout_dp(dev_priv);
1379 hsw_disable_lcpll(dev_priv, true, true);
1382 static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
1384 drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
1386 hsw_restore_lcpll(dev_priv);
1387 intel_init_pch_refclk(dev_priv);
1389 /* Many display registers don't survive PC8+ */
1390 intel_clock_gating_init(dev_priv);
1393 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
1399 if (IS_IVYBRIDGE(dev_priv)) {
1401 reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
1403 reg = HSW_NDE_RSTWRN_OPT;
1404 reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
1407 if (DISPLAY_VER(dev_priv) >= 14)
1408 reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
1410 intel_de_rmw(dev_priv, reg, reset_bits, enable ? reset_bits : 0);
1413 static void skl_display_core_init(struct drm_i915_private *dev_priv,
1416 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1417 struct i915_power_well *well;
1419 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1421 /* enable PCH reset handshake */
1422 intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1424 if (!HAS_DISPLAY(dev_priv))
1427 /* enable PG1 and Misc I/O */
1428 mutex_lock(&power_domains->lock);
1430 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1431 intel_power_well_enable(dev_priv, well);
1433 well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
1434 intel_power_well_enable(dev_priv, well);
1436 mutex_unlock(&power_domains->lock);
1438 intel_cdclk_init_hw(dev_priv);
1440 gen9_dbuf_enable(dev_priv);
1443 intel_dmc_load_program(dev_priv);
1446 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
1448 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1449 struct i915_power_well *well;
1451 if (!HAS_DISPLAY(dev_priv))
1454 gen9_disable_dc_states(dev_priv);
1455 /* TODO: disable DMC program */
1457 gen9_dbuf_disable(dev_priv);
1459 intel_cdclk_uninit_hw(dev_priv);
1461 /* The spec doesn't call for removing the reset handshake flag */
1462 /* disable PG1 and Misc I/O */
1464 mutex_lock(&power_domains->lock);
1467 * BSpec says to keep the MISC IO power well enabled here, only
1468 * remove our request for power well 1.
1469 * Note that even though the driver's request is removed power well 1
1470 * may stay enabled after this due to DMC's own request on it.
1472 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1473 intel_power_well_disable(dev_priv, well);
1475 mutex_unlock(&power_domains->lock);
1477 usleep_range(10, 30); /* 10 us delay per Bspec */
1480 static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
1482 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1483 struct i915_power_well *well;
1485 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1488 * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
1489 * or else the reset will hang because there is no PCH to respond.
1490 * Move the handshake programming to initialization sequence.
1491 * Previously was left up to BIOS.
1493 intel_pch_reset_handshake(dev_priv, false);
1495 if (!HAS_DISPLAY(dev_priv))
1499 mutex_lock(&power_domains->lock);
1501 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1502 intel_power_well_enable(dev_priv, well);
1504 mutex_unlock(&power_domains->lock);
1506 intel_cdclk_init_hw(dev_priv);
1508 gen9_dbuf_enable(dev_priv);
1511 intel_dmc_load_program(dev_priv);
1514 static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
1516 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1517 struct i915_power_well *well;
1519 if (!HAS_DISPLAY(dev_priv))
1522 gen9_disable_dc_states(dev_priv);
1523 /* TODO: disable DMC program */
1525 gen9_dbuf_disable(dev_priv);
1527 intel_cdclk_uninit_hw(dev_priv);
1529 /* The spec doesn't call for removing the reset handshake flag */
1532 * Disable PW1 (PG1).
1533 * Note that even though the driver's request is removed power well 1
1534 * may stay enabled after this due to DMC's own request on it.
1536 mutex_lock(&power_domains->lock);
1538 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1539 intel_power_well_disable(dev_priv, well);
1541 mutex_unlock(&power_domains->lock);
1543 usleep_range(10, 30); /* 10 us delay per Bspec */
1546 struct buddy_page_mask {
1552 static const struct buddy_page_mask tgl_buddy_page_masks[] = {
1553 { .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0xF },
1554 { .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0xF },
1555 { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
1556 { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
1557 { .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x1F },
1558 { .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x1E },
1559 { .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
1560 { .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
1564 static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
1565 { .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
1566 { .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0x1 },
1567 { .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0x1 },
1568 { .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
1569 { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
1570 { .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x3 },
1571 { .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x3 },
1572 { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
1576 static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
1578 enum intel_dram_type type = dev_priv->dram_info.type;
1579 u8 num_channels = dev_priv->dram_info.num_channels;
1580 const struct buddy_page_mask *table;
1581 unsigned long abox_mask = DISPLAY_INFO(dev_priv)->abox_mask;
1584 /* BW_BUDDY registers are not used on dgpu's beyond DG1 */
1585 if (IS_DGFX(dev_priv) && !IS_DG1(dev_priv))
1588 if (IS_ALDERLAKE_S(dev_priv) ||
1589 IS_RKL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
1591 table = wa_1409767108_buddy_page_masks;
1593 table = tgl_buddy_page_masks;
1595 for (config = 0; table[config].page_mask != 0; config++)
1596 if (table[config].num_channels == num_channels &&
1597 table[config].type == type)
1600 if (table[config].page_mask == 0) {
1601 drm_dbg(&dev_priv->drm,
1602 "Unknown memory configuration; disabling address buddy logic.\n");
1603 for_each_set_bit(i, &abox_mask, sizeof(abox_mask))
1604 intel_de_write(dev_priv, BW_BUDDY_CTL(i),
1607 for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) {
1608 intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i),
1609 table[config].page_mask);
1611 /* Wa_22010178259:tgl,dg1,rkl,adl-s */
1612 if (DISPLAY_VER(dev_priv) == 12)
1613 intel_de_rmw(dev_priv, BW_BUDDY_CTL(i),
1614 BW_BUDDY_TLB_REQ_TIMER_MASK,
1615 BW_BUDDY_TLB_REQ_TIMER(0x8));
1620 static void icl_display_core_init(struct drm_i915_private *dev_priv,
1623 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1624 struct i915_power_well *well;
1626 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1628 /* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
1629 if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
1630 INTEL_PCH_TYPE(dev_priv) < PCH_DG1)
1631 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D, 0,
1632 PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
1634 /* 1. Enable PCH reset handshake. */
1635 intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1637 if (!HAS_DISPLAY(dev_priv))
1640 /* 2. Initialize all combo phys */
1641 intel_combo_phy_init(dev_priv);
1644 * 3. Enable Power Well 1 (PG1).
1645 * The AUX IO power wells will be enabled on demand.
1647 mutex_lock(&power_domains->lock);
1648 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1649 intel_power_well_enable(dev_priv, well);
1650 mutex_unlock(&power_domains->lock);
1652 if (DISPLAY_VER(dev_priv) == 14)
1653 intel_de_rmw(dev_priv, DC_STATE_EN,
1654 HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH, 0);
1656 /* 4. Enable CDCLK. */
1657 intel_cdclk_init_hw(dev_priv);
1659 if (DISPLAY_VER(dev_priv) >= 12)
1660 gen12_dbuf_slices_config(dev_priv);
1662 /* 5. Enable DBUF. */
1663 gen9_dbuf_enable(dev_priv);
1665 /* 6. Setup MBUS. */
1666 icl_mbus_init(dev_priv);
1668 /* 7. Program arbiter BW_BUDDY registers */
1669 if (DISPLAY_VER(dev_priv) >= 12)
1670 tgl_bw_buddy_init(dev_priv);
1672 /* 8. Ensure PHYs have completed calibration and adaptation */
1673 if (IS_DG2(dev_priv))
1674 intel_snps_phy_wait_for_calibration(dev_priv);
1677 intel_dmc_load_program(dev_priv);
1679 /* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p */
1680 if (DISPLAY_VER(dev_priv) >= 12)
1681 intel_de_rmw(dev_priv, GEN11_CHICKEN_DCPR_2, 0,
1682 DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
1683 DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
1685 /* Wa_14011503030:xelpd */
1686 if (DISPLAY_VER(dev_priv) >= 13)
1687 intel_de_write(dev_priv, XELPD_DISPLAY_ERR_FATAL_MASK, ~0);
1690 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
1692 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1693 struct i915_power_well *well;
1695 if (!HAS_DISPLAY(dev_priv))
1698 gen9_disable_dc_states(dev_priv);
1699 intel_dmc_disable_program(dev_priv);
1701 /* 1. Disable all display engine functions -> aready done */
1703 /* 2. Disable DBUF */
1704 gen9_dbuf_disable(dev_priv);
1706 /* 3. Disable CD clock */
1707 intel_cdclk_uninit_hw(dev_priv);
1709 if (DISPLAY_VER(dev_priv) == 14)
1710 intel_de_rmw(dev_priv, DC_STATE_EN, 0,
1711 HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH);
1714 * 4. Disable Power Well 1 (PG1).
1715 * The AUX IO power wells are toggled on demand, so they are already
1716 * disabled at this point.
1718 mutex_lock(&power_domains->lock);
1719 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1720 intel_power_well_disable(dev_priv, well);
1721 mutex_unlock(&power_domains->lock);
1724 intel_combo_phy_uninit(dev_priv);
1727 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
1729 struct i915_power_well *cmn_bc =
1730 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1731 struct i915_power_well *cmn_d =
1732 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1735 * DISPLAY_PHY_CONTROL can get corrupted if read. As a
1736 * workaround never ever read DISPLAY_PHY_CONTROL, and
1737 * instead maintain a shadow copy ourselves. Use the actual
1738 * power well state and lane status to reconstruct the
1739 * expected initial value.
1741 dev_priv->display.power.chv_phy_control =
1742 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
1743 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
1744 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
1745 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
1746 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
1749 * If all lanes are disabled we leave the override disabled
1750 * with all power down bits cleared to match the state we
1751 * would use after disabling the port. Otherwise enable the
1752 * override and set the lane powerdown bits accding to the
1753 * current lane status.
1755 if (intel_power_well_is_enabled(dev_priv, cmn_bc)) {
1756 u32 status = intel_de_read(dev_priv, DPLL(PIPE_A));
1759 mask = status & DPLL_PORTB_READY_MASK;
1763 dev_priv->display.power.chv_phy_control |=
1764 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
1766 dev_priv->display.power.chv_phy_control |=
1767 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
1769 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
1773 dev_priv->display.power.chv_phy_control |=
1774 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
1776 dev_priv->display.power.chv_phy_control |=
1777 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
1779 dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
1781 dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = false;
1783 dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = true;
1786 if (intel_power_well_is_enabled(dev_priv, cmn_d)) {
1787 u32 status = intel_de_read(dev_priv, DPIO_PHY_STATUS);
1790 mask = status & DPLL_PORTD_READY_MASK;
1795 dev_priv->display.power.chv_phy_control |=
1796 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
1798 dev_priv->display.power.chv_phy_control |=
1799 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
1801 dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
1803 dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = false;
1805 dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = true;
1808 drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
1809 dev_priv->display.power.chv_phy_control);
1811 /* Defer application of initial phy_control to enabling the powerwell */
1814 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
1816 struct i915_power_well *cmn =
1817 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1818 struct i915_power_well *disp2d =
1819 lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
1821 /* If the display might be already active skip this */
1822 if (intel_power_well_is_enabled(dev_priv, cmn) &&
1823 intel_power_well_is_enabled(dev_priv, disp2d) &&
1824 intel_de_read(dev_priv, DPIO_CTL) & DPIO_CMNRST)
1827 drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
1829 /* cmnlane needs DPLL registers */
1830 intel_power_well_enable(dev_priv, disp2d);
1833 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
1834 * Need to assert and de-assert PHY SB reset by gating the
1835 * common lane power, then un-gating it.
1836 * Simply ungating isn't enough to reset the PHY enough to get
1837 * ports and lanes running.
1839 intel_power_well_disable(dev_priv, cmn);
1842 static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
1846 vlv_punit_get(dev_priv);
1847 ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
1848 vlv_punit_put(dev_priv);
1853 static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
1855 drm_WARN(&dev_priv->drm,
1856 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
1857 "VED not power gated\n");
1860 static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
1862 static const struct pci_device_id isp_ids[] = {
1863 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
1864 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
1868 drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
1869 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
1870 "ISP not power gated\n");
1873 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
1876 * intel_power_domains_init_hw - initialize hardware power domain state
1877 * @i915: i915 device instance
1878 * @resume: Called from resume code paths or not
1880 * This function initializes the hardware power domain state and enables all
1881 * power wells belonging to the INIT power domain. Power wells in other
1882 * domains (and not in the INIT domain) are referenced or disabled by
1883 * intel_modeset_readout_hw_state(). After that the reference count of each
1884 * power well must match its HW enabled state, see
1885 * intel_power_domains_verify_state().
1887 * It will return with power domains disabled (to be enabled later by
1888 * intel_power_domains_enable()) and must be paired with
1889 * intel_power_domains_driver_remove().
1891 void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
1893 struct i915_power_domains *power_domains = &i915->display.power.domains;
1895 power_domains->initializing = true;
1897 if (DISPLAY_VER(i915) >= 11) {
1898 icl_display_core_init(i915, resume);
1899 } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
1900 bxt_display_core_init(i915, resume);
1901 } else if (DISPLAY_VER(i915) == 9) {
1902 skl_display_core_init(i915, resume);
1903 } else if (IS_CHERRYVIEW(i915)) {
1904 mutex_lock(&power_domains->lock);
1905 chv_phy_control_init(i915);
1906 mutex_unlock(&power_domains->lock);
1907 assert_isp_power_gated(i915);
1908 } else if (IS_VALLEYVIEW(i915)) {
1909 mutex_lock(&power_domains->lock);
1910 vlv_cmnlane_wa(i915);
1911 mutex_unlock(&power_domains->lock);
1912 assert_ved_power_gated(i915);
1913 assert_isp_power_gated(i915);
1914 } else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
1915 hsw_assert_cdclk(i915);
1916 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1917 } else if (IS_IVYBRIDGE(i915)) {
1918 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1922 * Keep all power wells enabled for any dependent HW access during
1923 * initialization and to make sure we keep BIOS enabled display HW
1924 * resources powered until display HW readout is complete. We drop
1925 * this reference in intel_power_domains_enable().
1927 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
1928 power_domains->init_wakeref =
1929 intel_display_power_get(i915, POWER_DOMAIN_INIT);
1931 /* Disable power support if the user asked so. */
1932 if (!i915->params.disable_power_well) {
1933 drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
1934 i915->display.power.domains.disable_wakeref = intel_display_power_get(i915,
1937 intel_power_domains_sync_hw(i915);
1939 power_domains->initializing = false;
1943 * intel_power_domains_driver_remove - deinitialize hw power domain state
1944 * @i915: i915 device instance
1946 * De-initializes the display power domain HW state. It also ensures that the
1947 * device stays powered up so that the driver can be reloaded.
1949 * It must be called with power domains already disabled (after a call to
1950 * intel_power_domains_disable()) and must be paired with
1951 * intel_power_domains_init_hw().
1953 void intel_power_domains_driver_remove(struct drm_i915_private *i915)
1955 intel_wakeref_t wakeref __maybe_unused =
1956 fetch_and_zero(&i915->display.power.domains.init_wakeref);
1958 /* Remove the refcount we took to keep power well support disabled. */
1959 if (!i915->params.disable_power_well)
1960 intel_display_power_put(i915, POWER_DOMAIN_INIT,
1961 fetch_and_zero(&i915->display.power.domains.disable_wakeref));
1963 intel_display_power_flush_work_sync(i915);
1965 intel_power_domains_verify_state(i915);
1967 /* Keep the power well enabled, but cancel its rpm wakeref. */
1968 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1972 * intel_power_domains_sanitize_state - sanitize power domains state
1973 * @i915: i915 device instance
1975 * Sanitize the power domains state during driver loading and system resume.
1976 * The function will disable all display power wells that BIOS has enabled
1977 * without a user for it (any user for a power well has taken a reference
1978 * on it by the time this function is called, after the state of all the
1979 * pipe, encoder, etc. HW resources have been sanitized).
1981 void intel_power_domains_sanitize_state(struct drm_i915_private *i915)
1983 struct i915_power_domains *power_domains = &i915->display.power.domains;
1984 struct i915_power_well *power_well;
1986 mutex_lock(&power_domains->lock);
1988 for_each_power_well_reverse(i915, power_well) {
1989 if (power_well->desc->always_on || power_well->count ||
1990 !intel_power_well_is_enabled(i915, power_well))
1993 drm_dbg_kms(&i915->drm,
1994 "BIOS left unused %s power well enabled, disabling it\n",
1995 intel_power_well_name(power_well));
1996 intel_power_well_disable(i915, power_well);
1999 mutex_unlock(&power_domains->lock);
2003 * intel_power_domains_enable - enable toggling of display power wells
2004 * @i915: i915 device instance
2006 * Enable the ondemand enabling/disabling of the display power wells. Note that
2007 * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
2008 * only at specific points of the display modeset sequence, thus they are not
2009 * affected by the intel_power_domains_enable()/disable() calls. The purpose
2010 * of these function is to keep the rest of power wells enabled until the end
2011 * of display HW readout (which will acquire the power references reflecting
2012 * the current HW state).
2014 void intel_power_domains_enable(struct drm_i915_private *i915)
2016 intel_wakeref_t wakeref __maybe_unused =
2017 fetch_and_zero(&i915->display.power.domains.init_wakeref);
2019 intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2020 intel_power_domains_verify_state(i915);
2024 * intel_power_domains_disable - disable toggling of display power wells
2025 * @i915: i915 device instance
2027 * Disable the ondemand enabling/disabling of the display power wells. See
2028 * intel_power_domains_enable() for which power wells this call controls.
2030 void intel_power_domains_disable(struct drm_i915_private *i915)
2032 struct i915_power_domains *power_domains = &i915->display.power.domains;
2034 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2035 power_domains->init_wakeref =
2036 intel_display_power_get(i915, POWER_DOMAIN_INIT);
2038 intel_power_domains_verify_state(i915);
2042 * intel_power_domains_suspend - suspend power domain state
2043 * @i915: i915 device instance
2044 * @s2idle: specifies whether we go to idle, or deeper sleep
2046 * This function prepares the hardware power domain state before entering
2049 * It must be called with power domains already disabled (after a call to
2050 * intel_power_domains_disable()) and paired with intel_power_domains_resume().
2052 void intel_power_domains_suspend(struct drm_i915_private *i915, bool s2idle)
2054 struct i915_power_domains *power_domains = &i915->display.power.domains;
2055 intel_wakeref_t wakeref __maybe_unused =
2056 fetch_and_zero(&power_domains->init_wakeref);
2058 intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2061 * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
2062 * support don't manually deinit the power domains. This also means the
2063 * DMC firmware will stay active, it will power down any HW
2064 * resources as required and also enable deeper system power states
2065 * that would be blocked if the firmware was inactive.
2067 if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC9) && s2idle &&
2068 intel_dmc_has_payload(i915)) {
2069 intel_display_power_flush_work(i915);
2070 intel_power_domains_verify_state(i915);
2075 * Even if power well support was disabled we still want to disable
2076 * power wells if power domains must be deinitialized for suspend.
2078 if (!i915->params.disable_power_well)
2079 intel_display_power_put(i915, POWER_DOMAIN_INIT,
2080 fetch_and_zero(&i915->display.power.domains.disable_wakeref));
2082 intel_display_power_flush_work(i915);
2083 intel_power_domains_verify_state(i915);
2085 if (DISPLAY_VER(i915) >= 11)
2086 icl_display_core_uninit(i915);
2087 else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
2088 bxt_display_core_uninit(i915);
2089 else if (DISPLAY_VER(i915) == 9)
2090 skl_display_core_uninit(i915);
2092 power_domains->display_core_suspended = true;
2096 * intel_power_domains_resume - resume power domain state
2097 * @i915: i915 device instance
2099 * This function resume the hardware power domain state during system resume.
2101 * It will return with power domain support disabled (to be enabled later by
2102 * intel_power_domains_enable()) and must be paired with
2103 * intel_power_domains_suspend().
2105 void intel_power_domains_resume(struct drm_i915_private *i915)
2107 struct i915_power_domains *power_domains = &i915->display.power.domains;
2109 if (power_domains->display_core_suspended) {
2110 intel_power_domains_init_hw(i915, true);
2111 power_domains->display_core_suspended = false;
2113 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2114 power_domains->init_wakeref =
2115 intel_display_power_get(i915, POWER_DOMAIN_INIT);
2118 intel_power_domains_verify_state(i915);
2121 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2123 static void intel_power_domains_dump_info(struct drm_i915_private *i915)
2125 struct i915_power_domains *power_domains = &i915->display.power.domains;
2126 struct i915_power_well *power_well;
2128 for_each_power_well(i915, power_well) {
2129 enum intel_display_power_domain domain;
2131 drm_dbg(&i915->drm, "%-25s %d\n",
2132 intel_power_well_name(power_well), intel_power_well_refcount(power_well));
2134 for_each_power_domain(domain, intel_power_well_domains(power_well))
2135 drm_dbg(&i915->drm, " %-23s %d\n",
2136 intel_display_power_domain_str(domain),
2137 power_domains->domain_use_count[domain]);
2142 * intel_power_domains_verify_state - verify the HW/SW state for all power wells
2143 * @i915: i915 device instance
2145 * Verify if the reference count of each power well matches its HW enabled
2146 * state and the total refcount of the domains it belongs to. This must be
2147 * called after modeset HW state sanitization, which is responsible for
2148 * acquiring reference counts for any power wells in use and disabling the
2149 * ones left on by BIOS but not required by any active output.
2151 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2153 struct i915_power_domains *power_domains = &i915->display.power.domains;
2154 struct i915_power_well *power_well;
2155 bool dump_domain_info;
2157 mutex_lock(&power_domains->lock);
2159 verify_async_put_domains_state(power_domains);
2161 dump_domain_info = false;
2162 for_each_power_well(i915, power_well) {
2163 enum intel_display_power_domain domain;
2167 enabled = intel_power_well_is_enabled(i915, power_well);
2168 if ((intel_power_well_refcount(power_well) ||
2169 intel_power_well_is_always_on(power_well)) !=
2172 "power well %s state mismatch (refcount %d/enabled %d)",
2173 intel_power_well_name(power_well),
2174 intel_power_well_refcount(power_well), enabled);
2177 for_each_power_domain(domain, intel_power_well_domains(power_well))
2178 domains_count += power_domains->domain_use_count[domain];
2180 if (intel_power_well_refcount(power_well) != domains_count) {
2182 "power well %s refcount/domain refcount mismatch "
2183 "(refcount %d/domains refcount %d)\n",
2184 intel_power_well_name(power_well),
2185 intel_power_well_refcount(power_well),
2187 dump_domain_info = true;
2191 if (dump_domain_info) {
2195 intel_power_domains_dump_info(i915);
2200 mutex_unlock(&power_domains->lock);
2205 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2211 void intel_display_power_suspend_late(struct drm_i915_private *i915)
2213 if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2215 bxt_enable_dc9(i915);
2216 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2217 hsw_enable_pc8(i915);
2220 /* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2221 if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2222 intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
2225 void intel_display_power_resume_early(struct drm_i915_private *i915)
2227 if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2229 gen9_sanitize_dc_state(i915);
2230 bxt_disable_dc9(i915);
2231 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2232 hsw_disable_pc8(i915);
2235 /* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2236 if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2237 intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0);
2240 void intel_display_power_suspend(struct drm_i915_private *i915)
2242 if (DISPLAY_VER(i915) >= 11) {
2243 icl_display_core_uninit(i915);
2244 bxt_enable_dc9(i915);
2245 } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2246 bxt_display_core_uninit(i915);
2247 bxt_enable_dc9(i915);
2248 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2249 hsw_enable_pc8(i915);
2253 void intel_display_power_resume(struct drm_i915_private *i915)
2255 struct i915_power_domains *power_domains = &i915->display.power.domains;
2257 if (DISPLAY_VER(i915) >= 11) {
2258 bxt_disable_dc9(i915);
2259 icl_display_core_init(i915, true);
2260 if (intel_dmc_has_payload(i915)) {
2261 if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
2262 skl_enable_dc6(i915);
2263 else if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
2264 gen9_enable_dc5(i915);
2266 } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2267 bxt_disable_dc9(i915);
2268 bxt_display_core_init(i915, true);
2269 if (intel_dmc_has_payload(i915) &&
2270 (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2271 gen9_enable_dc5(i915);
2272 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2273 hsw_disable_pc8(i915);
2277 void intel_display_power_debug(struct drm_i915_private *i915, struct seq_file *m)
2279 struct i915_power_domains *power_domains = &i915->display.power.domains;
2282 mutex_lock(&power_domains->lock);
2284 seq_printf(m, "%-25s %s\n", "Power well/domain", "Use count");
2285 for (i = 0; i < power_domains->power_well_count; i++) {
2286 struct i915_power_well *power_well;
2287 enum intel_display_power_domain power_domain;
2289 power_well = &power_domains->power_wells[i];
2290 seq_printf(m, "%-25s %d\n", intel_power_well_name(power_well),
2291 intel_power_well_refcount(power_well));
2293 for_each_power_domain(power_domain, intel_power_well_domains(power_well))
2294 seq_printf(m, " %-23s %d\n",
2295 intel_display_power_domain_str(power_domain),
2296 power_domains->domain_use_count[power_domain]);
2299 mutex_unlock(&power_domains->lock);
2302 struct intel_ddi_port_domains {
2303 enum port port_start;
2305 enum aux_ch aux_ch_start;
2306 enum aux_ch aux_ch_end;
2308 enum intel_display_power_domain ddi_lanes;
2309 enum intel_display_power_domain ddi_io;
2310 enum intel_display_power_domain aux_io;
2311 enum intel_display_power_domain aux_legacy_usbc;
2312 enum intel_display_power_domain aux_tbt;
2315 static const struct intel_ddi_port_domains
2316 i9xx_port_domains[] = {
2318 .port_start = PORT_A,
2320 .aux_ch_start = AUX_CH_A,
2321 .aux_ch_end = AUX_CH_F,
2323 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2324 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2325 .aux_io = POWER_DOMAIN_AUX_IO_A,
2326 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2327 .aux_tbt = POWER_DOMAIN_INVALID,
2331 static const struct intel_ddi_port_domains
2332 d11_port_domains[] = {
2334 .port_start = PORT_A,
2336 .aux_ch_start = AUX_CH_A,
2337 .aux_ch_end = AUX_CH_B,
2339 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2340 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2341 .aux_io = POWER_DOMAIN_AUX_IO_A,
2342 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2343 .aux_tbt = POWER_DOMAIN_INVALID,
2345 .port_start = PORT_C,
2347 .aux_ch_start = AUX_CH_C,
2348 .aux_ch_end = AUX_CH_F,
2350 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_C,
2351 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_C,
2352 .aux_io = POWER_DOMAIN_AUX_IO_C,
2353 .aux_legacy_usbc = POWER_DOMAIN_AUX_C,
2354 .aux_tbt = POWER_DOMAIN_AUX_TBT1,
2358 static const struct intel_ddi_port_domains
2359 d12_port_domains[] = {
2361 .port_start = PORT_A,
2363 .aux_ch_start = AUX_CH_A,
2364 .aux_ch_end = AUX_CH_C,
2366 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2367 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2368 .aux_io = POWER_DOMAIN_AUX_IO_A,
2369 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2370 .aux_tbt = POWER_DOMAIN_INVALID,
2372 .port_start = PORT_TC1,
2373 .port_end = PORT_TC6,
2374 .aux_ch_start = AUX_CH_USBC1,
2375 .aux_ch_end = AUX_CH_USBC6,
2377 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2378 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2379 .aux_io = POWER_DOMAIN_INVALID,
2380 .aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2381 .aux_tbt = POWER_DOMAIN_AUX_TBT1,
2385 static const struct intel_ddi_port_domains
2386 d13_port_domains[] = {
2388 .port_start = PORT_A,
2390 .aux_ch_start = AUX_CH_A,
2391 .aux_ch_end = AUX_CH_C,
2393 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2394 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2395 .aux_io = POWER_DOMAIN_AUX_IO_A,
2396 .aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2397 .aux_tbt = POWER_DOMAIN_INVALID,
2399 .port_start = PORT_TC1,
2400 .port_end = PORT_TC4,
2401 .aux_ch_start = AUX_CH_USBC1,
2402 .aux_ch_end = AUX_CH_USBC4,
2404 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2405 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2406 .aux_io = POWER_DOMAIN_INVALID,
2407 .aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2408 .aux_tbt = POWER_DOMAIN_AUX_TBT1,
2410 .port_start = PORT_D_XELPD,
2411 .port_end = PORT_E_XELPD,
2412 .aux_ch_start = AUX_CH_D_XELPD,
2413 .aux_ch_end = AUX_CH_E_XELPD,
2415 .ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_D,
2416 .ddi_io = POWER_DOMAIN_PORT_DDI_IO_D,
2417 .aux_io = POWER_DOMAIN_AUX_IO_D,
2418 .aux_legacy_usbc = POWER_DOMAIN_AUX_D,
2419 .aux_tbt = POWER_DOMAIN_INVALID,
2424 intel_port_domains_for_platform(struct drm_i915_private *i915,
2425 const struct intel_ddi_port_domains **domains,
2428 if (DISPLAY_VER(i915) >= 13) {
2429 *domains = d13_port_domains;
2430 *domains_size = ARRAY_SIZE(d13_port_domains);
2431 } else if (DISPLAY_VER(i915) >= 12) {
2432 *domains = d12_port_domains;
2433 *domains_size = ARRAY_SIZE(d12_port_domains);
2434 } else if (DISPLAY_VER(i915) >= 11) {
2435 *domains = d11_port_domains;
2436 *domains_size = ARRAY_SIZE(d11_port_domains);
2438 *domains = i9xx_port_domains;
2439 *domains_size = ARRAY_SIZE(i9xx_port_domains);
2443 static const struct intel_ddi_port_domains *
2444 intel_port_domains_for_port(struct drm_i915_private *i915, enum port port)
2446 const struct intel_ddi_port_domains *domains;
2450 intel_port_domains_for_platform(i915, &domains, &domains_size);
2451 for (i = 0; i < domains_size; i++)
2452 if (port >= domains[i].port_start && port <= domains[i].port_end)
2458 enum intel_display_power_domain
2459 intel_display_power_ddi_io_domain(struct drm_i915_private *i915, enum port port)
2461 const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2463 if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_io == POWER_DOMAIN_INVALID))
2464 return POWER_DOMAIN_PORT_DDI_IO_A;
2466 return domains->ddi_io + (int)(port - domains->port_start);
2469 enum intel_display_power_domain
2470 intel_display_power_ddi_lanes_domain(struct drm_i915_private *i915, enum port port)
2472 const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2474 if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_lanes == POWER_DOMAIN_INVALID))
2475 return POWER_DOMAIN_PORT_DDI_LANES_A;
2477 return domains->ddi_lanes + (int)(port - domains->port_start);
2480 static const struct intel_ddi_port_domains *
2481 intel_port_domains_for_aux_ch(struct drm_i915_private *i915, enum aux_ch aux_ch)
2483 const struct intel_ddi_port_domains *domains;
2487 intel_port_domains_for_platform(i915, &domains, &domains_size);
2488 for (i = 0; i < domains_size; i++)
2489 if (aux_ch >= domains[i].aux_ch_start && aux_ch <= domains[i].aux_ch_end)
2495 enum intel_display_power_domain
2496 intel_display_power_aux_io_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2498 const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2500 if (drm_WARN_ON(&i915->drm, !domains || domains->aux_io == POWER_DOMAIN_INVALID))
2501 return POWER_DOMAIN_AUX_IO_A;
2503 return domains->aux_io + (int)(aux_ch - domains->aux_ch_start);
2506 enum intel_display_power_domain
2507 intel_display_power_legacy_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2509 const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2511 if (drm_WARN_ON(&i915->drm, !domains || domains->aux_legacy_usbc == POWER_DOMAIN_INVALID))
2512 return POWER_DOMAIN_AUX_A;
2514 return domains->aux_legacy_usbc + (int)(aux_ch - domains->aux_ch_start);
2517 enum intel_display_power_domain
2518 intel_display_power_tbt_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2520 const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2522 if (drm_WARN_ON(&i915->drm, !domains || domains->aux_tbt == POWER_DOMAIN_INVALID))
2523 return POWER_DOMAIN_AUX_TBT1;
2525 return domains->aux_tbt + (int)(aux_ch - domains->aux_ch_start);