2 * drm_irq.c IRQ and vblank support
4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 * \author Gareth Hughes <gareth@valinux.com>
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
27 #include <linux/export.h>
28 #include <linux/moduleparam.h>
30 #include <drm/drm_crtc.h>
31 #include <drm/drm_drv.h>
32 #include <drm/drm_framebuffer.h>
33 #include <drm/drm_print.h>
34 #include <drm/drm_vblank.h>
36 #include "drm_internal.h"
37 #include "drm_trace.h"
40 * DOC: vblank handling
42 * Vertical blanking plays a major role in graphics rendering. To achieve
43 * tear-free display, users must synchronize page flips and/or rendering to
44 * vertical blanking. The DRM API offers ioctls to perform page flips
45 * synchronized to vertical blanking and wait for vertical blanking.
47 * The DRM core handles most of the vertical blanking management logic, which
48 * involves filtering out spurious interrupts, keeping race-free blanking
49 * counters, coping with counter wrap-around and resets and keeping use counts.
50 * It relies on the driver to generate vertical blanking interrupts and
51 * optionally provide a hardware vertical blanking counter.
53 * Drivers must initialize the vertical blanking handling core with a call to
54 * drm_vblank_init(). Minimally, a driver needs to implement
55 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
56 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
59 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
60 * themselves (for instance to handle page flipping operations). The DRM core
61 * maintains a vertical blanking use count to ensure that the interrupts are not
62 * disabled while a user still needs them. To increment the use count, drivers
63 * call drm_crtc_vblank_get() and release the vblank reference again with
64 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
65 * guaranteed to be enabled.
67 * On many hardware disabling the vblank interrupt cannot be done in a race-free
68 * manner, see &drm_driver.vblank_disable_immediate and
69 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
70 * vblanks after a timer has expired, which can be configured through the
71 * ``vblankoffdelay`` module parameter.
74 /* Retry timestamp calculation up to 3 times to satisfy
75 * drm_timestamp_precision before giving up.
77 #define DRM_TIMESTAMP_MAXRETRIES 3
79 /* Threshold in nanoseconds for detection of redundant
80 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
82 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
85 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
86 ktime_t *tvblank, bool in_vblank_irq);
88 static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
90 static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
92 module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
93 module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
94 MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
95 MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
97 static void store_vblank(struct drm_device *dev, unsigned int pipe,
99 ktime_t t_vblank, u32 last)
101 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
103 assert_spin_locked(&dev->vblank_time_lock);
107 write_seqlock(&vblank->seqlock);
108 vblank->time = t_vblank;
109 vblank->count += vblank_count_inc;
110 write_sequnlock(&vblank->seqlock);
113 static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
115 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
117 return vblank->max_vblank_count ?: dev->max_vblank_count;
121 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
122 * if there is no useable hardware frame counter available.
124 static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
126 WARN_ON_ONCE(drm_max_vblank_count(dev, pipe) != 0);
130 static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
132 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
133 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
138 if (crtc->funcs->get_vblank_counter)
139 return crtc->funcs->get_vblank_counter(crtc);
142 if (dev->driver->get_vblank_counter)
143 return dev->driver->get_vblank_counter(dev, pipe);
145 return drm_vblank_no_hw_counter(dev, pipe);
149 * Reset the stored timestamp for the current vblank count to correspond
150 * to the last vblank occurred.
152 * Only to be called from drm_crtc_vblank_on().
154 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
155 * device vblank fields.
157 static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
162 int count = DRM_TIMESTAMP_MAXRETRIES;
164 spin_lock(&dev->vblank_time_lock);
167 * sample the current counter to avoid random jumps
168 * when drm_vblank_enable() applies the diff
171 cur_vblank = __get_vblank_counter(dev, pipe);
172 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
173 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
176 * Only reinitialize corresponding vblank timestamp if high-precision query
177 * available and didn't fail. Otherwise reinitialize delayed at next vblank
178 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
184 * +1 to make sure user will never see the same
185 * vblank counter value before and after a modeset
187 store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
189 spin_unlock(&dev->vblank_time_lock);
193 * Call back into the driver to update the appropriate vblank counter
194 * (specified by @pipe). Deal with wraparound, if it occurred, and
195 * update the last read value so we can deal with wraparound on the next
198 * Only necessary when going from off->on, to account for frames we
199 * didn't get an interrupt for.
201 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
202 * device vblank fields.
204 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
207 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
208 u32 cur_vblank, diff;
211 int count = DRM_TIMESTAMP_MAXRETRIES;
212 int framedur_ns = vblank->framedur_ns;
213 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
216 * Interrupts were disabled prior to this call, so deal with counter
218 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
219 * here if the register is small or we had vblank interrupts off for
222 * We repeat the hardware vblank counter & timestamp query until
223 * we get consistent results. This to prevent races between gpu
224 * updating its hardware counter while we are retrieving the
225 * corresponding vblank timestamp.
228 cur_vblank = __get_vblank_counter(dev, pipe);
229 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
230 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
232 if (max_vblank_count) {
233 /* trust the hw counter when it's around */
234 diff = (cur_vblank - vblank->last) & max_vblank_count;
235 } else if (rc && framedur_ns) {
236 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
239 * Figure out how many vblanks we've missed based
240 * on the difference in the timestamps and the
241 * frame/field duration.
244 DRM_DEBUG_VBL("crtc %u: Calculating number of vblanks."
245 " diff_ns = %lld, framedur_ns = %d)\n",
246 pipe, (long long) diff_ns, framedur_ns);
248 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
250 if (diff == 0 && in_vblank_irq)
251 DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored\n",
254 /* some kind of default for drivers w/o accurate vbl timestamping */
255 diff = in_vblank_irq ? 1 : 0;
259 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
260 * interval? If so then vblank irqs keep running and it will likely
261 * happen that the hardware vblank counter is not trustworthy as it
262 * might reset at some point in that interval and vblank timestamps
263 * are not trustworthy either in that interval. Iow. this can result
264 * in a bogus diff >> 1 which must be avoided as it would cause
265 * random large forward jumps of the software vblank counter.
267 if (diff > 1 && (vblank->inmodeset & 0x2)) {
268 DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
269 " due to pre-modeset.\n", pipe, diff);
273 DRM_DEBUG_VBL("updating vblank count on crtc %u:"
274 " current=%llu, diff=%u, hw=%u hw_last=%u\n",
275 pipe, vblank->count, diff, cur_vblank, vblank->last);
278 WARN_ON_ONCE(cur_vblank != vblank->last);
283 * Only reinitialize corresponding vblank timestamp if high-precision query
284 * available and didn't fail, or we were called from the vblank interrupt.
285 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
286 * for now, to mark the vblanktimestamp as invalid.
288 if (!rc && !in_vblank_irq)
291 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
294 static u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
296 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
298 if (WARN_ON(pipe >= dev->num_crtcs))
301 return vblank->count;
305 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
306 * @crtc: which counter to retrieve
308 * This function is similar to drm_crtc_vblank_count() but this function
309 * interpolates to handle a race with vblank interrupts using the high precision
310 * timestamping support.
312 * This is mostly useful for hardware that can obtain the scanout position, but
313 * doesn't have a hardware frame counter.
315 u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
317 struct drm_device *dev = crtc->dev;
318 unsigned int pipe = drm_crtc_index(crtc);
322 WARN_ONCE(drm_debug & DRM_UT_VBL && !dev->driver->get_vblank_timestamp,
323 "This function requires support for accurate vblank timestamps.");
325 spin_lock_irqsave(&dev->vblank_time_lock, flags);
327 drm_update_vblank_count(dev, pipe, false);
328 vblank = drm_vblank_count(dev, pipe);
330 spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
334 EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
336 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
338 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
339 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
344 if (crtc->funcs->disable_vblank) {
345 crtc->funcs->disable_vblank(crtc);
350 dev->driver->disable_vblank(dev, pipe);
354 * Disable vblank irq's on crtc, make sure that last vblank count
355 * of hardware and corresponding consistent software vblank counter
356 * are preserved, even if there are any spurious vblank irq's after
359 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
361 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
362 unsigned long irqflags;
364 assert_spin_locked(&dev->vbl_lock);
366 /* Prevent vblank irq processing while disabling vblank irqs,
367 * so no updates of timestamps or count can happen after we've
368 * disabled. Needed to prevent races in case of delayed irq's.
370 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
373 * Update vblank count and disable vblank interrupts only if the
374 * interrupts were enabled. This avoids calling the ->disable_vblank()
375 * operation in atomic context with the hardware potentially runtime
378 if (!vblank->enabled)
382 * Update the count and timestamp to maintain the
383 * appearance that the counter has been ticking all along until
384 * this time. This makes the count account for the entire time
385 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
387 drm_update_vblank_count(dev, pipe, false);
388 __disable_vblank(dev, pipe);
389 vblank->enabled = false;
392 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
395 static void vblank_disable_fn(struct timer_list *t)
397 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
398 struct drm_device *dev = vblank->dev;
399 unsigned int pipe = vblank->pipe;
400 unsigned long irqflags;
402 spin_lock_irqsave(&dev->vbl_lock, irqflags);
403 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
404 DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
405 drm_vblank_disable_and_save(dev, pipe);
407 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
410 void drm_vblank_cleanup(struct drm_device *dev)
414 /* Bail if the driver didn't call drm_vblank_init() */
415 if (dev->num_crtcs == 0)
418 for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
419 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
421 WARN_ON(READ_ONCE(vblank->enabled) &&
422 drm_core_check_feature(dev, DRIVER_MODESET));
424 del_timer_sync(&vblank->disable_timer);
433 * drm_vblank_init - initialize vblank support
435 * @num_crtcs: number of CRTCs supported by @dev
437 * This function initializes vblank support for @num_crtcs display pipelines.
438 * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for
439 * drivers with a &drm_driver.release callback.
442 * Zero on success or a negative error code on failure.
444 int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
449 spin_lock_init(&dev->vbl_lock);
450 spin_lock_init(&dev->vblank_time_lock);
452 dev->num_crtcs = num_crtcs;
454 dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
458 for (i = 0; i < num_crtcs; i++) {
459 struct drm_vblank_crtc *vblank = &dev->vblank[i];
463 init_waitqueue_head(&vblank->queue);
464 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
465 seqlock_init(&vblank->seqlock);
468 DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
470 /* Driver specific high-precision vblank timestamping supported? */
471 if (dev->driver->get_vblank_timestamp)
472 DRM_INFO("Driver supports precise vblank timestamp query.\n");
474 DRM_INFO("No driver support for vblank timestamp query.\n");
476 /* Must have precise timestamping for reliable vblank instant disable */
477 if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
478 dev->vblank_disable_immediate = false;
479 DRM_INFO("Setting vblank_disable_immediate to false because "
480 "get_vblank_timestamp == NULL\n");
489 EXPORT_SYMBOL(drm_vblank_init);
492 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
493 * @crtc: which CRTC's vblank waitqueue to retrieve
495 * This function returns a pointer to the vblank waitqueue for the CRTC.
496 * Drivers can use this to implement vblank waits using wait_event() and related
499 wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
501 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
503 EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
507 * drm_calc_timestamping_constants - calculate vblank timestamp constants
508 * @crtc: drm_crtc whose timestamp constants should be updated.
509 * @mode: display mode containing the scanout timings
511 * Calculate and store various constants which are later needed by vblank and
512 * swap-completion timestamping, e.g, by
513 * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true
514 * scanout timing, so they take things like panel scaling or other adjustments
517 void drm_calc_timestamping_constants(struct drm_crtc *crtc,
518 const struct drm_display_mode *mode)
520 struct drm_device *dev = crtc->dev;
521 unsigned int pipe = drm_crtc_index(crtc);
522 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
523 int linedur_ns = 0, framedur_ns = 0;
524 int dotclock = mode->crtc_clock;
529 if (WARN_ON(pipe >= dev->num_crtcs))
532 /* Valid dotclock? */
534 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
537 * Convert scanline length in pixels and video
538 * dot clock to line duration and frame duration
541 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
542 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
545 * Fields of interlaced scanout modes are only half a frame duration.
547 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
550 DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
553 vblank->linedur_ns = linedur_ns;
554 vblank->framedur_ns = framedur_ns;
555 vblank->hwmode = *mode;
557 DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
558 crtc->base.id, mode->crtc_htotal,
559 mode->crtc_vtotal, mode->crtc_vdisplay);
560 DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
561 crtc->base.id, dotclock, framedur_ns, linedur_ns);
563 EXPORT_SYMBOL(drm_calc_timestamping_constants);
566 * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
568 * @pipe: index of CRTC whose vblank timestamp to retrieve
569 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
570 * On return contains true maximum error of timestamp
571 * @vblank_time: Pointer to time which should receive the timestamp
573 * True when called from drm_crtc_handle_vblank(). Some drivers
574 * need to apply some workarounds for gpu-specific vblank irq quirks
577 * Implements calculation of exact vblank timestamps from given drm_display_mode
578 * timings and current video scanout position of a CRTC. This can be directly
579 * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver
580 * if &drm_driver.get_scanout_position is implemented.
582 * The current implementation only handles standard video modes. For double scan
583 * and interlaced modes the driver is supposed to adjust the hardware mode
584 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
585 * match the scanout position reported.
587 * Note that atomic drivers must call drm_calc_timestamping_constants() before
588 * enabling a CRTC. The atomic helpers already take care of that in
589 * drm_atomic_helper_update_legacy_modeset_state().
593 * Returns true on success, and false on failure, i.e. when no accurate
594 * timestamp could be acquired.
596 bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
599 ktime_t *vblank_time,
602 struct timespec64 ts_etime, ts_vblank_time;
603 ktime_t stime, etime;
605 struct drm_crtc *crtc;
606 const struct drm_display_mode *mode;
607 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
609 int delta_ns, duration_ns;
611 if (!drm_core_check_feature(dev, DRIVER_MODESET))
614 crtc = drm_crtc_from_index(dev, pipe);
616 if (pipe >= dev->num_crtcs || !crtc) {
617 DRM_ERROR("Invalid crtc %u\n", pipe);
621 /* Scanout position query not supported? Should not happen. */
622 if (!dev->driver->get_scanout_position) {
623 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
627 if (drm_drv_uses_atomic_modeset(dev))
628 mode = &vblank->hwmode;
630 mode = &crtc->hwmode;
632 /* If mode timing undefined, just return as no-op:
633 * Happens during initial modesetting of a crtc.
635 if (mode->crtc_clock == 0) {
636 DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
637 WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev));
642 /* Get current scanout position with system timestamp.
643 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
644 * if single query takes longer than max_error nanoseconds.
646 * This guarantees a tight bound on maximum error if
647 * code gets preempted or delayed for some reason.
649 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
651 * Get vertical and horizontal scanout position vpos, hpos,
652 * and bounding timestamps stime, etime, pre/post query.
654 vbl_status = dev->driver->get_scanout_position(dev, pipe,
660 /* Return as no-op if scanout query unsupported or failed. */
662 DRM_DEBUG("crtc %u : scanoutpos query failed.\n",
667 /* Compute uncertainty in timestamp of scanout position query. */
668 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
670 /* Accept result with < max_error nsecs timing uncertainty. */
671 if (duration_ns <= *max_error)
675 /* Noisy system timing? */
676 if (i == DRM_TIMESTAMP_MAXRETRIES) {
677 DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
678 pipe, duration_ns/1000, *max_error/1000, i);
681 /* Return upper bound of timestamp precision error. */
682 *max_error = duration_ns;
684 /* Convert scanout position into elapsed time at raw_time query
685 * since start of scanout at first display scanline. delta_ns
686 * can be negative if start of scanout hasn't happened yet.
688 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
691 /* Subtract time delta from raw timestamp to get final
692 * vblank_time timestamp for end of vblank.
694 *vblank_time = ktime_sub_ns(etime, delta_ns);
696 if ((drm_debug & DRM_UT_VBL) == 0)
699 ts_etime = ktime_to_timespec64(etime);
700 ts_vblank_time = ktime_to_timespec64(*vblank_time);
702 DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
704 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
705 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
706 duration_ns / 1000, i);
710 EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
713 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
716 * @pipe: index of CRTC whose vblank timestamp to retrieve
717 * @tvblank: Pointer to target time which should receive the timestamp
719 * True when called from drm_crtc_handle_vblank(). Some drivers
720 * need to apply some workarounds for gpu-specific vblank irq quirks
723 * Fetches the system timestamp corresponding to the time of the most recent
724 * vblank interval on specified CRTC. May call into kms-driver to
725 * compute the timestamp with a high-precision GPU specific method.
727 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
728 * call, i.e., it isn't very precisely locked to the true vblank.
731 * True if timestamp is considered to be very precise, false otherwise.
734 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
735 ktime_t *tvblank, bool in_vblank_irq)
739 /* Define requested maximum error on timestamps (nanoseconds). */
740 int max_error = (int) drm_timestamp_precision * 1000;
742 /* Query driver if possible and precision timestamping enabled. */
743 if (dev->driver->get_vblank_timestamp && (max_error > 0))
744 ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
745 tvblank, in_vblank_irq);
747 /* GPU high precision timestamp query unsupported or failed.
748 * Return current monotonic/gettimeofday timestamp as best estimate.
751 *tvblank = ktime_get();
757 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
758 * @crtc: which counter to retrieve
760 * Fetches the "cooked" vblank count value that represents the number of
761 * vblank events since the system was booted, including lost events due to
762 * modesetting activity. Note that this timer isn't correct against a racing
763 * vblank interrupt (since it only reports the software vblank counter), see
764 * drm_crtc_accurate_vblank_count() for such use-cases.
767 * The software vblank counter.
769 u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
771 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
773 EXPORT_SYMBOL(drm_crtc_vblank_count);
776 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
777 * system timestamp corresponding to that vblank counter value.
779 * @pipe: index of CRTC whose counter to retrieve
780 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
782 * Fetches the "cooked" vblank count value that represents the number of
783 * vblank events since the system was booted, including lost events due to
784 * modesetting activity. Returns corresponding system timestamp of the time
785 * of the vblank interval that corresponds to the current vblank counter value.
787 * This is the legacy version of drm_crtc_vblank_count_and_time().
789 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
792 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
796 if (WARN_ON(pipe >= dev->num_crtcs)) {
802 seq = read_seqbegin(&vblank->seqlock);
803 vblank_count = vblank->count;
804 *vblanktime = vblank->time;
805 } while (read_seqretry(&vblank->seqlock, seq));
811 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
812 * and the system timestamp corresponding to that vblank counter value
813 * @crtc: which counter to retrieve
814 * @vblanktime: Pointer to time to receive the vblank timestamp.
816 * Fetches the "cooked" vblank count value that represents the number of
817 * vblank events since the system was booted, including lost events due to
818 * modesetting activity. Returns corresponding system timestamp of the time
819 * of the vblank interval that corresponds to the current vblank counter value.
821 u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
824 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
827 EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
829 static void send_vblank_event(struct drm_device *dev,
830 struct drm_pending_vblank_event *e,
831 u64 seq, ktime_t now)
833 struct timespec64 tv;
835 switch (e->event.base.type) {
836 case DRM_EVENT_VBLANK:
837 case DRM_EVENT_FLIP_COMPLETE:
838 tv = ktime_to_timespec64(now);
839 e->event.vbl.sequence = seq;
841 * e->event is a user space structure, with hardcoded unsigned
842 * 32-bit seconds/microseconds. This is safe as we always use
843 * monotonic timestamps since linux-4.15
845 e->event.vbl.tv_sec = tv.tv_sec;
846 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
848 case DRM_EVENT_CRTC_SEQUENCE:
850 e->event.seq.sequence = seq;
851 e->event.seq.time_ns = ktime_to_ns(now);
854 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
855 drm_send_event_locked(dev, &e->base);
859 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
860 * @crtc: the source CRTC of the vblank event
861 * @e: the event to send
863 * A lot of drivers need to generate vblank events for the very next vblank
864 * interrupt. For example when the page flip interrupt happens when the page
865 * flip gets armed, but not when it actually executes within the next vblank
866 * period. This helper function implements exactly the required vblank arming
869 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
870 * atomic commit must ensure that the next vblank happens at exactly the same
871 * time as the atomic commit is committed to the hardware. This function itself
872 * does **not** protect against the next vblank interrupt racing with either this
873 * function call or the atomic commit operation. A possible sequence could be:
875 * 1. Driver commits new hardware state into vblank-synchronized registers.
876 * 2. A vblank happens, committing the hardware state. Also the corresponding
877 * vblank interrupt is fired off and fully processed by the interrupt
879 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
880 * 4. The event is only send out for the next vblank, which is wrong.
882 * An equivalent race can happen when the driver calls
883 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
885 * The only way to make this work safely is to prevent the vblank from firing
886 * (and the hardware from committing anything else) until the entire atomic
887 * commit sequence has run to completion. If the hardware does not have such a
888 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
889 * Instead drivers need to manually send out the event from their interrupt
890 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
891 * possible race with the hardware committing the atomic update.
893 * Caller must hold a vblank reference for the event @e acquired by a
894 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
896 void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
897 struct drm_pending_vblank_event *e)
899 struct drm_device *dev = crtc->dev;
900 unsigned int pipe = drm_crtc_index(crtc);
902 assert_spin_locked(&dev->event_lock);
905 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
906 list_add_tail(&e->base.link, &dev->vblank_event_list);
908 EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
911 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
912 * @crtc: the source CRTC of the vblank event
913 * @e: the event to send
915 * Updates sequence # and timestamp on event for the most recently processed
916 * vblank, and sends it to userspace. Caller must hold event lock.
918 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
919 * situation, especially to send out events for atomic commit operations.
921 void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
922 struct drm_pending_vblank_event *e)
924 struct drm_device *dev = crtc->dev;
926 unsigned int pipe = drm_crtc_index(crtc);
929 if (dev->num_crtcs > 0) {
930 seq = drm_vblank_count_and_time(dev, pipe, &now);
937 send_vblank_event(dev, e, seq, now);
939 EXPORT_SYMBOL(drm_crtc_send_vblank_event);
941 static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
943 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
944 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
949 if (crtc->funcs->enable_vblank)
950 return crtc->funcs->enable_vblank(crtc);
953 return dev->driver->enable_vblank(dev, pipe);
956 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
958 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
961 assert_spin_locked(&dev->vbl_lock);
963 spin_lock(&dev->vblank_time_lock);
965 if (!vblank->enabled) {
967 * Enable vblank irqs under vblank_time_lock protection.
968 * All vblank count & timestamp updates are held off
969 * until we are done reinitializing master counter and
970 * timestamps. Filtercode in drm_handle_vblank() will
971 * prevent double-accounting of same vblank interval.
973 ret = __enable_vblank(dev, pipe);
974 DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
976 atomic_dec(&vblank->refcount);
978 drm_update_vblank_count(dev, pipe, 0);
979 /* drm_update_vblank_count() includes a wmb so we just
980 * need to ensure that the compiler emits the write
981 * to mark the vblank as enabled after the call
982 * to drm_update_vblank_count().
984 WRITE_ONCE(vblank->enabled, true);
988 spin_unlock(&dev->vblank_time_lock);
993 static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
995 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
996 unsigned long irqflags;
1002 if (WARN_ON(pipe >= dev->num_crtcs))
1005 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1006 /* Going from 0->1 means we have to enable interrupts again */
1007 if (atomic_add_return(1, &vblank->refcount) == 1) {
1008 ret = drm_vblank_enable(dev, pipe);
1010 if (!vblank->enabled) {
1011 atomic_dec(&vblank->refcount);
1015 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1021 * drm_crtc_vblank_get - get a reference count on vblank events
1022 * @crtc: which CRTC to own
1024 * Acquire a reference count on vblank events to avoid having them disabled
1028 * Zero on success or a negative error code on failure.
1030 int drm_crtc_vblank_get(struct drm_crtc *crtc)
1032 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1034 EXPORT_SYMBOL(drm_crtc_vblank_get);
1036 static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1038 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1040 if (WARN_ON(pipe >= dev->num_crtcs))
1043 if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1046 /* Last user schedules interrupt disable */
1047 if (atomic_dec_and_test(&vblank->refcount)) {
1048 if (drm_vblank_offdelay == 0)
1050 else if (drm_vblank_offdelay < 0)
1051 vblank_disable_fn(&vblank->disable_timer);
1052 else if (!dev->vblank_disable_immediate)
1053 mod_timer(&vblank->disable_timer,
1054 jiffies + ((drm_vblank_offdelay * HZ)/1000));
1059 * drm_crtc_vblank_put - give up ownership of vblank events
1060 * @crtc: which counter to give up
1062 * Release ownership of a given vblank counter, turning off interrupts
1063 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1065 void drm_crtc_vblank_put(struct drm_crtc *crtc)
1067 drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1069 EXPORT_SYMBOL(drm_crtc_vblank_put);
1072 * drm_wait_one_vblank - wait for one vblank
1076 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1077 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1078 * due to lack of driver support or because the crtc is off.
1080 * This is the legacy version of drm_crtc_wait_one_vblank().
1082 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1084 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1088 if (WARN_ON(pipe >= dev->num_crtcs))
1091 ret = drm_vblank_get(dev, pipe);
1092 if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1095 last = drm_vblank_count(dev, pipe);
1097 ret = wait_event_timeout(vblank->queue,
1098 last != drm_vblank_count(dev, pipe),
1099 msecs_to_jiffies(100));
1101 WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1103 drm_vblank_put(dev, pipe);
1105 EXPORT_SYMBOL(drm_wait_one_vblank);
1108 * drm_crtc_wait_one_vblank - wait for one vblank
1111 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1112 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1113 * due to lack of driver support or because the crtc is off.
1115 void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1117 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1119 EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1122 * drm_crtc_vblank_off - disable vblank events on a CRTC
1123 * @crtc: CRTC in question
1125 * Drivers can use this function to shut down the vblank interrupt handling when
1126 * disabling a crtc. This function ensures that the latest vblank frame count is
1127 * stored so that drm_vblank_on can restore it again.
1129 * Drivers must use this function when the hardware vblank counter can get
1130 * reset, e.g. when suspending or disabling the @crtc in general.
1132 void drm_crtc_vblank_off(struct drm_crtc *crtc)
1134 struct drm_device *dev = crtc->dev;
1135 unsigned int pipe = drm_crtc_index(crtc);
1136 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1137 struct drm_pending_vblank_event *e, *t;
1140 unsigned long irqflags;
1143 if (WARN_ON(pipe >= dev->num_crtcs))
1146 spin_lock_irqsave(&dev->event_lock, irqflags);
1148 spin_lock(&dev->vbl_lock);
1149 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1150 pipe, vblank->enabled, vblank->inmodeset);
1152 /* Avoid redundant vblank disables without previous
1153 * drm_crtc_vblank_on(). */
1154 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1155 drm_vblank_disable_and_save(dev, pipe);
1157 wake_up(&vblank->queue);
1160 * Prevent subsequent drm_vblank_get() from re-enabling
1161 * the vblank interrupt by bumping the refcount.
1163 if (!vblank->inmodeset) {
1164 atomic_inc(&vblank->refcount);
1165 vblank->inmodeset = 1;
1167 spin_unlock(&dev->vbl_lock);
1169 /* Send any queued vblank events, lest the natives grow disquiet */
1170 seq = drm_vblank_count_and_time(dev, pipe, &now);
1172 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1173 if (e->pipe != pipe)
1175 DRM_DEBUG("Sending premature vblank event on disable: "
1176 "wanted %llu, current %llu\n",
1178 list_del(&e->base.link);
1179 drm_vblank_put(dev, pipe);
1180 send_vblank_event(dev, e, seq, now);
1182 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1184 /* Will be reset by the modeset helpers when re-enabling the crtc by
1185 * calling drm_calc_timestamping_constants(). */
1186 vblank->hwmode.crtc_clock = 0;
1188 EXPORT_SYMBOL(drm_crtc_vblank_off);
1191 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1192 * @crtc: CRTC in question
1194 * Drivers can use this function to reset the vblank state to off at load time.
1195 * Drivers should use this together with the drm_crtc_vblank_off() and
1196 * drm_crtc_vblank_on() functions. The difference compared to
1197 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1198 * and hence doesn't need to call any driver hooks.
1200 * This is useful for recovering driver state e.g. on driver load, or on resume.
1202 void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1204 struct drm_device *dev = crtc->dev;
1205 unsigned long irqflags;
1206 unsigned int pipe = drm_crtc_index(crtc);
1207 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1209 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1211 * Prevent subsequent drm_vblank_get() from enabling the vblank
1212 * interrupt by bumping the refcount.
1214 if (!vblank->inmodeset) {
1215 atomic_inc(&vblank->refcount);
1216 vblank->inmodeset = 1;
1218 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1220 WARN_ON(!list_empty(&dev->vblank_event_list));
1222 EXPORT_SYMBOL(drm_crtc_vblank_reset);
1225 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1226 * @crtc: CRTC in question
1227 * @max_vblank_count: max hardware vblank counter value
1229 * Update the maximum hardware vblank counter value for @crtc
1230 * at runtime. Useful for hardware where the operation of the
1231 * hardware vblank counter depends on the currently active
1232 * display configuration.
1234 * For example, if the hardware vblank counter does not work
1235 * when a specific connector is active the maximum can be set
1236 * to zero. And when that specific connector isn't active the
1237 * maximum can again be set to the appropriate non-zero value.
1239 * If used, must be called before drm_vblank_on().
1241 void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1242 u32 max_vblank_count)
1244 struct drm_device *dev = crtc->dev;
1245 unsigned int pipe = drm_crtc_index(crtc);
1246 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1248 WARN_ON(dev->max_vblank_count);
1249 WARN_ON(!READ_ONCE(vblank->inmodeset));
1251 vblank->max_vblank_count = max_vblank_count;
1253 EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1256 * drm_crtc_vblank_on - enable vblank events on a CRTC
1257 * @crtc: CRTC in question
1259 * This functions restores the vblank interrupt state captured with
1260 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1261 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1262 * unbalanced and so can also be unconditionally called in driver load code to
1263 * reflect the current hardware state of the crtc.
1265 void drm_crtc_vblank_on(struct drm_crtc *crtc)
1267 struct drm_device *dev = crtc->dev;
1268 unsigned int pipe = drm_crtc_index(crtc);
1269 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1270 unsigned long irqflags;
1272 if (WARN_ON(pipe >= dev->num_crtcs))
1275 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1276 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1277 pipe, vblank->enabled, vblank->inmodeset);
1279 /* Drop our private "prevent drm_vblank_get" refcount */
1280 if (vblank->inmodeset) {
1281 atomic_dec(&vblank->refcount);
1282 vblank->inmodeset = 0;
1285 drm_reset_vblank_timestamp(dev, pipe);
1288 * re-enable interrupts if there are users left, or the
1289 * user wishes vblank interrupts to be enabled all the time.
1291 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1292 WARN_ON(drm_vblank_enable(dev, pipe));
1293 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1295 EXPORT_SYMBOL(drm_crtc_vblank_on);
1298 * drm_vblank_restore - estimate missed vblanks and update vblank count.
1302 * Power manamement features can cause frame counter resets between vblank
1303 * disable and enable. Drivers can use this function in their
1304 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1305 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1308 * This function is the legacy version of drm_crtc_vblank_restore().
1310 void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1313 struct drm_vblank_crtc *vblank;
1316 u32 cur_vblank, diff = 1;
1317 int count = DRM_TIMESTAMP_MAXRETRIES;
1319 if (WARN_ON(pipe >= dev->num_crtcs))
1322 assert_spin_locked(&dev->vbl_lock);
1323 assert_spin_locked(&dev->vblank_time_lock);
1325 vblank = &dev->vblank[pipe];
1326 WARN_ONCE((drm_debug & DRM_UT_VBL) && !vblank->framedur_ns,
1327 "Cannot compute missed vblanks without frame duration\n");
1328 framedur_ns = vblank->framedur_ns;
1331 cur_vblank = __get_vblank_counter(dev, pipe);
1332 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1333 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1335 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1337 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1340 DRM_DEBUG_VBL("missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1341 diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1342 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
1344 EXPORT_SYMBOL(drm_vblank_restore);
1347 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1348 * @crtc: CRTC in question
1350 * Power manamement features can cause frame counter resets between vblank
1351 * disable and enable. Drivers can use this function in their
1352 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1353 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1356 void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1358 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1360 EXPORT_SYMBOL(drm_crtc_vblank_restore);
1362 static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1365 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1367 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1368 if (!dev->num_crtcs)
1371 if (WARN_ON(pipe >= dev->num_crtcs))
1375 * To avoid all the problems that might happen if interrupts
1376 * were enabled/disabled around or between these calls, we just
1377 * have the kernel take a reference on the CRTC (just once though
1378 * to avoid corrupting the count if multiple, mismatch calls occur),
1379 * so that interrupts remain enabled in the interim.
1381 if (!vblank->inmodeset) {
1382 vblank->inmodeset = 0x1;
1383 if (drm_vblank_get(dev, pipe) == 0)
1384 vblank->inmodeset |= 0x2;
1388 static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1391 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1392 unsigned long irqflags;
1394 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1395 if (!dev->num_crtcs)
1398 if (WARN_ON(pipe >= dev->num_crtcs))
1401 if (vblank->inmodeset) {
1402 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1403 drm_reset_vblank_timestamp(dev, pipe);
1404 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1406 if (vblank->inmodeset & 0x2)
1407 drm_vblank_put(dev, pipe);
1409 vblank->inmodeset = 0;
1413 int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1414 struct drm_file *file_priv)
1416 struct drm_modeset_ctl *modeset = data;
1419 /* If drm_vblank_init() hasn't been called yet, just no-op */
1420 if (!dev->num_crtcs)
1423 /* KMS drivers handle this internally */
1424 if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1427 pipe = modeset->crtc;
1428 if (pipe >= dev->num_crtcs)
1431 switch (modeset->cmd) {
1432 case _DRM_PRE_MODESET:
1433 drm_legacy_vblank_pre_modeset(dev, pipe);
1435 case _DRM_POST_MODESET:
1436 drm_legacy_vblank_post_modeset(dev, pipe);
1445 static inline bool vblank_passed(u64 seq, u64 ref)
1447 return (seq - ref) <= (1 << 23);
1450 static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1452 union drm_wait_vblank *vblwait,
1453 struct drm_file *file_priv)
1455 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1456 struct drm_pending_vblank_event *e;
1458 unsigned long flags;
1462 e = kzalloc(sizeof(*e), GFP_KERNEL);
1469 e->event.base.type = DRM_EVENT_VBLANK;
1470 e->event.base.length = sizeof(e->event.vbl);
1471 e->event.vbl.user_data = vblwait->request.signal;
1472 e->event.vbl.crtc_id = 0;
1473 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1474 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1476 e->event.vbl.crtc_id = crtc->base.id;
1479 spin_lock_irqsave(&dev->event_lock, flags);
1482 * drm_crtc_vblank_off() might have been called after we called
1483 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1484 * vblank disable, so no need for further locking. The reference from
1485 * drm_vblank_get() protects against vblank disable from another source.
1487 if (!READ_ONCE(vblank->enabled)) {
1492 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1498 seq = drm_vblank_count_and_time(dev, pipe, &now);
1500 DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n",
1501 req_seq, seq, pipe);
1503 trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1505 e->sequence = req_seq;
1506 if (vblank_passed(seq, req_seq)) {
1507 drm_vblank_put(dev, pipe);
1508 send_vblank_event(dev, e, seq, now);
1509 vblwait->reply.sequence = seq;
1511 /* drm_handle_vblank_events will call drm_vblank_put */
1512 list_add_tail(&e->base.link, &dev->vblank_event_list);
1513 vblwait->reply.sequence = req_seq;
1516 spin_unlock_irqrestore(&dev->event_lock, flags);
1521 spin_unlock_irqrestore(&dev->event_lock, flags);
1524 drm_vblank_put(dev, pipe);
1528 static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1530 if (vblwait->request.sequence)
1533 return _DRM_VBLANK_RELATIVE ==
1534 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1536 _DRM_VBLANK_NEXTONMISS));
1540 * Widen a 32-bit param to 64-bits.
1542 * \param narrow 32-bit value (missing upper 32 bits)
1543 * \param near 64-bit value that should be 'close' to near
1545 * This function returns a 64-bit value using the lower 32-bits from
1546 * 'narrow' and constructing the upper 32-bits so that the result is
1547 * as close as possible to 'near'.
1550 static u64 widen_32_to_64(u32 narrow, u64 near)
1552 return near + (s32) (narrow - near);
1555 static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1556 struct drm_wait_vblank_reply *reply)
1559 struct timespec64 ts;
1562 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1563 * to store the seconds. This is safe as we always use monotonic
1564 * timestamps since linux-4.15.
1566 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1567 ts = ktime_to_timespec64(now);
1568 reply->tval_sec = (u32)ts.tv_sec;
1569 reply->tval_usec = ts.tv_nsec / 1000;
1572 int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1573 struct drm_file *file_priv)
1575 struct drm_crtc *crtc;
1576 struct drm_vblank_crtc *vblank;
1577 union drm_wait_vblank *vblwait = data;
1580 unsigned int pipe_index;
1581 unsigned int flags, pipe, high_pipe;
1583 if (!dev->irq_enabled)
1586 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1589 if (vblwait->request.type &
1590 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1591 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1592 DRM_DEBUG("Unsupported type value 0x%x, supported mask 0x%x\n",
1593 vblwait->request.type,
1594 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1595 _DRM_VBLANK_HIGH_CRTC_MASK));
1599 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1600 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1602 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1604 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1606 /* Convert lease-relative crtc index into global crtc index */
1607 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1609 drm_for_each_crtc(crtc, dev) {
1610 if (drm_lease_held(file_priv, crtc->base.id)) {
1611 if (pipe_index == 0)
1621 if (pipe >= dev->num_crtcs)
1624 vblank = &dev->vblank[pipe];
1626 /* If the counter is currently enabled and accurate, short-circuit
1627 * queries to return the cached timestamp of the last vblank.
1629 if (dev->vblank_disable_immediate &&
1630 drm_wait_vblank_is_query(vblwait) &&
1631 READ_ONCE(vblank->enabled)) {
1632 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1636 ret = drm_vblank_get(dev, pipe);
1638 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1641 seq = drm_vblank_count(dev, pipe);
1643 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1644 case _DRM_VBLANK_RELATIVE:
1645 req_seq = seq + vblwait->request.sequence;
1646 vblwait->request.sequence = req_seq;
1647 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1649 case _DRM_VBLANK_ABSOLUTE:
1650 req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1657 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1658 vblank_passed(seq, req_seq)) {
1660 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1661 vblwait->request.sequence = req_seq;
1664 if (flags & _DRM_VBLANK_EVENT) {
1665 /* must hold on to the vblank ref until the event fires
1666 * drm_vblank_put will be called asynchronously
1668 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1671 if (req_seq != seq) {
1674 DRM_DEBUG("waiting on vblank count %llu, crtc %u\n",
1676 wait = wait_event_interruptible_timeout(vblank->queue,
1677 vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1678 !READ_ONCE(vblank->enabled),
1679 msecs_to_jiffies(3000));
1687 /* interrupted by signal */
1696 if (ret != -EINTR) {
1697 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1699 DRM_DEBUG("crtc %d returning %u to client\n",
1700 pipe, vblwait->reply.sequence);
1702 DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe);
1706 drm_vblank_put(dev, pipe);
1710 static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1712 struct drm_pending_vblank_event *e, *t;
1716 assert_spin_locked(&dev->event_lock);
1718 seq = drm_vblank_count_and_time(dev, pipe, &now);
1720 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1721 if (e->pipe != pipe)
1723 if (!vblank_passed(seq, e->sequence))
1726 DRM_DEBUG("vblank event on %llu, current %llu\n",
1729 list_del(&e->base.link);
1730 drm_vblank_put(dev, pipe);
1731 send_vblank_event(dev, e, seq, now);
1734 trace_drm_vblank_event(pipe, seq);
1738 * drm_handle_vblank - handle a vblank event
1740 * @pipe: index of CRTC where this event occurred
1742 * Drivers should call this routine in their vblank interrupt handlers to
1743 * update the vblank counter and send any signals that may be pending.
1745 * This is the legacy version of drm_crtc_handle_vblank().
1747 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1749 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1750 unsigned long irqflags;
1753 if (WARN_ON_ONCE(!dev->num_crtcs))
1756 if (WARN_ON(pipe >= dev->num_crtcs))
1759 spin_lock_irqsave(&dev->event_lock, irqflags);
1761 /* Need timestamp lock to prevent concurrent execution with
1762 * vblank enable/disable, as this would cause inconsistent
1763 * or corrupted timestamps and vblank counts.
1765 spin_lock(&dev->vblank_time_lock);
1767 /* Vblank irq handling disabled. Nothing to do. */
1768 if (!vblank->enabled) {
1769 spin_unlock(&dev->vblank_time_lock);
1770 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1774 drm_update_vblank_count(dev, pipe, true);
1776 spin_unlock(&dev->vblank_time_lock);
1778 wake_up(&vblank->queue);
1780 /* With instant-off, we defer disabling the interrupt until after
1781 * we finish processing the following vblank after all events have
1782 * been signaled. The disable has to be last (after
1783 * drm_handle_vblank_events) so that the timestamp is always accurate.
1785 disable_irq = (dev->vblank_disable_immediate &&
1786 drm_vblank_offdelay > 0 &&
1787 !atomic_read(&vblank->refcount));
1789 drm_handle_vblank_events(dev, pipe);
1791 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1794 vblank_disable_fn(&vblank->disable_timer);
1798 EXPORT_SYMBOL(drm_handle_vblank);
1801 * drm_crtc_handle_vblank - handle a vblank event
1802 * @crtc: where this event occurred
1804 * Drivers should call this routine in their vblank interrupt handlers to
1805 * update the vblank counter and send any signals that may be pending.
1807 * This is the native KMS version of drm_handle_vblank().
1810 * True if the event was successfully handled, false on failure.
1812 bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1814 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1816 EXPORT_SYMBOL(drm_crtc_handle_vblank);
1819 * Get crtc VBLANK count.
1821 * \param dev DRM device
1822 * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
1823 * \param file_priv drm file private for the user's open file descriptor
1826 int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1827 struct drm_file *file_priv)
1829 struct drm_crtc *crtc;
1830 struct drm_vblank_crtc *vblank;
1832 struct drm_crtc_get_sequence *get_seq = data;
1834 bool vblank_enabled;
1837 if (!drm_core_check_feature(dev, DRIVER_MODESET))
1840 if (!dev->irq_enabled)
1843 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1847 pipe = drm_crtc_index(crtc);
1849 vblank = &dev->vblank[pipe];
1850 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1852 if (!vblank_enabled) {
1853 ret = drm_crtc_vblank_get(crtc);
1855 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1859 drm_modeset_lock(&crtc->mutex, NULL);
1861 get_seq->active = crtc->state->enable;
1863 get_seq->active = crtc->enabled;
1864 drm_modeset_unlock(&crtc->mutex);
1865 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1866 get_seq->sequence_ns = ktime_to_ns(now);
1867 if (!vblank_enabled)
1868 drm_crtc_vblank_put(crtc);
1873 * Queue a event for VBLANK sequence
1875 * \param dev DRM device
1876 * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
1877 * \param file_priv drm file private for the user's open file descriptor
1880 int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
1881 struct drm_file *file_priv)
1883 struct drm_crtc *crtc;
1884 struct drm_vblank_crtc *vblank;
1886 struct drm_crtc_queue_sequence *queue_seq = data;
1888 struct drm_pending_vblank_event *e;
1893 unsigned long spin_flags;
1895 if (!drm_core_check_feature(dev, DRIVER_MODESET))
1898 if (!dev->irq_enabled)
1901 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
1905 flags = queue_seq->flags;
1906 /* Check valid flag bits */
1907 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
1908 DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
1911 pipe = drm_crtc_index(crtc);
1913 vblank = &dev->vblank[pipe];
1915 e = kzalloc(sizeof(*e), GFP_KERNEL);
1919 ret = drm_crtc_vblank_get(crtc);
1921 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1925 seq = drm_vblank_count_and_time(dev, pipe, &now);
1926 req_seq = queue_seq->sequence;
1928 if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
1931 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
1935 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
1936 e->event.base.length = sizeof(e->event.seq);
1937 e->event.seq.user_data = queue_seq->user_data;
1939 spin_lock_irqsave(&dev->event_lock, spin_flags);
1942 * drm_crtc_vblank_off() might have been called after we called
1943 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1944 * vblank disable, so no need for further locking. The reference from
1945 * drm_crtc_vblank_get() protects against vblank disable from another source.
1947 if (!READ_ONCE(vblank->enabled)) {
1952 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1958 e->sequence = req_seq;
1960 if (vblank_passed(seq, req_seq)) {
1961 drm_crtc_vblank_put(crtc);
1962 send_vblank_event(dev, e, seq, now);
1963 queue_seq->sequence = seq;
1965 /* drm_handle_vblank_events will call drm_vblank_put */
1966 list_add_tail(&e->base.link, &dev->vblank_event_list);
1967 queue_seq->sequence = req_seq;
1970 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1974 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1975 drm_crtc_vblank_put(crtc);