/**************************************************************************
- *
- * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
+ *
+ * Copyright (c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
* All Rights Reserved.
- *
+ *
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
- *
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- *
**************************************************************************/
/*
* Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
#include "drmP.h"
+
+/*
+ * Convenience function to be called by fence::wait methods that
+ * need polling.
+ */
+
+int drm_fence_wait_polling(struct drm_fence_object *fence, int lazy,
+ int interruptible, uint32_t mask,
+ unsigned long end_jiffies)
+{
+ struct drm_device *dev = fence->dev;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class];
+ uint32_t count = 0;
+ int ret;
+
+ DECLARE_WAITQUEUE(entry, current);
+ add_wait_queue(&fc->fence_queue, &entry);
+
+ ret = 0;
+
+ for (;;) {
+ __set_current_state((interruptible) ?
+ TASK_INTERRUPTIBLE :
+ TASK_UNINTERRUPTIBLE);
+ if (drm_fence_object_signaled(fence, mask))
+ break;
+ if (time_after_eq(jiffies, end_jiffies)) {
+ ret = -EBUSY;
+ break;
+ }
+ if (lazy)
+ schedule_timeout(1);
+ else if ((++count & 0x0F) == 0){
+ __set_current_state(TASK_RUNNING);
+ schedule();
+ __set_current_state((interruptible) ?
+ TASK_INTERRUPTIBLE :
+ TASK_UNINTERRUPTIBLE);
+ }
+ if (interruptible && signal_pending(current)) {
+ ret = -EAGAIN;
+ break;
+ }
+ }
+ __set_current_state(TASK_RUNNING);
+ remove_wait_queue(&fc->fence_queue, &entry);
+ return ret;
+}
+EXPORT_SYMBOL(drm_fence_wait_polling);
+
/*
* Typically called by the IRQ handler.
*/
-void drm_fence_handler(drm_device_t * dev, uint32_t sequence, uint32_t type)
+void drm_fence_handler(struct drm_device *dev, uint32_t fence_class,
+ uint32_t sequence, uint32_t type, uint32_t error)
{
int wake = 0;
uint32_t diff;
- uint32_t relevant;
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_driver_t *driver = dev->driver->fence_driver;
- struct list_head *list, *prev;
- drm_fence_object_t *fence;
+ uint32_t relevant_type;
+ uint32_t new_type;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_class_manager *fc = &fm->fence_class[fence_class];
+ struct drm_fence_driver *driver = dev->driver->fence_driver;
+ struct list_head *head;
+ struct drm_fence_object *fence, *next;
int found = 0;
- if (list_empty(&fm->ring))
+ if (list_empty(&fc->ring))
return;
- list_for_each_entry(fence, &fm->ring, ring) {
+ list_for_each_entry(fence, &fc->ring, ring) {
diff = (sequence - fence->sequence) & driver->sequence_mask;
if (diff > driver->wrap_diff) {
found = 1;
}
}
- list = (found) ? fence->ring.prev : fm->ring.prev;
- prev = list->prev;
-
- for (; list != &fm->ring; list = prev, prev = list->prev) {
- fence = list_entry(list, drm_fence_object_t, ring);
+ fc->waiting_types &= ~type;
+ head = (found) ? &fence->ring : &fc->ring;
- type |= fence->native_type;
- relevant = type & fence->type;
+ list_for_each_entry_safe_reverse(fence, next, head, ring) {
+ if (&fence->ring == &fc->ring)
+ break;
- if ((fence->signaled | relevant) != fence->signaled) {
- fence->signaled |= relevant;
- DRM_DEBUG("Fence 0x%08lx signaled 0x%08x\n",
- fence->base.hash.key, fence->signaled);
- fence->submitted_flush |= relevant;
+ if (error) {
+ fence->error = error;
+ fence->signaled_types = fence->type;
+ list_del_init(&fence->ring);
wake = 1;
+ break;
}
- relevant = fence->flush_mask &
- ~(fence->signaled | fence->submitted_flush);
+ if (type & DRM_FENCE_TYPE_EXE)
+ type |= fence->native_types;
- if (relevant) {
- fm->pending_flush |= relevant;
- fence->submitted_flush = fence->flush_mask;
+ relevant_type = type & fence->type;
+ new_type = (fence->signaled_types | relevant_type) ^
+ fence->signaled_types;
+
+ if (new_type) {
+ fence->signaled_types |= new_type;
+ DRM_DEBUG("Fence 0x%08lx signaled 0x%08x\n",
+ fence->base.hash.key, fence->signaled_types);
+
+ if (driver->needed_flush)
+ fc->pending_flush |= driver->needed_flush(fence);
+
+ if (new_type & fence->waiting_types)
+ wake = 1;
}
- if (!(fence->type & ~fence->signaled)) {
+ fc->waiting_types |= fence->waiting_types & ~fence->signaled_types;
+
+ if (!(fence->type & ~fence->signaled_types)) {
DRM_DEBUG("Fence completely signaled 0x%08lx\n",
fence->base.hash.key);
list_del_init(&fence->ring);
}
-
}
- if (wake) {
- DRM_WAKEUP(&fm->fence_queue);
+ /*
+ * Reinstate lost waiting types.
+ */
+
+ if ((fc->waiting_types & type) != type) {
+ head = head->prev;
+ list_for_each_entry(fence, head, ring) {
+ if (&fence->ring == &fc->ring)
+ break;
+ diff = (fc->highest_waiting_sequence - fence->sequence) &
+ driver->sequence_mask;
+ if (diff > driver->wrap_diff)
+ break;
+
+ fc->waiting_types |= fence->waiting_types & ~fence->signaled_types;
+ }
}
-}
+ if (wake)
+ wake_up_all(&fc->fence_queue);
+}
EXPORT_SYMBOL(drm_fence_handler);
-static void drm_fence_unring(drm_device_t * dev, struct list_head *ring)
+static void drm_fence_unring(struct drm_device *dev, struct list_head *ring)
{
- drm_fence_manager_t *fm = &dev->fm;
+ struct drm_fence_manager *fm = &dev->fm;
unsigned long flags;
write_lock_irqsave(&fm->lock, flags);
write_unlock_irqrestore(&fm->lock, flags);
}
-void drm_fence_usage_deref_locked(drm_device_t * dev,
- drm_fence_object_t * fence)
+void drm_fence_usage_deref_locked(struct drm_fence_object **fence)
{
- drm_fence_manager_t *fm = &dev->fm;
-
- if (atomic_dec_and_test(&fence->usage)) {
- drm_fence_unring(dev, &fence->ring);
+ struct drm_fence_object *tmp_fence = *fence;
+ struct drm_device *dev = tmp_fence->dev;
+ struct drm_fence_manager *fm = &dev->fm;
+
+ DRM_ASSERT_LOCKED(&dev->struct_mutex);
+ *fence = NULL;
+ if (atomic_dec_and_test(&tmp_fence->usage)) {
+ drm_fence_unring(dev, &tmp_fence->ring);
DRM_DEBUG("Destroyed a fence object 0x%08lx\n",
- fence->base.hash.key);
+ tmp_fence->base.hash.key);
atomic_dec(&fm->count);
- drm_ctl_cache_free(drm_cache.fence_object, sizeof(*fence),
- fence);
+ BUG_ON(!list_empty(&tmp_fence->base.list));
+ drm_ctl_free(tmp_fence, sizeof(*tmp_fence), DRM_MEM_FENCE);
}
}
+EXPORT_SYMBOL(drm_fence_usage_deref_locked);
-void drm_fence_usage_deref_unlocked(drm_device_t * dev,
- drm_fence_object_t * fence)
+void drm_fence_usage_deref_unlocked(struct drm_fence_object **fence)
{
- drm_fence_manager_t *fm = &dev->fm;
+ struct drm_fence_object *tmp_fence = *fence;
+ struct drm_device *dev = tmp_fence->dev;
+ struct drm_fence_manager *fm = &dev->fm;
- if (atomic_dec_and_test(&fence->usage)) {
+ *fence = NULL;
+ if (atomic_dec_and_test(&tmp_fence->usage)) {
mutex_lock(&dev->struct_mutex);
- if (atomic_read(&fence->usage) == 0) {
- drm_fence_unring(dev, &fence->ring);
+ if (atomic_read(&tmp_fence->usage) == 0) {
+ drm_fence_unring(dev, &tmp_fence->ring);
atomic_dec(&fm->count);
- drm_ctl_cache_free(drm_cache.fence_object,
- sizeof(*fence), fence);
+ BUG_ON(!list_empty(&tmp_fence->base.list));
+ drm_ctl_free(tmp_fence, sizeof(*tmp_fence), DRM_MEM_FENCE);
}
mutex_unlock(&dev->struct_mutex);
}
}
+EXPORT_SYMBOL(drm_fence_usage_deref_unlocked);
-static void drm_fence_object_destroy(drm_file_t * priv,
- drm_user_object_t * base)
+struct drm_fence_object
+*drm_fence_reference_locked(struct drm_fence_object *src)
{
- drm_device_t *dev = priv->head->dev;
- drm_fence_object_t *fence =
- drm_user_object_entry(base, drm_fence_object_t, base);
+ DRM_ASSERT_LOCKED(&src->dev->struct_mutex);
- drm_fence_usage_deref_locked(dev, fence);
+ atomic_inc(&src->usage);
+ return src;
}
-static int fence_signaled(drm_device_t * dev, volatile
- drm_fence_object_t * fence,
- uint32_t mask, int poke_flush)
+void drm_fence_reference_unlocked(struct drm_fence_object **dst,
+ struct drm_fence_object *src)
{
- unsigned long flags;
- int signaled;
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_driver_t *driver = dev->driver->fence_driver;
-
- if (poke_flush)
- driver->poke_flush(dev);
- read_lock_irqsave(&fm->lock, flags);
- signaled =
- (fence->type & mask & fence->signaled) == (fence->type & mask);
- read_unlock_irqrestore(&fm->lock, flags);
-
- return signaled;
+ mutex_lock(&src->dev->struct_mutex);
+ *dst = src;
+ atomic_inc(&src->usage);
+ mutex_unlock(&src->dev->struct_mutex);
}
+EXPORT_SYMBOL(drm_fence_reference_unlocked);
-static void drm_fence_flush_exe(drm_fence_manager_t * fm,
- drm_fence_driver_t * driver, uint32_t sequence)
+static void drm_fence_object_destroy(struct drm_file *priv,
+ struct drm_user_object *base)
{
- uint32_t diff;
+ struct drm_fence_object *fence =
+ drm_user_object_entry(base, struct drm_fence_object, base);
- if (!fm->pending_exe_flush) {
- volatile struct list_head *list;
-
- /*
- * Last_exe_flush is invalid. Find oldest sequence.
- */
-
-/* list = fm->fence_types[_DRM_FENCE_TYPE_EXE];*/
- list = &fm->ring;
- if (list->next == &fm->ring) {
- return;
- } else {
- drm_fence_object_t *fence =
- list_entry(list->next, drm_fence_object_t, ring);
- fm->last_exe_flush = (fence->sequence - 1) &
- driver->sequence_mask;
- }
- diff = (sequence - fm->last_exe_flush) & driver->sequence_mask;
- if (diff >= driver->wrap_diff)
- return;
- fm->exe_flush_sequence = sequence;
- fm->pending_exe_flush = 1;
- } else {
- diff =
- (sequence - fm->exe_flush_sequence) & driver->sequence_mask;
- if (diff < driver->wrap_diff) {
- fm->exe_flush_sequence = sequence;
- }
- }
+ drm_fence_usage_deref_locked(&fence);
}
-int drm_fence_object_signaled(volatile drm_fence_object_t * fence,
- uint32_t type)
+int drm_fence_object_signaled(struct drm_fence_object *fence, uint32_t mask)
{
- return ((fence->signaled & type) == type);
+ unsigned long flags;
+ int signaled;
+ struct drm_device *dev = fence->dev;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_driver *driver = dev->driver->fence_driver;
+
+ mask &= fence->type;
+ read_lock_irqsave(&fm->lock, flags);
+ signaled = (mask & fence->signaled_types) == mask;
+ read_unlock_irqrestore(&fm->lock, flags);
+ if (!signaled && driver->poll) {
+ write_lock_irqsave(&fm->lock, flags);
+ driver->poll(dev, fence->fence_class, mask);
+ signaled = (mask & fence->signaled_types) == mask;
+ write_unlock_irqrestore(&fm->lock, flags);
+ }
+ return signaled;
}
+EXPORT_SYMBOL(drm_fence_object_signaled);
+
-int drm_fence_object_flush(drm_device_t * dev,
- volatile drm_fence_object_t * fence, uint32_t type)
+int drm_fence_object_flush(struct drm_fence_object *fence,
+ uint32_t type)
{
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_driver_t *driver = dev->driver->fence_driver;
- unsigned long flags;
+ struct drm_device *dev = fence->dev;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class];
+ struct drm_fence_driver *driver = dev->driver->fence_driver;
+ unsigned long irq_flags;
+ uint32_t saved_pending_flush;
+ uint32_t diff;
+ int call_flush;
if (type & ~fence->type) {
DRM_ERROR("Flush trying to extend fence type, "
return -EINVAL;
}
- write_lock_irqsave(&fm->lock, flags);
- fence->flush_mask |= type;
- if (fence->submitted_flush == fence->signaled) {
- if ((fence->type & DRM_FENCE_TYPE_EXE) &&
- !(fence->submitted_flush & DRM_FENCE_TYPE_EXE)) {
- drm_fence_flush_exe(fm, driver, fence->sequence);
- fence->submitted_flush |= DRM_FENCE_TYPE_EXE;
- } else {
- fm->pending_flush |= (fence->flush_mask &
- ~fence->submitted_flush);
- fence->submitted_flush = fence->flush_mask;
- }
- }
- write_unlock_irqrestore(&fm->lock, flags);
- driver->poke_flush(dev);
+ write_lock_irqsave(&fm->lock, irq_flags);
+ fence->waiting_types |= type;
+ fc->waiting_types |= fence->waiting_types;
+ diff = (fence->sequence - fc->highest_waiting_sequence) &
+ driver->sequence_mask;
+
+ if (diff < driver->wrap_diff)
+ fc->highest_waiting_sequence = fence->sequence;
+
+ /*
+ * fence->waiting_types has changed. Determine whether
+ * we need to initiate some kind of flush as a result of this.
+ */
+
+ saved_pending_flush = fc->pending_flush;
+ if (driver->needed_flush)
+ fc->pending_flush |= driver->needed_flush(fence);
+
+ if (driver->poll)
+ driver->poll(dev, fence->fence_class, fence->waiting_types);
+
+ call_flush = fc->pending_flush;
+ write_unlock_irqrestore(&fm->lock, irq_flags);
+
+ if (call_flush && driver->flush)
+ driver->flush(dev, fence->fence_class);
+
return 0;
}
+EXPORT_SYMBOL(drm_fence_object_flush);
/*
* Make sure old fence objects are signaled before their fence sequences are
* wrapped around and reused.
*/
-void drm_fence_flush_old(drm_device_t * dev, uint32_t sequence)
+void drm_fence_flush_old(struct drm_device *dev, uint32_t fence_class,
+ uint32_t sequence)
{
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_driver_t *driver = dev->driver->fence_driver;
- uint32_t old_sequence;
- unsigned long flags;
- drm_fence_object_t *fence;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_class_manager *fc = &fm->fence_class[fence_class];
+ struct drm_fence_object *fence;
+ unsigned long irq_flags;
+ struct drm_fence_driver *driver = dev->driver->fence_driver;
+ int call_flush;
+
uint32_t diff;
- mutex_lock(&dev->struct_mutex);
- read_lock_irqsave(&fm->lock, flags);
- if (fm->ring.next == &fm->ring) {
- read_unlock_irqrestore(&fm->lock, flags);
- mutex_unlock(&dev->struct_mutex);
- return;
- }
- old_sequence = (sequence - driver->flush_diff) & driver->sequence_mask;
- fence = list_entry(fm->ring.next, drm_fence_object_t, ring);
- atomic_inc(&fence->usage);
- mutex_unlock(&dev->struct_mutex);
- diff = (old_sequence - fence->sequence) & driver->sequence_mask;
- read_unlock_irqrestore(&fm->lock, flags);
- if (diff < driver->wrap_diff) {
- drm_fence_object_flush(dev, fence, fence->type);
- }
- drm_fence_usage_deref_unlocked(dev, fence);
-}
+ write_lock_irqsave(&fm->lock, irq_flags);
+
+ list_for_each_entry_reverse(fence, &fc->ring, ring) {
+ diff = (sequence - fence->sequence) & driver->sequence_mask;
+ if (diff <= driver->flush_diff)
+ break;
+
+ fence->waiting_types = fence->type;
+ fc->waiting_types |= fence->type;
+
+ if (driver->needed_flush)
+ fc->pending_flush |= driver->needed_flush(fence);
+ }
+
+ if (driver->poll)
+ driver->poll(dev, fence_class, fc->waiting_types);
+ call_flush = fc->pending_flush;
+ write_unlock_irqrestore(&fm->lock, irq_flags);
+
+ if (call_flush && driver->flush)
+ driver->flush(dev, fence->fence_class);
+
+ /*
+ * FIXME: Shold we implement a wait here for really old fences?
+ */
+
+}
EXPORT_SYMBOL(drm_fence_flush_old);
-int drm_fence_object_wait(drm_device_t * dev,
- volatile drm_fence_object_t * fence,
+int drm_fence_object_wait(struct drm_fence_object *fence,
int lazy, int ignore_signals, uint32_t mask)
{
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_driver_t *driver = dev->driver->fence_driver;
+ struct drm_device *dev = fence->dev;
+ struct drm_fence_driver *driver = dev->driver->fence_driver;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class];
int ret = 0;
- unsigned long _end;
- int signaled;
+ unsigned long _end = 3 * DRM_HZ;
if (mask & ~fence->type) {
DRM_ERROR("Wait trying to extend fence type"
" 0x%08x 0x%08x\n", mask, fence->type);
+ BUG();
return -EINVAL;
}
- if (fence_signaled(dev, fence, mask, 0))
- return 0;
+ if (driver->wait)
+ return driver->wait(fence, lazy, !ignore_signals, mask);
- _end = jiffies + 3 * DRM_HZ;
- drm_fence_object_flush(dev, fence, mask);
+ drm_fence_object_flush(fence, mask);
+ if (driver->has_irq(dev, fence->fence_class, mask)) {
+ if (!ignore_signals)
+ ret = wait_event_interruptible_timeout
+ (fc->fence_queue,
+ drm_fence_object_signaled(fence, mask),
+ 3 * DRM_HZ);
+ else
+ ret = wait_event_timeout
+ (fc->fence_queue,
+ drm_fence_object_signaled(fence, mask),
+ 3 * DRM_HZ);
- if (lazy && driver->lazy_capable) {
+ if (unlikely(ret == -ERESTARTSYS))
+ return -EAGAIN;
- do {
- DRM_WAIT_ON(ret, fm->fence_queue, 3 * DRM_HZ,
- fence_signaled(dev, fence, mask, 1));
- if (time_after_eq(jiffies, _end))
- break;
- } while (ret == -EINTR && ignore_signals);
- if (time_after_eq(jiffies, _end) && (ret != 0))
- ret = -EBUSY;
- if (ret) {
- if (ret == -EBUSY) {
- DRM_ERROR("Fence timeout. "
- "GPU lockup or fence driver was "
- "taken down.\n");
- }
- return ((ret == -EINTR) ? -EAGAIN : ret);
- }
- } else if ((fence->class == 0) && (mask & DRM_FENCE_TYPE_EXE) &&
- driver->lazy_capable) {
-
- /*
- * We use IRQ wait for EXE fence if available to gain
- * CPU in some cases.
- */
-
- do {
- DRM_WAIT_ON(ret, fm->fence_queue, 3 * DRM_HZ,
- fence_signaled(dev, fence,
- DRM_FENCE_TYPE_EXE, 1));
- if (time_after_eq(jiffies, _end))
- break;
- } while (ret == -EINTR && ignore_signals);
- if (time_after_eq(jiffies, _end) && (ret != 0))
- ret = -EBUSY;
- if (ret)
- return ((ret == -EINTR) ? -EAGAIN : ret);
- }
+ if (unlikely(ret == 0))
+ return -EBUSY;
- if (fence_signaled(dev, fence, mask, 0))
return 0;
+ }
- /*
- * Avoid kernel-space busy-waits.
- */
-#if 1
- if (!ignore_signals)
- return -EAGAIN;
-#endif
- do {
- schedule();
- signaled = fence_signaled(dev, fence, mask, 1);
- } while (!signaled && !time_after_eq(jiffies, _end));
-
- if (!signaled)
- return -EBUSY;
-
- return 0;
+ return drm_fence_wait_polling(fence, lazy, !ignore_signals, mask,
+ _end);
}
+EXPORT_SYMBOL(drm_fence_object_wait);
+
+
-int drm_fence_object_emit(drm_device_t * dev, drm_fence_object_t * fence,
- uint32_t fence_flags, uint32_t type)
+int drm_fence_object_emit(struct drm_fence_object *fence, uint32_t fence_flags,
+ uint32_t fence_class, uint32_t type)
{
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_driver_t *driver = dev->driver->fence_driver;
+ struct drm_device *dev = fence->dev;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_driver *driver = dev->driver->fence_driver;
+ struct drm_fence_class_manager *fc = &fm->fence_class[fence->fence_class];
unsigned long flags;
uint32_t sequence;
- uint32_t native_type;
+ uint32_t native_types;
int ret;
drm_fence_unring(dev, &fence->ring);
- ret = driver->emit(dev, fence_flags, &sequence, &native_type);
+ ret = driver->emit(dev, fence_class, fence_flags, &sequence,
+ &native_types);
if (ret)
return ret;
write_lock_irqsave(&fm->lock, flags);
+ fence->fence_class = fence_class;
fence->type = type;
- fence->flush_mask = 0x00;
- fence->submitted_flush = 0x00;
- fence->signaled = 0x00;
+ fence->waiting_types = 0;
+ fence->signaled_types = 0;
+ fence->error = 0;
fence->sequence = sequence;
- fence->native_type = native_type;
- list_add_tail(&fence->ring, &fm->ring);
+ fence->native_types = native_types;
+ if (list_empty(&fc->ring))
+ fc->highest_waiting_sequence = sequence - 1;
+ list_add_tail(&fence->ring, &fc->ring);
+ fc->latest_queued_sequence = sequence;
write_unlock_irqrestore(&fm->lock, flags);
return 0;
}
+EXPORT_SYMBOL(drm_fence_object_emit);
-static int drm_fence_object_init(drm_device_t * dev, uint32_t type,
+static int drm_fence_object_init(struct drm_device *dev, uint32_t fence_class,
+ uint32_t type,
uint32_t fence_flags,
- drm_fence_object_t * fence)
+ struct drm_fence_object *fence)
{
int ret = 0;
unsigned long flags;
- drm_fence_manager_t *fm = &dev->fm;
+ struct drm_fence_manager *fm = &dev->fm;
mutex_lock(&dev->struct_mutex);
atomic_set(&fence->usage, 1);
write_lock_irqsave(&fm->lock, flags);
INIT_LIST_HEAD(&fence->ring);
- fence->class = 0;
+
+ /*
+ * Avoid hitting BUG() for kernel-only fence objects.
+ */
+
+ INIT_LIST_HEAD(&fence->base.list);
+ fence->fence_class = fence_class;
fence->type = type;
- fence->flush_mask = 0;
- fence->submitted_flush = 0;
- fence->signaled = 0;
+ fence->signaled_types = 0;
+ fence->waiting_types = 0;
fence->sequence = 0;
+ fence->error = 0;
+ fence->dev = dev;
write_unlock_irqrestore(&fm->lock, flags);
if (fence_flags & DRM_FENCE_FLAG_EMIT) {
- ret = drm_fence_object_emit(dev, fence, fence_flags, type);
+ ret = drm_fence_object_emit(fence, fence_flags,
+ fence->fence_class, type);
}
return ret;
}
-int drm_fence_add_user_object(drm_file_t * priv, drm_fence_object_t * fence,
- int shareable)
+int drm_fence_add_user_object(struct drm_file *priv,
+ struct drm_fence_object *fence, int shareable)
{
- drm_device_t *dev = priv->head->dev;
+ struct drm_device *dev = priv->minor->dev;
int ret;
mutex_lock(&dev->struct_mutex);
ret = drm_add_user_object(priv, &fence->base, shareable);
- mutex_unlock(&dev->struct_mutex);
if (ret)
- return ret;
+ goto out;
+ atomic_inc(&fence->usage);
fence->base.type = drm_fence_type;
fence->base.remove = &drm_fence_object_destroy;
DRM_DEBUG("Fence 0x%08lx created\n", fence->base.hash.key);
- return 0;
+out:
+ mutex_unlock(&dev->struct_mutex);
+ return ret;
}
-
EXPORT_SYMBOL(drm_fence_add_user_object);
-int drm_fence_object_create(drm_device_t * dev, uint32_t type,
- unsigned flags, drm_fence_object_t ** c_fence)
+int drm_fence_object_create(struct drm_device *dev, uint32_t fence_class,
+ uint32_t type, unsigned flags,
+ struct drm_fence_object **c_fence)
{
- drm_fence_object_t *fence;
+ struct drm_fence_object *fence;
int ret;
- drm_fence_manager_t *fm = &dev->fm;
+ struct drm_fence_manager *fm = &dev->fm;
- fence = drm_ctl_cache_alloc(drm_cache.fence_object,
- sizeof(*fence), GFP_KERNEL);
- if (!fence)
+ fence = drm_ctl_calloc(1, sizeof(*fence), DRM_MEM_FENCE);
+ if (!fence) {
+ DRM_ERROR("Out of memory creating fence object\n");
return -ENOMEM;
- ret = drm_fence_object_init(dev, type, flags, fence);
+ }
+ ret = drm_fence_object_init(dev, fence_class, type, flags, fence);
if (ret) {
- drm_fence_usage_deref_unlocked(dev, fence);
+ drm_fence_usage_deref_unlocked(&fence);
return ret;
}
*c_fence = fence;
return 0;
}
-
EXPORT_SYMBOL(drm_fence_object_create);
-void drm_fence_manager_init(drm_device_t * dev)
+void drm_fence_manager_init(struct drm_device *dev)
{
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_driver_t *fed = dev->driver->fence_driver;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_class_manager *fence_class;
+ struct drm_fence_driver *fed = dev->driver->fence_driver;
int i;
+ unsigned long flags;
- fm->lock = RW_LOCK_UNLOCKED;
- write_lock(&fm->lock);
- INIT_LIST_HEAD(&fm->ring);
- fm->pending_flush = 0;
- DRM_INIT_WAITQUEUE(&fm->fence_queue);
+ rwlock_init(&fm->lock);
+ write_lock_irqsave(&fm->lock, flags);
fm->initialized = 0;
- if (fed) {
- fm->initialized = 1;
- atomic_set(&fm->count, 0);
- for (i = 0; i < fed->no_types; ++i) {
- fm->fence_types[i] = &fm->ring;
- }
+ if (!fed)
+ goto out_unlock;
+
+ fm->initialized = 1;
+ fm->num_classes = fed->num_classes;
+ BUG_ON(fm->num_classes > _DRM_FENCE_CLASSES);
+
+ for (i = 0; i < fm->num_classes; ++i) {
+ fence_class = &fm->fence_class[i];
+
+ memset(fence_class, 0, sizeof(*fence_class));
+ INIT_LIST_HEAD(&fence_class->ring);
+ DRM_INIT_WAITQUEUE(&fence_class->fence_queue);
}
- write_unlock(&fm->lock);
+
+ atomic_set(&fm->count, 0);
+ out_unlock:
+ write_unlock_irqrestore(&fm->lock, flags);
}
-void drm_fence_manager_takedown(drm_device_t * dev)
+void drm_fence_fill_arg(struct drm_fence_object *fence,
+ struct drm_fence_arg *arg)
{
+ struct drm_device *dev = fence->dev;
+ struct drm_fence_manager *fm = &dev->fm;
+ unsigned long irq_flags;
+
+ read_lock_irqsave(&fm->lock, irq_flags);
+ arg->handle = fence->base.hash.key;
+ arg->fence_class = fence->fence_class;
+ arg->type = fence->type;
+ arg->signaled = fence->signaled_types;
+ arg->error = fence->error;
+ arg->sequence = fence->sequence;
+ read_unlock_irqrestore(&fm->lock, irq_flags);
}
+EXPORT_SYMBOL(drm_fence_fill_arg);
-drm_fence_object_t *drm_lookup_fence_object(drm_file_t * priv, uint32_t handle)
+void drm_fence_manager_takedown(struct drm_device *dev)
{
- drm_device_t *dev = priv->head->dev;
- drm_user_object_t *uo;
- drm_fence_object_t *fence;
+}
+
+struct drm_fence_object *drm_lookup_fence_object(struct drm_file *priv,
+ uint32_t handle)
+{
+ struct drm_device *dev = priv->minor->dev;
+ struct drm_user_object *uo;
+ struct drm_fence_object *fence;
mutex_lock(&dev->struct_mutex);
uo = drm_lookup_user_object(priv, handle);
mutex_unlock(&dev->struct_mutex);
return NULL;
}
- fence = drm_user_object_entry(uo, drm_fence_object_t, base);
- atomic_inc(&fence->usage);
+ fence = drm_fence_reference_locked(drm_user_object_entry(uo, struct drm_fence_object, base));
mutex_unlock(&dev->struct_mutex);
return fence;
}
-int drm_fence_ioctl(DRM_IOCTL_ARGS)
+int drm_fence_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
{
- DRM_DEVICE;
int ret;
- drm_fence_manager_t *fm = &dev->fm;
- drm_fence_arg_t arg;
- drm_fence_object_t *fence;
- drm_user_object_t *uo;
- unsigned long flags;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ struct drm_fence_object *fence;
ret = 0;
if (!fm->initialized) {
return -EINVAL;
}
- DRM_COPY_FROM_USER_IOCTL(arg, (void __user *)data, sizeof(arg));
- switch (arg.op) {
- case drm_fence_create:
- if (arg.flags & DRM_FENCE_FLAG_EMIT)
- LOCK_TEST_WITH_RETURN(dev, filp);
- ret = drm_fence_object_create(dev, arg.type, arg.flags, &fence);
- if (ret)
- return ret;
- ret = drm_fence_add_user_object(priv, fence,
- arg.flags &
- DRM_FENCE_FLAG_SHAREABLE);
- if (ret) {
- drm_fence_usage_deref_unlocked(dev, fence);
- return ret;
- }
+ if (arg->flags & DRM_FENCE_FLAG_EMIT)
+ LOCK_TEST_WITH_RETURN(dev, file_priv);
+ ret = drm_fence_object_create(dev, arg->fence_class,
+ arg->type, arg->flags, &fence);
+ if (ret)
+ return ret;
+ ret = drm_fence_add_user_object(file_priv, fence,
+ arg->flags &
+ DRM_FENCE_FLAG_SHAREABLE);
+ if (ret) {
+ drm_fence_usage_deref_unlocked(&fence);
+ return ret;
+ }
+
+ /*
+ * usage > 0. No need to lock dev->struct_mutex;
+ */
- /*
- * usage > 0. No need to lock dev->struct_mutex;
- */
+ arg->handle = fence->base.hash.key;
- atomic_inc(&fence->usage);
- arg.handle = fence->base.hash.key;
- break;
- case drm_fence_destroy:
- mutex_lock(&dev->struct_mutex);
- uo = drm_lookup_user_object(priv, arg.handle);
- if (!uo || (uo->type != drm_fence_type) || uo->owner != priv) {
- mutex_unlock(&dev->struct_mutex);
- return -EINVAL;
- }
- ret = drm_remove_user_object(priv, uo);
- mutex_unlock(&dev->struct_mutex);
+ drm_fence_fill_arg(fence, arg);
+ drm_fence_usage_deref_unlocked(&fence);
+
+ return ret;
+}
+
+int drm_fence_reference_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ int ret;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ struct drm_fence_object *fence;
+ struct drm_user_object *uo;
+ ret = 0;
+
+ if (!fm->initialized) {
+ DRM_ERROR("The DRM driver does not support fencing.\n");
+ return -EINVAL;
+ }
+
+ ret = drm_user_object_ref(file_priv, arg->handle, drm_fence_type, &uo);
+ if (ret)
return ret;
- case drm_fence_reference:
- ret =
- drm_user_object_ref(priv, arg.handle, drm_fence_type, &uo);
- if (ret)
- return ret;
- fence = drm_lookup_fence_object(priv, arg.handle);
- break;
- case drm_fence_unreference:
- ret = drm_user_object_unref(priv, arg.handle, drm_fence_type);
+ fence = drm_lookup_fence_object(file_priv, arg->handle);
+ drm_fence_fill_arg(fence, arg);
+ drm_fence_usage_deref_unlocked(&fence);
+
+ return ret;
+}
+
+
+int drm_fence_unreference_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ int ret;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ ret = 0;
+
+ if (!fm->initialized) {
+ DRM_ERROR("The DRM driver does not support fencing.\n");
+ return -EINVAL;
+ }
+
+ return drm_user_object_unref(file_priv, arg->handle, drm_fence_type);
+}
+
+int drm_fence_signaled_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ int ret;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ struct drm_fence_object *fence;
+ ret = 0;
+
+ if (!fm->initialized) {
+ DRM_ERROR("The DRM driver does not support fencing.\n");
+ return -EINVAL;
+ }
+
+ fence = drm_lookup_fence_object(file_priv, arg->handle);
+ if (!fence)
+ return -EINVAL;
+
+ drm_fence_fill_arg(fence, arg);
+ drm_fence_usage_deref_unlocked(&fence);
+
+ return ret;
+}
+
+int drm_fence_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ int ret;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ struct drm_fence_object *fence;
+ ret = 0;
+
+ if (!fm->initialized) {
+ DRM_ERROR("The DRM driver does not support fencing.\n");
+ return -EINVAL;
+ }
+
+ fence = drm_lookup_fence_object(file_priv, arg->handle);
+ if (!fence)
+ return -EINVAL;
+ ret = drm_fence_object_flush(fence, arg->type);
+
+ drm_fence_fill_arg(fence, arg);
+ drm_fence_usage_deref_unlocked(&fence);
+
+ return ret;
+}
+
+
+int drm_fence_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ int ret;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ struct drm_fence_object *fence;
+ ret = 0;
+
+ if (!fm->initialized) {
+ DRM_ERROR("The DRM driver does not support fencing.\n");
+ return -EINVAL;
+ }
+
+ fence = drm_lookup_fence_object(file_priv, arg->handle);
+ if (!fence)
+ return -EINVAL;
+ ret = drm_fence_object_wait(fence,
+ arg->flags & DRM_FENCE_FLAG_WAIT_LAZY,
+ 0, arg->type);
+
+ drm_fence_fill_arg(fence, arg);
+ drm_fence_usage_deref_unlocked(&fence);
+
+ return ret;
+}
+
+
+int drm_fence_emit_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ int ret;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ struct drm_fence_object *fence;
+ ret = 0;
+
+ if (!fm->initialized) {
+ DRM_ERROR("The DRM driver does not support fencing.\n");
+ return -EINVAL;
+ }
+
+ LOCK_TEST_WITH_RETURN(dev, file_priv);
+ fence = drm_lookup_fence_object(file_priv, arg->handle);
+ if (!fence)
+ return -EINVAL;
+ ret = drm_fence_object_emit(fence, arg->flags, arg->fence_class,
+ arg->type);
+
+ drm_fence_fill_arg(fence, arg);
+ drm_fence_usage_deref_unlocked(&fence);
+
+ return ret;
+}
+
+int drm_fence_buffers_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ int ret;
+ struct drm_fence_manager *fm = &dev->fm;
+ struct drm_fence_arg *arg = data;
+ struct drm_fence_object *fence;
+ ret = 0;
+
+ if (!fm->initialized) {
+ DRM_ERROR("The DRM driver does not support fencing.\n");
+ return -EINVAL;
+ }
+
+ if (!dev->bm.initialized) {
+ DRM_ERROR("Buffer object manager is not initialized\n");
+ return -EINVAL;
+ }
+ LOCK_TEST_WITH_RETURN(dev, file_priv);
+ ret = drm_fence_buffer_objects(dev, NULL, arg->flags,
+ NULL, &fence);
+ if (ret)
return ret;
- case drm_fence_signaled:
- fence = drm_lookup_fence_object(priv, arg.handle);
- if (!fence)
- return -EINVAL;
- break;
- case drm_fence_flush:
- fence = drm_lookup_fence_object(priv, arg.handle);
- if (!fence)
- return -EINVAL;
- ret = drm_fence_object_flush(dev, fence, arg.type);
- break;
- case drm_fence_wait:
- fence = drm_lookup_fence_object(priv, arg.handle);
- if (!fence)
- return -EINVAL;
- ret =
- drm_fence_object_wait(dev, fence,
- arg.flags & DRM_FENCE_FLAG_WAIT_LAZY,
- 0, arg.type);
- break;
- case drm_fence_emit:
- LOCK_TEST_WITH_RETURN(dev, filp);
- fence = drm_lookup_fence_object(priv, arg.handle);
- if (!fence)
- return -EINVAL;
- ret = drm_fence_object_emit(dev, fence, arg.flags, arg.type);
- break;
- case drm_fence_buffers:
- if (!dev->bm.initialized) {
- DRM_ERROR("Buffer object manager is not initialized\n");
- return -EINVAL;
- }
- LOCK_TEST_WITH_RETURN(dev, filp);
- ret = drm_fence_buffer_objects(priv, NULL, arg.flags,
- NULL, &fence);
- if (ret)
- return ret;
- ret = drm_fence_add_user_object(priv, fence,
- arg.flags &
+
+ if (!(arg->flags & DRM_FENCE_FLAG_NO_USER)) {
+ ret = drm_fence_add_user_object(file_priv, fence,
+ arg->flags &
DRM_FENCE_FLAG_SHAREABLE);
if (ret)
return ret;
- atomic_inc(&fence->usage);
- arg.handle = fence->base.hash.key;
- break;
- default:
- return -EINVAL;
}
- read_lock_irqsave(&fm->lock, flags);
- arg.class = fence->class;
- arg.type = fence->type;
- arg.signaled = fence->signaled;
- read_unlock_irqrestore(&fm->lock, flags);
- drm_fence_usage_deref_unlocked(dev, fence);
- DRM_COPY_TO_USER_IOCTL((void __user *)data, arg, sizeof(arg));
+ arg->handle = fence->base.hash.key;
+
+ drm_fence_fill_arg(fence, arg);
+ drm_fence_usage_deref_unlocked(&fence);
+
return ret;
}
projects / platform / upstream / libdrm.git / blobdiff